diff --git a/.github/workflows/management-recovery-epoch.yml b/.github/workflows/management-recovery-epoch.yml new file mode 100644 index 000000000..c2d21bd0d --- /dev/null +++ b/.github/workflows/management-recovery-epoch.yml @@ -0,0 +1,209 @@ +name: Management Recovery Epoch Acceptance + +'on': + pull_request: + +permissions: + contents: read + actions: read + +concurrency: + group: ${{ github.workflow }}-pr-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + acceptance-plan: + name: Select the complete native recovery acceptance matrix + runs-on: ubuntu-24.04 + timeout-minutes: 10 + outputs: + scenarios: ${{ steps.scenarios.outputs.scenarios }} + steps: + - name: Checkout exact reviewed source + uses: actions/checkout@v7 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: Setup Python + uses: actions/setup-python@v7 + with: + python-version: "3.12" + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + version: "0.7.22" + - name: Install locked test dependencies + run: uv sync --locked --dev --extra test + - name: Select every independent production scenario + id: scenarios + run: | + .venv/bin/python -B - <<'PY' + import json + import os + from scripts.test_management_recovery_epoch_host import ACCEPTANCE_SCENARIOS, required_cases + + assert len(ACCEPTANCE_SCENARIOS) == len(set(ACCEPTANCE_SCENARIOS)) + assert all(required_cases(scenario) for scenario in ACCEPTANCE_SCENARIOS) + with open(os.environ["GITHUB_OUTPUT"], "a") as stream: + stream.write("scenarios=" + json.dumps(ACCEPTANCE_SCENARIOS) + "\n") + PY + recovery-epoch: + name: Recovery epoch (${{ matrix.scenario }}) in isolated VM + needs: acceptance-plan + runs-on: ubuntu-24.04 + timeout-minutes: 150 + strategy: + fail-fast: false + max-parallel: 4 + matrix: + scenario: ${{ fromJSON(needs.acceptance-plan.outputs.scenarios) }} + env: + UV_LOCKED: "1" + PYTHONDONTWRITEBYTECODE: "1" + RECOVERY_FIXTURE: /var/tmp/ac-recovery-vm-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.scenario }} + APPROVED_AUTHORIZATION_ID: ci-recovery-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.scenario }} + RECOVERY_SCENARIO: ${{ matrix.scenario }} + steps: + - name: Checkout exact reviewed source + uses: actions/checkout@v7 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + - name: Setup Python + uses: actions/setup-python@v7 + with: + python-version: "3.12" + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + version: "0.7.22" + - name: Install locked test dependencies + run: uv sync --locked --dev --extra test + - name: Inspect the actual guest acceleration device + run: | + set -euo pipefail + if test -c /dev/kvm; then + stat --format='KVM device: %n mode=%a uid=%u gid=%g' /dev/kvm + else + echo 'No KVM device: the fresh QEMU guest will use bounded software CPU emulation.' + fi + - name: Build and verify exact installed management image + run: uv run python scripts/prepare_management_test_image.py + - name: Prepare the approved fresh recovery VM + env: + FIXTURE_READ_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + umask 077 + token_file="$RUNNER_TEMP/recovery-source-read-token" + trap 'rm -f "$token_file"' EXIT + printf '%s' "$FIXTURE_READ_TOKEN" > "$token_file" + docker pull caddy:2 + manager_id="$(docker image inspect --format '{{.Id}}' "$AUTOCODER_FRESH_TEST_BASE_IMAGE")" + caddy_id="$(docker image inspect --format '{{.Id}}' caddy:2)" + bash scripts/recovery_vm/prepare.sh "$RECOVERY_FIXTURE" "$APPROVED_AUTHORIZATION_ID" "$manager_id" "$caddy_id" "$token_file" "$RECOVERY_SCENARIO" + - name: Build bounded QEMU runner + run: docker compose --env-file "$RECOVERY_FIXTURE/compose.env" build + - name: Run actual original producers, containment and one-shot issuer + timeout-minutes: 125 + run: | + set -euo pipefail + docker compose --env-file "$RECOVERY_FIXTURE/compose.env" run --rm fixture + - name: Remove the ephemeral read credential + if: always() + run: | + rm -f "$RUNNER_TEMP/recovery-source-read-token" + rm -f "$RECOVERY_FIXTURE/input/source-token" + - name: Retain bounded native acceptance diagnostics + if: always() + uses: actions/upload-artifact@v7 + with: + name: management-recovery-epoch-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.scenario }} + path: /var/tmp/ac-recovery-vm-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.scenario }}/output/ + if-no-files-found: error + retention-days: 7 + verifier-omission-control: + name: Detect omitted post-consumption verification in isolated VM + needs: acceptance-plan + runs-on: ubuntu-24.04 + timeout-minutes: 150 + env: + UV_LOCKED: "1" + PYTHONDONTWRITEBYTECODE: "1" + RECOVERY_FIXTURE: /var/tmp/ac-recovery-vm-${{ github.run_id }}-${{ github.run_attempt }}-verifier-omission-control + APPROVED_AUTHORIZATION_ID: ci-recovery-${{ github.run_id }}-${{ github.run_attempt }}-verifier-omission-control + RECOVERY_SCENARIO: post-receipt-ready-mutation + steps: + - name: Checkout exact reviewed source + uses: actions/checkout@v7 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + - name: Setup Python + uses: actions/setup-python@v7 + with: + python-version: "3.12" + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + version: "0.7.22" + - name: Install locked test dependencies + run: uv sync --locked --dev --extra test + - name: Inspect the actual guest acceleration device + run: | + set -euo pipefail + if test -c /dev/kvm; then + stat --format='KVM device: %n mode=%a uid=%u gid=%g' /dev/kvm + else + echo 'No KVM device: the fresh QEMU guest will use bounded software CPU emulation.' + fi + - name: Build and verify exact installed management image + run: uv run python scripts/prepare_management_test_image.py + - name: Prepare the approved fresh recovery VM + env: + FIXTURE_READ_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + umask 077 + token_file="$RUNNER_TEMP/recovery-source-read-token" + trap 'rm -f "$token_file"' EXIT + printf '%s' "$FIXTURE_READ_TOKEN" > "$token_file" + docker pull caddy:2 + manager_id="$(docker image inspect --format '{{.Id}}' "$AUTOCODER_FRESH_TEST_BASE_IMAGE")" + caddy_id="$(docker image inspect --format '{{.Id}}' caddy:2)" + bash scripts/recovery_vm/prepare.sh "$RECOVERY_FIXTURE" "$APPROVED_AUTHORIZATION_ID" "$manager_id" "$caddy_id" "$token_file" "$RECOVERY_SCENARIO" omit-post-consumption-verifier + - name: Build bounded QEMU runner + run: docker compose --env-file "$RECOVERY_FIXTURE/compose.env" build + - name: Detect the exact omitted verifier with the unchanged native oracle + timeout-minutes: 125 + run: | + set -euo pipefail + docker compose --env-file "$RECOVERY_FIXTURE/compose.env" run --rm fixture + - name: Remove the ephemeral read credential + if: always() + run: | + rm -f "$RUNNER_TEMP/recovery-source-read-token" + rm -f "$RECOVERY_FIXTURE/input/source-token" + - name: Retain the distinct mutation-control evidence + if: always() + uses: actions/upload-artifact@v7 + with: + name: management-recovery-epoch-${{ github.run_id }}-${{ github.run_attempt }}-verifier-omission-control + path: /var/tmp/ac-recovery-vm-${{ github.run_id }}-${{ github.run_attempt }}-verifier-omission-control/output/ + if-no-files-found: error + retention-days: 7 + acceptance: + name: Management Recovery Epoch Acceptance + needs: [acceptance-plan, recovery-epoch, verifier-omission-control] + if: always() + runs-on: ubuntu-24.04 + timeout-minutes: 2 + steps: + - name: Require every native scenario to succeed + env: + PLAN_RESULT: ${{ needs.acceptance-plan.result }} + ACCEPTANCE_RESULT: ${{ needs.recovery-epoch.result }} + MUTATION_CONTROL_RESULT: ${{ needs.verifier-omission-control.result }} + run: | + test "$PLAN_RESULT" = success + test "$ACCEPTANCE_RESULT" = success + test "$MUTATION_CONTROL_RESULT" = success diff --git a/Dockerfile b/Dockerfile index d44114f88..0380736b2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,7 +42,7 @@ RUN groupadd --gid 65532 auto-coder-worker && useradd --uid 65532 --gid 65532 -- COPY --from=python-environment /opt/auto-coder/.venv /opt/auto-coder/.venv ENV PATH="/opt/auto-coder/.venv/bin:/home/node/.opencode/bin:${PATH}" # Embed the trusted build input so the installed artifact reports it without Docker/Git/network access. -RUN python -m auto_coder.build_provenance embed +RUN python -B -m auto_coder.build_provenance embed COPY --from=build /usr/local/bin/opencode /usr/local/bin/opencode COPY --from=build /usr/local/bin/codex /usr/local/bin/codex WORKDIR /workspace diff --git a/docs/client-features/management-current-containment-observation.md b/docs/client-features/management-current-containment-observation.md index 0b30b65c9..80722d32e 100644 --- a/docs/client-features/management-current-containment-observation.md +++ b/docs/client-features/management-current-containment-observation.md @@ -275,6 +275,17 @@ fresh reads. A process running a scheduler outside its normal service also refuses. This initial profile conservatively refuses unrelated masked services that retain work as well. +A process or thread may disappear while the complete procfs inventory is read. +Only disappearance independently confirmed by a fresh PID/TID inventory, or a +changed complete inventory, permits a retry. At most three attempts are made; +each discards the incomplete pair and reads two new complete matching +source/controller/cgroup/task snapshots. Earlier protected source, controller, +boot and mask bindings remain refusal anchors across attempts. A still-listed +process or thread with missing evidence, an empty live task inventory, permission +or malformed-data errors, changed protected facts, and any positively observed +inhibited task or scheduler still refuse immediately. No task is skipped to +complete a positive observation, and persistent inventory churn remains unavailable. + A detected remaining task has a typed refusal at the route boundary. Its public diagnostic contains fixed text and a bounded validated controller-scope identifier so an independent consumer can distinguish the canonical and differently named diff --git a/docs/client-features/management-fresh-initialization.md b/docs/client-features/management-fresh-initialization.md index e79ae6cf2..8679d64e6 100644 --- a/docs/client-features/management-fresh-initialization.md +++ b/docs/client-features/management-fresh-initialization.md @@ -30,6 +30,10 @@ container profiles with the expected mounts are accepted. The evidence file is created once in the new owner-only guard directory. Access errors are refusal, including inability to inspect an old process namespace. +Supervisor-ledger observations close their read-only SQLite connections after +each read, including refusals. They retain no database handle beyond that check; +the owner fences and namespace descriptors have their separate required lifetimes. + The host commands accept only explicit, private files and exact container IDs: ```sh diff --git a/docs/client-features/management-original-attempt-provenance.md b/docs/client-features/management-original-attempt-provenance.md index d47c1b87c..18b561c91 100644 --- a/docs/client-features/management-original-attempt-provenance.md +++ b/docs/client-features/management-original-attempt-provenance.md @@ -23,6 +23,13 @@ its current fences; it must separately decide whether that explicitly bounded coverage supplies the history its own contract requires. Neither enrollment nor a matching evidence object supplies recovery or launch authority. +The separate `ordinary-original-genesis-v1` profile described in +[Original ordinary deployment genesis and retained lineage](management-original-genesis.md) +adds an affirmative original root-creation and ordinary first-launch baseline. +It uses `ordinary-original-history-v1` authority and the independent +`derive_lineage`/`verify_lineage` API. It does not change this prospective-only +profile's pre-enrollment gap or convert any earlier deployment. + ## Original producer and lifetime An operator must separately provision a new canonical mode-0700 history directory diff --git a/docs/client-features/management-original-genesis.md b/docs/client-features/management-original-genesis.md new file mode 100644 index 000000000..2a81b6c6c --- /dev/null +++ b/docs/client-features/management-original-genesis.md @@ -0,0 +1,202 @@ +# Original ordinary deployment genesis and retained lineage + +The explicitly selected `ordinary-original-genesis-v1` producer supplies the +prospective original baseline required by the recovery epoch's lineage reader. +It implements the original-evidence responsibility in #2839. It creates no +recovery epoch, issues no recovery grant, and does not authorize a deployment. +The existing ordinary bootstrap is still the separately authorized process +that launches its configured management child. + +An original deployment cannot be inferred from an empty old directory or a +currently stopped Docker object. This profile therefore has two distinct +original capture boundaries before the ordinary bootstrap can run: + +1. `prepare_original_genesis_roots` consumes a protected explicit new-deployment + authorization. It first creates an exclusive original claim, then creates + the state, management-operation journal, OAuth and history directories with + `mkdir` and no `exist_ok`. Every preexisting path is refused. The original + root receipt binds the actual directory inodes, the selected protected + configuration files, producer source and Linux process lifetime. A partial + claim remains unavailable and is never repaired or retried. +2. `capture_original_genesis` consumes the separate final authorization naming + the exact manager and paired proxy Docker IDs. It requires both objects to + be `created`, with zero PID, zero restarts and never-started timestamps. It + compares the complete local-engine inventory, effective original commands, + protected configuration mounts, original root identities, and the complete + installed Python package with the executing reviewed producer source. The + selected manager and proxy may not have other discovered management routes. + This capture reads Docker only; it does not create, start, stop or reconfigure + Docker objects. + +These observations do not convert a legacy baseline into original evidence. +The root creation receipt must already exist, and its authorization and +configuration identities must match the final selection. Existing or partial +history, a reused authorization claim, replaced roots, changed source, +conflicting original files or a previously started object remain unavailable. + +## Explicit source and configuration selection + +`BootstrapConfig.original_genesis` is optional. Selecting this producer requires +its exact value `/bootstrap/original-genesis.json`, the ordinary bootstrap +command and the fixed installed management console entrypoint. It cannot be +combined with fresh initialization, NEW initialization, inherited exclusions +or the installed fresh-runtime selector. Omitting it preserves existing +ordinary behavior. NEW/#2672 and ordinary fresh/upgrade refusal behavior are +unchanged. + +The selected ordinary manager has these original mounts: + +| Role | Container path | Access | +| --- | --- | --- | +| Original bootstrap configuration and genesis files | `/bootstrap` | Read-only | +| Original child, handoff and recovery stores | `/management-child-state` | Read/write | +| Original management-operation journal | `/management-operations` | Read/write | +| Originally created OAuth root | `/management-oauth` | Read/write | +| Original history authority and lock | `/management-original-history` | Read-only | + +The same mapped inodes are independently observed by the host and bootstrap. +The fixed configuration members are `/bootstrap/config.json` and +`/bootstrap/management.yaml`. The paired proxy has the ordinary Caddy command, +its exact protected configuration bind mount, and the manager's network +namespace. All original files are owner-only regular files with one link; +protected original directories have mode `0700`. + +Capture parses the effective management settings, rather than accepting only a +configuration hash. Its OAuth root, operation journal root, bootstrap path and +listener must select those same original objects. The mounted bootstrap repeats +that check before diagnostics can create a missing original directory and before +any native store or launch record. It also verifies the selected initial +executable receipt, the installed management console wrapper and its supported +environment. A missing original root or an executable environment hook refuses +without reconstructing the missing state. + +The existing ordinary successor capture keeps its strict single Caddyfile bind +profile. The integrated genesis fixture copies the verified Caddy binary from +its exact imported source image into an image without inherited data volumes, +then creates the proxy with `/srv` as its working directory and only the +read-only Caddyfile bind. Extra `/data` or `/config` mounts are not admitted by +that existing capture path. Stock Caddy's separate current-containment capture +does not relax this contract. A genesis predecessor carries the same read-only +`/management-original-history` mount into successor capture, whose host reader +verifies that binding against the original root authority. + +Root authorization JSON contains exactly the fields of +`GenesisRootAuthorization`: `profile` (`ordinary-original-roots-v1`), `purpose` +(`separately-authorized-new-deployment`), `deployment_id`, canonical UUID +`authorization_id`, `bootstrap_path`, `management_path`, `proxy_config_path`, +`state_root`, `journal_root`, `oauth_root`, and `history_root`. The four root +paths must not yet exist, and their parent directories must already be +protected. Configurations and authorization files must already be protected. +The writer records `original-genesis-root-claim.json` and +`original-genesis-roots.json` beside the bootstrap configuration. + +The later `GenesisSelection` has `profile` (`ordinary-original-genesis-v1`), +`purpose`, `deployment_id`, `authorization_id`, `management_id`, `proxy_id`, +`history_root`, `bootstrap_path`, `management_path`, `proxy_config_path`, and +`root_authority_path`. Its deployment, authorization, configuration and history +scope must match the actual original root authority. Docker IDs are exact +64-character IDs, not names or mutable tags. Both inputs use canonical JSON. + +The host-root CLI stages are: + +```sh +python -m auto_coder.management.original_genesis \ + --prepare-roots /protected/new-deployment-root-authorization.json \ + --log-file /protected/logs/original-root-capture.log + +# The separately authorized deployment workflow creates the exact ordinary +# manager and proxy with the original roots, then writes the final selection. +python -m auto_coder.management.original_genesis \ + --authorization /protected/new-deployment-final-authorization.json \ + --log-file /protected/logs/original-genesis-capture.log +``` + +This module does not perform the deployment workflow represented by the comment. +Loguru writes producer diagnostics to the selected file and stdout. Log files +must be outside the pristine root namespaces during host capture. + +## Native bootstrap boundaries + +Before any child or management effect, the actual `_run_owned` ordinary path +opens its original owner file independently and obtains its exclusive kernel +flock. Its first owner uses exclusive file creation; an interrupted first +owner cannot be reclassified as an untouched root. The bootstrap obtains a +shared history flock through the original read-only mount, verifies the exact +original authority and genesis, checks current source/configuration/root +identities, and requires the original history to remain empty for its first +native initialization. + +The existing native initializers create all four original SQLite stores under +an owner-only umask. The bootstrap durably records their exact inodes in +`original-genesis-stores.json`, followed by +`original-genesis-first-launch.json`, which binds the exact first operation, +attempt, producer lifetime and native receipt. The normal ordinary launch path +uses those same operation and attempt IDs. These records describe original +boundaries; neither record asserts that a child actually launched or that the +application became ready. + +An interruption before a required native record remains unavailable. The +bootstrap cannot recreate missing originals or replay the first owner. An +originally recorded but unresolved first launch retains its unknown outcome. +Normal store and unknown-state refusal rules continue to apply on later +bootstrap reconstruction. + +## Full retained lineage API + +`original_attempts.read_frontier` recognizes `ordinary-original-history-v1` +authority in addition to the existing prospective fresh-attempt authority. +The new authority pins the exact original genesis file, which fixes the +baseline instance and Docker object before the first successor exists. +Every supported later ordinary fresh, upgrade and exclusion-cutover host +preflight must preserve that same history binding. A successor cannot select +an arbitrary first predecessor or reuse a baseline protected root. All +original failed branches remain in the append-only frontier. + +`original_attempt_evidence.derive_lineage(binding, history_fd, owner_fences)` +and `verify_lineage(submitted, binding, history_fd, owner_fences)` independently +read original genesis, every retained frontier event and every expected native +store. The caller holds its own current history shared flock and exactly one +current shared owner flock for each native generation. A never-created branch +has its affirmative original terminal boundary and no invented owner file. +The reader joins each native successor to its predecessor's exact owner inode +and the original retained OAuth root. + +The frozen `OriginalLineageEvidence` exposes original `genesis`, `attempts`, +`generations`, `originals`, `frontier`, and `unavailable` reasons. Each +`OriginalGeneration` includes the exact management/source/configuration/root +scope, native owner identity, expected stores and factual `outcome`. The +original baseline also pins its proxy identity. A fresh successor's proxy is +not invented from an intent that never recorded one; current containment must +independently establish that pair. + +Complete original evidence with an `unknown` or unresolved application outcome +is distinct from missing original evidence. For example, the native store +receipt is produced after the original first-launch claim, so a missing claim +is a provenance gap even if a status reader can report `unknown`. The lineage +reader preserves the unknown and every retained native record. It never +checkpoints SQLite, clears unknowns, repairs evidence, rewrites exclusions, or +issues launch authority. A nonempty unresolved WAL or rollback journal remains +unavailable. Existing `derive`/`verify` prospective-reader behavior is preserved. + +## Verification boundaries + +`tests/test_management_original_genesis.py` exercises actual protected +filesystem publication, affirmative root creation, original claim refusal, +root/source/configuration pinning, actual native store initialization under an +exclusive owner flock, shared-fence independent derivation, preserved unknown +outcomes, complete failed branches, and missing-original refusal. Docker +observations and container bind mapping are explicit component substitutes; +these tests are not real-Docker, first-child or recovery-epoch acceptance. + +`tests/test_management_original_genesis_cli.py` invokes the actual bootstrap CLI +against a missing original state root and proves that logger setup does not +recreate it. The component selection also checks effective configuration mismatches +and executable/environment failures before native publication. The additive +predecessor binding and strict proxy mount checks are exercised in +`tests/test_management_original_genesis_capture.py`. + +The production interfaces are suitable for isolated real-Docker and host-root +acceptance. Full #2824 acceptance additionally requires the actual complete +current containment verifier, issuer, OS descriptor transfer, consumer receipt +and durable epoch boundaries. Component provenance tests do not substitute for +that integrated acceptance. diff --git a/docs/client-features/management-recovery-containment-observation.md b/docs/client-features/management-recovery-containment-observation.md new file mode 100644 index 000000000..95a545d1d --- /dev/null +++ b/docs/client-features/management-recovery-containment-observation.md @@ -0,0 +1,228 @@ +# Current containment of an ordinary recovery scope + +Issue #2840 adds a read-only `ordinary-recovery-v1` observation profile for a +separately approved set of original ordinary management generations and one +exact prelaunch recovery consumer. It supplies current facts to the recovery +issuer. It does not establish original-history completeness, repair an original, +issue a grant, create operation admission, start a child or assert readiness. +The original `ordinary-v1`, `quarantined-original-v1`, fresh replacement and NEW +admission profiles keep their existing contracts. + +## Fixed authority and finite selection + +The production reader loads only +`/var/lib/auto-coder/approved-recovery-containment-enrollments/.json`. +The directory must be canonical, root-owned and mode 0700; the file must be a +canonical, root-owned, single-link mode-0600 regular file. No request or environment +variable can substitute the enrollment root. A separate trusted operator installs +approval; the observer cannot create, repair or replace it. + +The exact schema is `approved-management-recovery-containment-v1`. Its fields are +`deployment_id`, `enrollment_id`, `authorization_id`, `profile`, `engine_id`, +`source_identity`, `originals`, `target`, `watchers`, and +`approved_observation_only: true`, together with `schema`. Unknown fields, aliases, +unsorted membership and incompatible versions refuse. The typed definitions and +strict parsers are in `recovery_containment_model.py`. + +| Selection | Independently bound information | +| --- | --- | +| Original generation | Generation and instance IDs; exact manager/proxy IDs and images; bootstrap, management and Caddy configuration file identities; complete actual bind roots; current-state selection | +| Started original | Original boot ID; protected original manager/proxy lifetime capture; exact original owner file; live original namespace holder and its actual retained descriptor | +| Never-started original | Existing Docker objects currently in the pristine `created` state; no invented owner, process lifetime, prior record or namespace pin | +| Target | Exact separate manager/image/instance; protected bootstrap and management configuration; complete bind roots; prepared owner inode; distinct loopback published port | +| Watcher | Exact installed systemd unit and retained `unit_source` file identity, actual installed module and protected configuration, and the existing `proxy-watch.lock` that the real watcher takes before tick/reconcile | + +There are at most 16 original pairs and 32 watchers. Their complete eventual peer +custody set must also satisfy `3 + 2 × started originals + watchers <= 64`; the +three fixed resources are history, target owner and deployment epoch. Original +pairs, target, owner files and controller files cannot alias. Reusing one proxy +object across multiple enrolled generations is outside this finite profile. + +A lineage branch with positive original never-created evidence and no current +Docker object is not fabricated as a containment pair. The issuer joins that +original history to absence from the independently enumerated whole engine. Any +actual branch object remains subject to complete selection and profile checks. +Current `created` state alone supplies no historical never-created certificate. + +## Actual executable source and configuration + +The observer copies the complete installed `auto_coder` package from the exact +selected manager objects on the local system Docker engine. It reads a bounded +uncompressed archive without executing it. Every Python source filename and byte +sequence contributes to the same length-framed identity as +`original_attempts.source_identity()`. The protected enrollment, current installed +observer and every selected manager must agree on that identity. + +Absolute or ambiguous archive paths, duplicate members, links, special files, +missing original-history modules, bytecode and native import extensions refuse. +Package `.pyc`, `.pyo`, `.so` or `.pyd` files cannot substitute different executable +bytes for the observed Python source. The supported container environment sets +`PYTHONDONTWRITEBYTECODE=1` before the first bootstrap import. Existing cached +package bytecode remains unsupported; setting the environment later cannot make +it acceptable. The production Dockerfile embeds build provenance with `python -B` +to avoid introducing package bytecode during that build step. + +Effective commands must be the actual ordinary bootstrap entrypoint, the ordinary +fresh entrypoint for an original successor, or the target's explicit +`--recovery-epoch` entrypoint. Originals using fresh initialization keep their +unknown or failed historical outcome; this observation path never requires or +invents a successful fresh outcome. Their historical justification belongs to the +independent lineage producer and issuer join. + +The mount reader independently checks the full current Docker inspection, +protected source/configuration bytes, actual bind root device/inode/mode, Python +resolution, runtime command and effective configuration mapping. Interpreter, +package and working-directory shadowing refuse. Original genesis uses the fixed +read-only `/management-original-history` mapping and +`/bootstrap/original-genesis.json` input. The target uses the fixed +`/bootstrap/recovery-epoch.json` input and a separately checked installed runtime; +its version root is read-only at the same absolute host/container path. The issuer +owns the additional tested-source and READY verification. + +The target's state and journal roots are separate from every original mutable +root. Its OAuth root remains the exact original OAuth path and object. The new +published endpoint must differ from every original published port, so a target +listener cannot hide an old listener. The actual target listener must be the +local Docker userland proxy for its exact container address and endpoint. + +The target's read-only custody mappings are `/management-recovery` from the fixed +host `/var/lib/auto-coder/recovery-epoch-peer`, the exact history lock at +`/management-recovery-history/history.lock`, and the independently selected old +owner files at `/management-recovery-originals//owner.lock`. Indices follow +the canonical ordered original list. Their actual device/inode and configured +roles must agree. These inputs do not themselves establish acquired custody. + +## Complete current controllers and kernel facts + +Whole-engine enumeration is unconditional. Every selected object must exist on +the exact local engine. Unselected objects sharing selected images, management +commands, selected namespaces or original bind sources remain management +candidates and refuse. Docker endpoint overrides, foreign engines, changing +inventories and unsupported profiles refuse. Ordinary unrelated objects remain +visible in the complete engine evidence. + +The sibling `.routes.json` must use +`approved-management-recovery-routes-v1`, bind the exact enrollment hash, and carry +separate explicit complete-route approval. It reuses the existing exhaustive +system/user systemd, activation, cron/at and installed-script readers over the +union of all original and target IDs and actual roots. Every restarter must be +technically inhibited; restart policies must be `no`. Unknown exclusions and +unsupported remote origins cannot be converted into closure by an operator list. +Each real watcher must use its enrolled source/configuration and actual existing +controller lock. The observer changes no unit, restart policy, file or controller. + +A masked effective unit has no executable command inventory. Recovery therefore +requires a separate original `unit_source: RecoveryFile` for each watcher. The +reader queries the actual system manager's `UnitPath`, verifies one protected +mask above the selected regular fragment, and refuses competing fragments, +aliases, or unit-specific, dash-prefix and type-wide drop-ins. The retained unit +must use the supported simple-service profile with `Restart=no` and exactly the +current observer's literal interpreter path followed by +`-I -B -m auto_coder.management.proxy_watch_cli --config `. +Another virtual environment sharing the Python binary is a different selection. +Environment, working-directory, root-directory and additional executable +directives are unsupported. The observer independently repeats the fragment and +load-path reads while retaining the existing actual mask, current task and +controller-lock requirements. Its non-authorizing watcher facts retain this +source evidence; no admission, dispatch or readiness event is added. + +For each started original, the observer checks the exact captured manager/proxy +lifetimes and mount identities, every remaining cgroup task, absence of captured +escaped tasks, the whole host's task references to the retained original network +namespace, and absence of the old host listener. Stopped Docker state or an old +namespace number is insufficient. The original boot must still match, and a live +holder must still retain the genuine original namespace object. + +`RecoveryResources` names a current host holder by boot, PID and process start +token. Its `original_owner_fds`, `controller_fds` and `namespace_fds` name exact +existing descriptors. Original names use the enrolled `generation_id`; controller +names use the exact watcher unit. The observer independently reopens procfs +references, holds observation references, checks protected paths and file objects, +reads kernel flock records, and pins holder lifetimes while it samples. + +Every original-owner flock must be SH and independently attributed to that issuer +PID. An inherited parent's SH descriptor is refused. Every real watcher lock must +be held EX by that same issuer. The target separately holds its own owner EX under +its actual prelaunch process. File replacement, released locks, reused descriptor +numbers, holder exit and missing resource membership refuse. No observation claims +that these present fences existed during the historical gap. + +## Library and CLI + +The production library exposes: + +- `load_enrollment(enrollment_id) -> RecoveryEnrollmentObservation` +- `observe(enrollment_id, resources) -> RecoveryContainmentReport` +- `verify(enrollment_id, previous, resources) -> RecoveryContainmentReport` + +Missing or unsupported facts raise `RecoveryContainmentUnavailable`. A positive +report has status `observed-current-recovery-containment`, typed complete facts, +a fresh observation ID and boot/process sampling interval, and a deterministic +`evidence_sha256` over stable scope/source/resource/controller/kernel facts. +Diagnostic observer PID and sampling timestamps are outside that stable digest. +`grant_issued`, `history_asserted` and `admission_created` are always false. + +```sh +python -m auto_coder.management.recovery_containment observe \ + --enrollment separately-approved-recovery \ + --resources /private/current-issuer-descriptors.json +python -m auto_coder.management.recovery_containment verify \ + --enrollment separately-approved-recovery \ + --resources /private/current-issuer-descriptors.json \ + --previous /private/previous-current-observation.json +``` + +CLI success exits 0. Unavailable evidence exits 2 with a structured unavailable +result. A resource input only identifies existing descriptors; supplying a number +cannot create a lock, pin or approval. Reports expose hashes and identities rather +than configuration contents. Logger diagnostics may also appear on stdout, so a +machine caller selects the structured JSON result line. + +Unavailable CLI results and Loguru stdout/file diagnostics also retain at most +eight explicit exception causes, limited to type, integer error number and safe +code basename/function/line. They omit inner messages, arguments, full paths and +frame locals. The native fixture preserves validated causes in its separate +`containment_failure` result field, so truncating ordinary diagnostic log lines +cannot erase the causal locations. This metadata never changes a refusal, +acceptance predicate, report authority or exit status. + +The separate verifier repeats real source, kernel, controller and holder reads. +It rejects changed facts, malformed or authorizing reports, invalid digests, +foreign boots, future samples and samples older than ten minutes. The earlier +read-only sampling command may have exited; its PID is attribution. Actual issuer +and namespace-resource producers must remain alive and independently inspectable. +A saved positive result never replaces a current check or establishes continuity. + +## Reproduction and observability + +Component tests use the required repository runner: + +```sh +CI=true UV_NO_SYNC=1 PYTHONPATH=.:src bash scripts/test.sh --ci-tests-only \ + tests/test_management_recovery_containment.py \ + tests/test_management_recovery_containment_resources.py +``` + +The parser/source/profile tests are component evidence. The resource tests use real +independent processes, SH/EX flocks, namespace descriptors, process death and file +replacement. Neither suite by itself proves whole-host route or Docker closure. + +The native recovery harness uses the existing separately provisioned disposable +KVM/QEMU VM authorization, initial host namespaces, PID 1 systemd and the local +system Docker engine. `scripts/recovery_epoch_containment_fixture.py` captures +live original generations before stopping them, retains genuine namespace FDs, +runs and inhibits actual installed watchers, and constructs the separately +approved finite fixture scope. It imports production observation readers and +creates no substitute successful evidence. The production observer does not +import this fixture helper. Native runs must include genuine lineage and failed +branches, the prelaunch target, actual unrelated workers and preserved original +records; a component-only pass cannot be reported as native acceptance. + +Observation logging uses Loguru's configured file and stdout sinks. Its explicit +fields are `observation_only=True`, `recovery_grant=False` and +`admission_created=False`. No operation journal admission, dispatch, recovery +success or readiness event is emitted. The native harness verifies original +journal/state and unrelated worker lifetime/request fingerprints across the +production entrypoints. Diagnostic sampling output is not an operational success +row in the dashboard; the shared dashboard observability documentation records +this distinction. diff --git a/docs/client-features/management-recovery-epoch-design.md b/docs/client-features/management-recovery-epoch-design.md new file mode 100644 index 000000000..3d9c7c5db --- /dev/null +++ b/docs/client-features/management-recovery-epoch-design.md @@ -0,0 +1,400 @@ +# Management recovery epoch integration design + +Issue [#2824](https://github.com/kitamura-tetsuo/auto-coder/issues/2824) owns a +new host ownership epoch after continuous exclusion was lost. Its required +result is a current, exact-target, one-use grant composed from independently +verified original history, current containment and retained fences. This +fragment records implementation boundaries found at source +[`e742338b3f311e6d30a22a8f5341b8072513d813`](https://github.com/kitamura-tetsuo/auto-coder/commit/e742338b3f311e6d30a22a8f5341b8072513d813). +This is the retained audit of that original source, not a recovery procedure or +live-operation authority. The implemented producer and its current interfaces are +described in [management-recovery-epoch.md](management-recovery-epoch.md), +[management-original-genesis.md](management-original-genesis.md), and +[management-recovery-containment-observation.md](management-recovery-containment-observation.md). +The missing-capability tables below describe the audited base and explain why +the two additional evidence-only prerequisites were necessary. + +An unused state API, component test, operator inventory or saved prerequisite +verdict cannot satisfy the full Issue. Existing unknown-outcome refusals remain +correct. No original claim, missing journal, operation key, target root or +initialization outcome is repaired or reset by this design or implementation. + +## Existing authoritative producers + +The prerequisite implementations have useful, distinct contracts: + +| Producer and verifier | Existing fact | Limit relevant to the recovery issuer | +| --- | --- | --- | +| `original_attempts.enroll`, `capture_claim`, `grant_boundary`, `read_frontier` | Protected prospective attempt anchors and a serialized, append-only history frontier. | Enrollment starts a new prospective scope. It does not capture the origin of a previously running baseline deployment. | +| `original_attempt_evidence.derive` and `verify` | Exact original attempt/store bytes and identities, reverse successor enumeration within the recorded scope, current fences, original terminal never-created facts. | The first anchor's old instance is an external baseline. The verifier also refuses original outcomes other than `succeeded`; it does not expose a complete-but-unknown recovery evidence contract. | +| `quarantine_history.load_manifest` and `QuarantineHistory` | Read-only inspection of preserved operation journals and their key/target exclusions. | An inspection manifest is a selection, not complete lineage authority. | +| `containment_observation.observe` and `verify` | Independently repeated installed-route, Docker, source, configuration, kernel, listener and enrollment observations for one supported manager/proxy pair. | It requires the current single-pair profile and an independently held exclusive original owner fence. Additional management-like objects are refused. | +| `fresh_begin.begin_new_instance` and `inspect_new_instance_origin` | A durable, source/root-bound NEW origin before first launch. | `NewInstanceRequest` has no original-history authority binding. NEW original start requires the separately held private-engine profile. | +| `fresh_host_observation.serve_one_fresh_grant` | An authenticated local peer joined to a selected container, original shared fence and supervisor lifetime. | It is the existing fresh protocol, not a deployment-wide recovery epoch issuer, and it has no complete controller/deployment fence-transfer receipt. | + +The declarations above describe actual functions, not replacement interfaces. +The implementation of #2824 must invoke and reverify the applicable original +producers. Copying their serialized output into a new receipt does not enlarge +what that output proves. + +## Demonstrated integration gaps + +### A prospective frontier does not contain its baseline's origin + +`original_attempt_evidence.derive` selects the first anchor's +`(old_instance, old_container_id)` as `baseline`. Later unanchored predecessors +must match that pair, but no corresponding attempt is created or derived for +that baseline. `original_attempts.enroll` does not inspect prior activity and +correctly describes itself as beginning a prospective scope. + +The observable counterexample is an original manager A, an earlier unrecorded +failed successor X, later enrollment L, and a recorded A-to-B attempt. L can +correctly enumerate all attempts captured after its enrollment, but the +unchanged A-to-B frontier contains no evidence deciding whether X existed. +Backward traversal from B and present stopped-state inspection cannot supply +that absent reverse completeness. + +A genuine NEW origin is useful evidence about A, but its current request does +not bind L before A's first-start authority. Joining A's NEW receipt to a later +L enrollment still leaves the uncovered interval. The join must be recorded by +an actual original admission producer, before the earliest relevant launch or +attempt, and enforced by every supported successor admission route. A new +recovery record cannot backfill that event. + +There is a second, separate history distinction. With complete original store +receipts, `derive` still adds an unavailable reason when +`read_fresh_status(..., host=True)` is not `succeeded`, and `verify` refuses any +unavailable reason. A recovery composition cannot discard that refusal text. +If the supported recovery profile includes validly recorded unknown outcomes, +its authoritative evidence contract must distinguish complete original facts +with an unresolved outcome from missing or unverifiable original facts. Both +retain their original outcome; neither becomes a launch permission. Missing +expected stores continue to refuse unless the original producer's affirmative +terminal never-created boundary proves that exact exception. + +### Current containment's owner lock conflicts with required shared custody + +`containment_kernel_subobservation._external_owner_fence` requires an external +holder distinct from the observer and verifies a kernel `WRITE` flock on the +exact original owner file. REQ-104 requires the recovery issuer to hold a newly +acquired independent shared lock on every original owner file while it verifies +containment at the issuance boundary. Those two locks cannot coexist on the +same Linux file. + +An isolated kernel probe using a real exclusive holder and a separate process +that independently opens the same file observed `LOCK_SH | LOCK_NB` refuse +while the exclusive lock was held, then acquire after its release. This is +positive evidence of the composition conflict, not Docker or recovery +acceptance. Duplicating or inheriting the exclusive holder's descriptor would +not establish the independent shared acquisition required by REQ-104. + +A recovery observation profile therefore needs a defined, independently +verifiable shared-fence contract. It must recheck the exact file, kernel owner, +process/boot lifetime and current custody without weakening the existing +exclusive-fence profile or asserting coverage of the earlier gap. + +### Single-pair, source and engine profiles do not currently join + +`containment_observation.observe` refuses a management/proxy-like Docker object +outside its selected pair as `docker-alternative-management`. A recovery +frontier can contain multiple retained ancestor and failed-successor objects; +the selected consumer may also exist when the grant is issued. Filtering those +objects out before calling the verifier would hide exactly the scope that +REQ-101 through REQ-103 require it to establish. + +`installed_bootstrap_source.inspect_installed_bootstrap_source` recognizes two +explicit original bootstrap digests: + +- Ordinary original: `66bcfefa86aa2bd88dd4a253405fc6faac65675917e8d393e587d5a62fff83bb`. +- Quarantined original: `6a7da1f8f57aab76acb062a911598f75f4ea0b237308389f3807bc6368785e59`. + +A matching image name, caller digest, current checked-out source or successful +original-history test is not another recognized source profile. Meanwhile, +`original_attempt_evidence.derive` requires the enrolled source identity to +match `original_attempts.source_identity()` of the verifying package. The +supported set of original source profiles must be explicit; it cannot be +inferred from unrelated component success. + +Finally, `fresh_new_host_start.start_new_docker_once` requires an independently +verified private Docker Engine and its recorded `DOCKER_HOST` socket. Current +containment source and route discovery reject `DOCKER_HOST`/`DOCKER_CONTEXT` +overrides and inspect the local system engine. Running one verifier against the +outer engine and another against an unbound inner engine does not prove a +single deployment's ownership or controller scope. + +## Minimal prerequisite capabilities proposed for review + +These capability boundaries preserve #2824's Objective and Requirements. +The existing recovery issuer remains responsible for REQ-101 through REQ-108; +extracting prerequisites neither supplies live-operation authorization nor +waives its integrated obligations. + +**Prospective deployment genesis and complete lineage bridge.** Establish an +original, protected binding between a never-before-launched deployment origin +and its attempt history before any supported first-start or successor authority +can escape. The producer must carry that binding through ordinary and failed +successors and refuse a missing or advanced frontier. Its independent verifier +must prove the baseline and all recorded successor branches from original +facts, preserving unknown outcomes and distinguishing incomplete facts from +known unresolved outcomes. The output remains evidence-only. Historical +pre-enrollment deployments and unsupported source profiles remain unavailable; +there is no legacy conversion. + +**Current containment for a complete recovery deployment.** Independently +observe the authorized set of original generations, their proxies and all +ordinary controller routes against the same engine/source profile as that +history. Define issuer-held independent SH fences and separately retained +controller/deployment custody, preserve exact current namespace/resource pins, +and repeat the observations in the actual issuer invocation. An additional +container is accepted only when its exact role and original or target binding +are verified; an omitted or unrecognized controller still refuses. This adds +a recovery-specific contract without relaxing ordinary fresh admission or the +quarantined-original distinction. + +These are independent producers. The lineage bridge can produce correct +non-authorizing original evidence without a recovery grant. Current +containment can produce correct non-authorizing observations without issuing +or consuming an epoch. Their state/lifetime and runtime authorities differ. +The host issuer should compose their defined outputs at one durable issuance +boundary rather than silently invent both producers inside a nominal adapter. + +**Selected implementation profile:** retain the ordinary deployment on the local +system Docker Engine. Add prospective original provenance at its actual +original production boundary, before any supported first launch or successor +admission. The trusted producer must create and authenticate a genuinely new +deployment identity and its original roots, configuration and history binding; +an empty preexisting directory is not proof that a prior launch never happened. +All ordinary bootstrap routes in this profile must enforce that binding. + +This follows the existing-deployment profile in #2824 and leaves NEW/#2672's +private-engine admission unchanged. Reusing the private-engine NEW receipt was +considered, but would add an unnecessary engine/topology dependency. A missing +producer is implementation work, not by itself an architectural approval +boundary. The selected prospective profile still cannot migrate, re-enroll or +launch the current incident's deployment, or fill any missing original history. + +## Requirement-by-requirement audit of the original base + +| Requirement | Reusable implementation | Missing production fact or integration | +| --- | --- | --- | +| REQ-101 | Original history derivation/verification; quarantine journal reader. | Original baseline-to-history bridge, full ancestor scope, and explicit complete-but-unknown evidence semantics. No synthetic baseline or lost-journal conversion. | +| REQ-102 | Approved enrollment, installed source/mount verification, complete route observer. | One accepted source/engine profile covering all lineage generations and the selected consumer. Operator lists remain selections. | +| REQ-103 | Exact original process/cgroup observations, pinned namespace checks, listener and controller inspection. | Recovery-specific current SH/resource-custody observation repeated in the issuer immediately before grant. | +| REQ-104 | `acquire_prior_owner_fence`, protected file opens and `_assert_pid_flock`. | Issuer-owned independent SH acquisition for the exact complete original set, rechecked after acquisition and at issuance. | +| REQ-105 | Protected, exclusive, append-only persistence patterns; bounded recovery state prototype. | Deployment-fixed production inventory and complete evidence binding, then a single reservation/intent/consumption/receipt state machine reached by the real issuer. A prototype alone is not this gate. | +| REQ-106 | Existing local peer credentials, challenge and selected-container/owner checks; immutable READY validators. | Current whole-scope verification, complete consumer fence-custody receipt, durable one-use consumption before dispatch, and current host-lifetime/readiness handling. | +| REQ-107 | Existing old-root unknown-outcome and first-launch refusals. | Explicit new-epoch authorization and original-exclusion composition that cannot reuse an old root, plan, claim or operation key. | +| REQ-108 | Real original-host boundary fixtures, Docker fresh tests and disposable closed-host fixture components. | All integrated AS-B01 through AS-B05 scenarios through the recovery production issuer and actual prerequisite verifiers. Existing component receipts do not satisfy this row. | + +The durable component verifies persistence and locking independently. It cannot +establish any of these prerequisite joins by itself. The current implementation +connects the new original-genesis and full-scope containment producers to the +zero-argument issuer; all integrated acceptance obligations remain required. + +## Issuer and consumer protocol boundary + +`recovery_epoch_peer.PeerSession` implements bounded authenticated transport and +current custody observation. The issuer supplies an `ExpectedPeer` already +bound by its real Docker/target validator, the complete `HeldFence` set and any +`HeldResource` network-namespace pins. The session pins the peer with a pidfd, +checks `SO_PEERCRED` and the exact current start-token/boot lifetime, exchanges a +fresh challenge-bound offer and receipt, and returns immutable `PeerCustody`. +`session.check(custody)` repeats actual file, lock, namespace and peer-lifetime +observations. There is no grant method: the issuer separately owns durable +consumption and grant ordering. Caller-owned fences remain open on refusal; +the session closes its own socket/PID pin and any unexpected received FDs. + +Real subprocess tests in `tests/test_management_recovery_epoch_peer.py` exercise +independent SH/target EX acquisition, passed EX open-file-description custody, +network namespace transfer, observable queued/received/acknowledged boundaries, +malformed or incomplete receipts, inherited-as-independent locks and current +custody loss. The independent consumer creates a local socket pair and passes +the issuer endpoint over an inherited mailbox, preserving real consumer PID +credentials without requiring a filesystem listener. These are kernel/peer +component tests, not Docker, complete deployment scope or recovery-grant tests. + +A prospective issuer needs a fresh authenticated local challenge bound to the +fixed deployment, explicit authorization, exact target/preparation and exact +consumer process/boot lifetime. The existing `SO_PEERCRED` and container/ledger +joins are useful mechanisms, but the recovery protocol must additionally prove +custody of the complete required fence set. + +Original shared locks require independent acquisition by each relevant holder. +A passed or inherited descriptor must not be reported as that independent +acquisition. Controller/deployment exclusive fences have a different custody +question: if they are transferred using file descriptors, the issuer must wait +for a challenge-bound receipt and independently inspect the exact receiving +process's descriptors and kernel locks before consuming the epoch. A queued +`SCM_RIGHTS` message or successful `sendmsg` is not receipt. A bare caller claim +that it holds a descriptor is insufficient as well. + +The target source/configuration and installed READY must be revalidated at that +same boundary using their actual validators. `ManagementVersionStager.lookup`, +`self_update_handoff.verify_installed_runtime` and `fresh_runtime_binding` are +existing sources to evaluate for the selected runtime profile; their contracts +must be matched, not replaced by the text `READY` or by an image SHA alone. +All original lineage, current containment, target and fence observations must +remain exact across final durable consumption and dispatch. + +Consumption precedes any escaping grant. Lost response, interrupted receipt, +partial publication or host exit leaves the original state preserved and +non-reusable, including for another target/preparation. Current readiness must +become unavailable after host loss without rewriting a historical observation. +An already emitted grant may race the first launch: the testable requirement +is retained custody and no second authorization, not an impossible promise +that no `Popen` can occur after host death. + +## Integrated production-origin test plan + +| Scenario | Production origin and material barrier | Required observed outcome | +| --- | --- | --- | +| AS-B01 / REQ-101, 107, 108 | Genuine prospective genesis followed by A-to-B-to-C and an original refused successor; actual original verifiers inside issuer. Remove/mutate each baseline, anchor, store, expected failed record and attempt binding in separate preserved cases. | Positive complete scope; exact refusal for missing, stale, forged or cross-deployment facts. Genuine original terminal never-created control succeeds independently. No new historical record, cleared unknown outcome or store repair. | +| AS-B02 / REQ-102, 103, 108 | Full selected closed profile in a disposable systemd VM with real Docker, ordinary restart/watch/service/timer/cron/remote routes and a preserved worker. Observe actual cgroup and namespace membership before each action. | Every omitted, enabled or unknown route, escaped/nonleader task and occupied current listener/namespace refuses. Positive exited-task control is independently measured. Worker/configuration and all original stores remain intact. | +| AS-B03 / REQ-104, 106, 108 | Independent real host and consumer processes open exact original files. Pause at an observed acquisition/receive boundary, then have another process attempt target EX while the intended holds overlap. | Independent SH ownership and exact peer lifetime pass. Busy/replaced/missing files, inherited-as-independent locks, omitted/duplicate descriptors, queued-only transfer and wrong receipt refuse before grant. | +| AS-B04 / REQ-105, 107, 108 | Two real production-factory commands use different preparation/target names but one fixed inventory. Pause the first at a recorded reservation boundary and observe the second actually contend before releasing the first. Kill around each record/create/fsync/head boundary. | At most one reservation/consumption path, no alternate-root bypass, no accepted environment/request override, no repair or reuse of a partial/uncertain state. Preserve bytes and identities for independent readback. | +| AS-B05 / REQ-103, 105, 106, 108 | Pause immediately before the actual final grant boundary, mutate source/READY/target/controller/process/fence evidence, then resume. Separately lose replies or kill the host around consumption and dispatch. | Changed/stale input refuses; a consumed epoch never grants again. Host loss makes current readiness unavailable. If a grant already escaped, observe retained consumer custody and no second grant rather than assuming launch cannot race. | + +`tests/fixtures/original_host_boundaries.py` demonstrates suitable barriers at +`before-event`, `after-event`, `after-pending`, `after-replace` and `after-sync` +around actual producer publication. Its subprocesses execute the production +claim/grant path; instrumentation observes or interrupts real operations. +Merely starting two processes concurrently does not establish contention. + +`tests/test_management_fresh_end_to_end.py` supplies genuine Docker histories +and authenticated-child flows. It does not independently establish the full +closed deployment profile. Conversely, +`scripts/test_management_containment_closed_host.py` exercises actual producer +and separate-consumer containment commands, real routes and current kernel +controls, but does not issue a recovery grant or establish complete lineage. +The integrated test must join these authorities through the new issuer. + +The closed-host fixture requires its preexisting protected synthetic-host +authorization, an initially empty KVM/QEMU VM with PID 1 systemd and the initial +host namespaces. It deliberately inhibits fixture host routes. It is not a +command to run on a populated production host. The existing Supervisor +`fresh-host` CI profile runs privileged Docker tests, but does not itself run +this full closed-host recovery scenario. New integrated nodes and their native +receipts must be collected and required by an appropriate isolated profile; +no missing-node or skip allowance may silently convert unavailable authority +into acceptance. + +## Durable state component and its authority boundary + +`recovery_epoch_state.hold_recovery_epoch()` has no arguments. Its production +root is fixed at `/var/lib/auto-coder/recovery-epoch`; environment variables, +request paths, target names and preparation names cannot select another root. +The factory requires the approved Unix owner and protected, canonical directory +ancestors. It does not provision or repair a missing directory, inventory or +lock. An operator would have to separately review and install the inventory; +the feature implementation performs no such live operation. + +The finite `EpochInventory` binds the exact root and lock device/inode identities, +deployment, authorization, sorted unique enrollment set, request, preparation, +Docker target, source digest and configuration digest. An `EpochBinding` must +match those fields and additionally binds the verified-evidence digest and the +actual current issuer boot/PID/start lifetime. The component validates those +joins and their exact schema, but cannot establish that the supplied evidence +was independently produced. The issuer in `recovery_epoch.py` establishes that +fact before calling it; inventing a digest is not an authorization mechanism. + +An independently opened exclusive flock serializes the whole state invocation. +The component rechecks the current process's actual kernel lock, protected +inventory identity and bytes, every retained record and the complete namespace +before each transition. A forked or released holder, replaced lock/root/record, +changed inventory, conflicting alias or existing phase refuses. Once a reservation +is durable, a conflicting authorization, evidence or issuer-lifetime binding +poisons that same invocation, even when its caller catches the refusal. A +subsequent call using the original binding cannot continue consumption. + +| Transition | Persisted fact | Reentry behavior | +| --- | --- | --- | +| `reserve(binding)` | `reserved.json` and immutable `reserved-head.json` | Any subsequent invocation refuses this deployment state, including a different target/preparation. | +| `record_intent(binding)` | `intent.json` and immutable `intent-head.json`, linked to the prior head | A partial or uncertain write is preserved and cannot be repaired or resumed. | +| `consume(binding)` | `consumed.json`/head followed by `receipt.json`/head | Returns only after both transitions are durable. A lost return, exception or second call cannot produce another receipt. | + +Each new file uses exclusive creation and no-follow opens, file fsync and +directory fsync. The append heads and records retain the exact binding and +previous-head digest. The library never deletes a partial file, replaces a +committed head, resets a reservation or reconstructs state after restart. +`EpochReceipt.grant_delivered` is always `False`: this component neither has a +transport nor observes a grant. The host issuer must retain and independently +reverify consumer custody and current evidence at its eventual dispatch boundary. + +`_hold_recovery_epoch(root, expected_uid=...)` is an internal component-test seam. +It must not become a production CLI/environment override. The maintained tests +use isolated user-owned files and real process/kernel flocks. They exercise +material contention after a post-reservation barrier, I/O failures around each +record/head fsync, SIGKILL at observable publication barriers, changed bindings, +path replacement and replay refusal. These are bounded persistence tests, not +production-host, Docker, complete-lineage or integrated grant acceptance. +Process interruption tests do not claim simulated storage-power-loss durability. + +Run the maintained component selection with: + +```sh +bash scripts/test.sh --ci-tests-only tests/test_management_recovery_epoch_state.py +``` + +The component uses the existing Loguru configuration with console and rotating +file sinks. Its phase records explicitly carry `grant_delivered=False`; they +never report authenticated readiness or recovery success. The issuer's real +phase, refusal and dispatch events are documented in +`docs/dashboard-observability.md` and retain the same distinction. + +## Prelaunch consumer custody + +`recovery_epoch_consumer.connect_recovery_epoch(config_path, target_owner_fd, +timeout=...)` accepts only `/bootstrap/recovery-epoch.json` and connects only +`/management-recovery/issuer.sock`. The target's independently verified Docker +configuration must bind that configuration's bytes, the read-only mounted issuer +directory and its device/inode identity. The configuration contains the exact +request, authorization and target IDs; each independently acquired original or +history shared fence's local path and device/inode; the target owner identity; +and every transferred controller, deployment or network-namespace identity. +Host paths are never assumed to be valid target paths. + +The consumer authenticates the connected issuer with `SO_PEERCRED` and obtains +its lifetime pin directly from the kernel with `SO_PEERPIDFD`. That capability is +mandatory: a private Docker PID namespace may expose the host peer as PID zero, +so a caller-supplied PID or pidfd cannot substitute for socket-bound kernel +identity. The consumer independently acquires the declared shared locks and +checks its already held target exclusive lock. It acknowledges only the exact +challenge-bound received resource set, with current kernel flock records for +exclusive descriptors and exact nsfs objects for namespace pins. It accepts a +grant only after that receipt and only with the exact request/challenge binding, +all four durable digest fields and `granted="recovery-epoch-consumed"`. + +`RecoveryEpochConsumer` retains its original shared locks, transferred exclusive +locks and namespace pins, socket and issuer pidfd. The caller continues to own +the target exclusive descriptor. `check_current()` rechecks custody and the +issuer's process and socket lifetime; failure does not implicitly close retained +resources. An explicit `close()` or context exit releases them only when the +owning Supervisor establishes that it is safe. The helper has no child claim, +store creation, launch or readiness operation; those remain the separate child +contract. The configurable overall offer/grant deadline is positive and at most +one hour, allowing full host observations while keeping each protocol send +bounded to at most sixty seconds. + +`tests/test_management_recovery_epoch_consumer.py` uses independent real issuer +and consumer processes, actual flocks, namespace descriptors and kernel peer +pidfds. It observes the offer/receipt/grant barriers, rejects incomplete custody +and stale grants, and proves that both the independent shared fence and received +exclusive fence still prevent a new exclusive contender after issuer death, +until explicit consumer close. Its wire issuer supplies protocol grant bytes; +these tests do not establish Docker, lineage, durable consumption or complete +production grant authority. A real connected production issuer/consumer path +in the isolated native profile is still required for that claim. + +## Observability and completion boundary + +The host issuer adds no GitHub processing attempt, management operation admission, +worker action or dashboard-owned state. Its actual reservation, refusal, +consumption and grant-send observations use the existing Loguru stdout/file +configuration and explicitly distinguish dispatch from current readiness. +`docs/dashboard-observability.md` describes those events and their maintained +checks. Full native VM results must still substantiate the integrated contract. + +Do not close #2824, merge an implementation as its completion, or deploy a +recovery issuer until the missing producers are implemented, all eight +requirements are actually connected, and the integrated native evidence +passes. Completion of a prototype or this audit is not that result. diff --git a/docs/client-features/management-recovery-epoch.md b/docs/client-features/management-recovery-epoch.md new file mode 100644 index 000000000..bdf131d01 --- /dev/null +++ b/docs/client-features/management-recovery-epoch.md @@ -0,0 +1,295 @@ +# Management recovery ownership epoch + +The host producer in `management/recovery_epoch.py` establishes one new, +explicitly authorized ownership epoch for the finite ordinary local-system-Docker +profile. It joins original provenance from #2839, current full-scope containment +from #2840, quarantined operation inspection from #2823, genuine installed READY, +and actual receiving-process fence custody. Each prerequisite remains evidence +only. None can issue a grant or repair a missing original record. + +The separate #2825 consumer owns a durable child-launch claim, one child launch, +authenticated application readiness, and quarantined admission/dispatch. This +producer's `--recovery-epoch` bootstrap endpoint acquires and retains custody but +never starts a child, creates an operation store, or announces readiness. A +successful socket send is not proof of delivery or an application outcome. + +## Supported original and current scope + +A supported deployment begins with the affirmative original root-creation and +ordinary-bootstrap genesis producers described in +`management-original-genesis.md`. Later supported attempts remain in that same +protected history frontier, including failed branches. Complete original facts +can retain an unknown outcome. A missing expected journal, missing genesis, +unsettled original SQLite sidecar, changed file, or external legacy baseline +remains unavailable. Current enrollment cannot supply historical completeness. + +The current scope is one approved local system Docker engine, exact ordinary +manager/proxy pairs, every actual watcher and normal restart route, and one +separate prelaunch target. Every existing selected original owner inode must be +independently held SH by the issuer and consumer. The issuer also acquires the +actual existing watcher locks EX and its deployment-fixed epoch lock EX. It +transfers those exclusive open file descriptions and the actual retained nsfs +objects, then independently observes their presence in the authenticated +consumer's `/proc/PID/fd` and kernel lock data. Queued SCM_RIGHTS descriptors or +`sendmsg` success do not establish receipt. + +Original lifetime/cgroup/network evidence comes from actual capture while the +original processes were available. A current namespace pin holds the actual +kernel object. A saved historical inode or Docker's stopped status cannot replace +that object, all-task inspection, current namespace occupancy, or endpoint +listener absence. A positively recorded never-created attempt may have no current +Docker object; its exact absence must be present in full-engine observations and +its original failed branch remains retained. Existing objects cannot be omitted +from enrollment to obtain that result. + +The target retains the original OAuth root and inode. Its state and journal roots, +Docker identity, management instance, and published host port are distinct from +every original. The issuer does not stop/start containers, inhibit controllers, +change OAuth, reset old roots, or touch workers. Those conditions must already +hold under separate reviewed operational authorization. Unknown controller routes +or unsupported profiles refuse. NEW/#2672 and the existing ordinary/quarantined +observation profiles retain their admission contracts. + +Recovery route approval binds the exact cron sources, masks, controller +configuration, boot, cgroup mount, and every inhibited scope's required roots, +absence, empty-task facts and directory identities, including their ancestors. +An unrelated cgroup directory, such as the selected target's ordinary Docker +scope, may appear or be recreated after approval. Every current observation still +enumerates the complete cgroup tree and all tasks, retains those unrelated +directory identities, and requires the full repeated observations to match. +The ordinary route profiles retain their exact persisted-snapshot comparison. + +## Deployment-fixed authority + +The host CLI is `python -m auto_coder.management.recovery_epoch`. It takes no +request, inventory, authorization, or state-root argument. Production files are +fixed; test-only internal factories are not CLI alternatives. + +| Location | Required original authority | +| --- | --- | +| `/var/lib/auto-coder/recovery-epoch/inventory.json` | Canonical version-two `EpochInventory`, including exact directory/epoch-lock identities, deployment, authorization, one enrollment, request, preparation, Docker target, bootstrap/source digests and the full authorization digest. | +| `/var/lib/auto-coder/recovery-epoch/epoch.lock` | Preexisting protected original file; independently acquired EX. The issuer never creates or repairs it. | +| `/etc/auto-coder/recovery-epoch/authorization.json` | Canonical `EpochAuthorization`: exact history binding, protected consumer and READY file identities, configured source repository/read credential/installer/interpreter, exact tested source and socket-directory identity. | +| `/var/lib/auto-coder/approved-recovery-containment-enrollments/.json` and its approved routes file | The complete scope independently checked by the current-containment producer. An operator list is not a containment verdict. | +| `/var/lib/auto-coder/recovery-epoch-peer/issuer.sock` | One protected authenticated local endpoint, created only inside the separately provisioned fixed private directory. Existing endpoints refuse; no stale-socket repair is attempted. | + +Authority directories are root-owned private canonical directories with trusted +ancestors. Files are private original single-link files. The inventory's +`authorization_sha256` binds the complete separately reviewed scope, and that +scope must still match the inventory and original inode/bytes at issuance. +`AUTO_CODER_RECOVERY_EPOCH_ROOT`, `AUTO_CODER_RECOVERY_EPOCH_INVENTORY`, +`AUTO_CODER_RECOVERY_AUTHORIZATION_ROOT`, and +`AUTO_CODER_RECOVERY_CONTAINMENT_ROOT` are refused rather than used as overrides. +No provisioning or alternate-root deduplication path exists in the issuer. + +Version one remains a non-authorizing persistence component format; it cannot +enter the complete issuer. Version two requires canonical retained evidence and +its matching digest. An original request or genesis authorization cannot be +reused as the new recovery request/authorization. + +## Exact installed target and receiving process + +The target selects `BootstrapConfig.recovery_epoch_config` as +`/bootstrap/recovery-epoch.json` and explicitly invokes: + +```sh +python -m auto_coder.management.bootstrap_supervisor \ + --config /bootstrap/config.json --recovery-epoch +``` + +The ordinary supervisor constructor refuses this selected profile, so omitting +the mode or entering the normal first-child path does not bypass recovery. The +endpoint opens the already provisioned target `.supervisor-owner.lock` without +creation, obtains its own independent EX, and requires no previous target launch +or operation state. It connects only to `/management-recovery/issuer.sock` and +holds its own independently opened original/history SH files. + +The endpoint's private configuration binds request/authorization/target, +expected issuer UID, actual socket-directory device/inode, every original +resource name/path/device/inode, target-owner identity, and each transferred +controller/epoch/namespace identity. The issuer independently joins the complete +set to its actual held descriptors. The target mounts the fixed host peer +directory read-only at `/management-recovery`, the exact history lock read-only +at `/management-recovery-history/history.lock`, and selected original owner files +read-only at `/management-recovery-originals//owner.lock`. + +The host authenticates the local peer with SO_PEERCRED, an independently opened +pidfd, the actual Docker leader PID, Linux start token and boot identity. The +consumer uses kernel SO_PEERPIDFD to pin the actual issuer across Docker PID +namespaces; a sender-supplied pidfd or claimed host PID is not authority. Kernels +without this capability refuse. Every resource check remains live after receipt. +The overall consumer verification wait is bounded by the configured bootstrap +verification timeout, up to one hour; individual exchange sends remain bounded. +SIGTERM/SIGINT immediately interrupt a pre-grant wait and close the non-launching +endpoint's resources. Host death does not silently release an already returned +consumer handle's descriptors: its owner must explicitly end custody when safe. + +The selected initial runtime is `management-installed`. The issuer invokes the +actual `ManagementVersionStager.lookup` against original READY, current immutable +tested-source evidence and installed bytes. The management configuration and all +runtime arguments/environment/source identities must agree with +`runtime_for_receipt`. The READY preparation directory must equal the approved +preparation ID. Unsupported executable environment hooks refuse. + +The installer pin retains the complete `uv --version` text after its `uv ` +prefix, including any build descriptor. Its grammar is a numeric three-part +version with one optional parenthesized ASCII build descriptor, bounded to 128 +characters in total. The stager still requires exact equality with the selected +installer's actual output; a matching numeric version alone cannot authorize a +different build. The native fixture verifies its separately selected numeric +version before passing that complete observed pin into authorization and READY. + +Both the complete interpreter distribution and READY environment must be inside +the exact same-path read-only version-root mount. In particular, a host-only venv +interpreter symlink cannot satisfy target verification. The authorization's +configured interpreter and the runtime's resolved interpreter must be the same +file inside that closure. This permits an actual portable CPython installation +within the separately reserved version root; it does not rewrite a receipt's +original paths or accept an invented READY file. + +## Single-use durability and grant ordering + +The issuer first independently derives and verifies the full original lineage, +inspects all actual quarantined journals, obtains current containment, verifies +exact installed target READY, and observes the consumer's complete acknowledged +custody. It then records the fixed sequence: + +| Phase | Durable consequence | +| --- | --- | +| `reserved` | The one deployment-wide invocation and complete original/current evidence binding are reserved. | +| `intent` | The exact target, authorization, source, full original fingerprints, controller/resource identities and receiving custody remain retained. | +| `consumed` | That epoch can never yield another grant, including after interrupted persistence or a lost reply. | +| `receipt` | A final durable record binds the consumed record and exact evidence; it is not itself delivery proof. | + +Each phase uses O_EXCL publication, file and directory fsync, an immutable +hash-linked head, actual writer-inode readback and complete directory/inventory +revalidation. `evidence_json` retains the canonical verified original fingerprints +and current bindings, and must match `evidence_sha256`. The original outcome is +preserved in that evidence. A conflicting binding after reservation poisons the +current invocation, including when a caller catches the error. Cancellation or +uncertainty during publication also poisons it. Every later invocation rejects +existing or partial phase evidence; nothing resumes, resets or repairs it. + +Original lineage, exact target READY, protected authorization/enrollment, +quarantined journals, actual current containment and receiving custody are all +rechecked before consumption and again after persistence, immediately before the +single send. The grant is bound to the fresh peer challenge and issuer receipt +nonce, exact request/authorization/target, four durable digests and actual issuer +lifetime. There is no retransmission path. A completed send followed by host +loss can race a future consumer's first launch; correctness requires retained +custody and no second grant, not an impossible promise that host death forbids an +already-authorized Popen. + +After the one send, the host remains alive with its fences and session. Loss of +that host, socket or custody makes current consumer checks unavailable. The +historical durable receipt remains unchanged. This endpoint never marks an +application ready; #2825 must own that separate proof and protected operation +lifecycle. + +## Observability and validation + +The CLI and bootstrap endpoint use the existing Loguru stdout plus file sinks. +Durable phase events name `recovery_epoch_phase`, request and target. Explicit +`current_ready=False` accompanies grant-send and unavailable observations. No +GitHub issue/PR processing attempt, dashboard slot, management operation admission +or worker outcome is created by this host ownership protocol. See the matching +entry in `docs/dashboard-observability.md`. + +Refusal diagnostics retain up to eight explicit exception causes with only their +type, integer error number, and code basename/function/line. They omit inner +exception messages, arguments, frame locals and full paths, and never change the +refusal decision or preserved state. + +The maintained component selections cover real persistence failures/process +interruption, exact fixed-scope parsing, independent-process SH/EX/nsfs custody, +actual received descriptors, current peer lifetime and cancellation. Component +fixtures are not original historical authority or live Docker containment. + +```sh +bash scripts/test.sh --ci-tests-only \ + tests/test_management_recovery_epoch_state.py \ + tests/test_management_recovery_epoch_authorization.py \ + tests/test_management_recovery_epoch_peer.py \ + tests/test_management_recovery_epoch_consumer.py +``` + +The separate isolated initial-namespace systemd/Docker VM acceptance must traverse +the actual genesis/attempt producers, complete current observer and exact host +issuer/consumer entrypoints. Its AS-B01 through AS-B05 obligations include +observable contention, missing/mutated/forged prerequisites, all-controller/task/ +listener refusals and interruption/host-death boundaries. Passing the component +selection alone does not establish those native obligations or applicability to +a current incident with missing originals. + +### Maintained native acceptance + +`scripts/test_management_recovery_epoch_host.py` owns the actual host-entrypoint +driver and each scenario's required outcomes. The +`Management Recovery Epoch Acceptance` workflow derives its complete matrix from +that driver's `ACCEPTANCE_SCENARIOS`; its final job requires every scenario to +succeed. A missing case, failed VM, unavailable native kernel operation or failed +preservation check cannot become a passed result. The VM runner and reviewed +reproduction procedure are documented in `scripts/recovery_vm/README.md`. + +The fresh recovery VM stops and masks six explicitly named timer/service pairs +from its observed stock timer inventory: `systemd-tmpfiles-clean`, `apt-daily`, +`apt-daily-upgrade`, `dpkg-db-backup`, `e2scrub_all` and `fstrim`, each with its +exact `.timer` and `.service` name. All selected timers stop before any selected +service, before original producers and route approval. The existing stock-route +fixture retains ownership of the separate `motd-news` pair. The guest records +actual stopped and inhibited systemd facts and protected mask identities in +`maintenance-profile.json`; unavailable, failed or changing state and preexisting +mask paths fail preparation. This finite setup prevents the observed scheduled +maintenance from changing its approved execution readback during native +acceptance, including the measured midnight backup execution documented in +`scripts/recovery_vm/README.md`. Production catalog/digest equality and all +ordinary or quarantined containment profiles remain unchanged. + +Before recovery preparation, the fixture proves that all original A/B/C managers, +proxies and cutover readers have stopped. It may then close only the empty WAL +and derived SHM residue of their twelve known original stores, preserving every +database's bytes and identity. The separate A/B readiness observation stores are +required and preserved without checkpointing; unresolved C has none. Unknown +stores, nonempty WAL/rollback journals, or readiness sidecars refuse. This fixture +setup does not repair an operational recovery source or change the production +quarantine policy. + +| Contract | Actual fixture action and independent outcome | +| --- | --- | +| AS-B01 | Run genuine genesis, successful successor, never-created failed branch and unresolved successor producers. Verify full original lineage, reject forged/cross-deployment/stale reports, remove an original journal, and mutate the retained failed branch's exact terminal evidence after intent. Refusal must preserve the original uncertainty and stores. | +| AS-B02 | Install the ordinary service/timer/watcher/alternate-root fixture routes, bind them in approved enrollment and verify their actual inhibition. Exercise unknown/omitted/startable controllers, retained namespace listeners, an escaped nonleader task and independently measured exit. Repeat the real containment verifier in a separate process and preserve unrelated worker claims and effect counters. | +| AS-B03 | Use the real host and Docker bootstrap processes, kernel peer credentials/pidfds and independent SH/target EX acquisition. Refuse busy/replaced/missing fences, a foreign acquiring process, queued rights without acknowledgment, received rights before acknowledgment, and a changed target lifetime. Later successful issuance must retain the complete observed custody. | +| AS-B04 | Reject actual CLI/environment root and inventory overrides. Stop the real issuer at its fsynced reservation head while a contender runs, then test conflicting target/preparation binding. In separate fresh VMs, SIGKILL the original process before and after every record/head publication and inspect the exact retained prefix and refused replay. | +| AS-B05 | After the original intent head, change exact configuration, READY, installed source, original evidence, watcher/controller state, fence acquisition or target lifetime. Require refusal before consumption. In a distinct fresh VM, change exact READY after the fsynced receipt head and resume the same issuer; require actual refusal and an empty authenticated receiver queue while the receiver is stopped. Preserve consumed records, the injected mutation and workers, and refuse replay. Separately require a real queued grant before issuer death, then resume the receiver and require unavailable current custody with no replay. | + +Publication and protocol fault helpers only observe real syscall entry/exit and +kernel descriptors. They do not replace system-call results, inject verifier +verdicts or alter production code. Deliberate late fixture mutations remain +present through refusal and replay; their exported preservation baseline is +explicitly the state after that injection. The successful path measures original +OAuth/history/journal bytes and identities, unrelated beta/release/agent-table +worker ownership and effect requests, and the absence of any replacement child +or application admission. The current PR's custody endpoint and a separately +qualified installed READY are exercised independently; these checks do not claim +that the separate #2825 management-child launch/readiness contract is complete. + +The separate `omit-post-consumption-verifier` mutation-control job proves that +the post-receipt scenario detects omission of the second final verification. +Before creating any original fixture, it copies the project into a fresh private +directory and removes exactly that one standalone call. An AST comparison +retains the first verifier and every other source expression; the original and +copied file digests are retained. The copied project is installed and supplies +every actual host process and fixture image, so source, history and enrollment +identities continue to agree without replacing an authority provider. + +The control runs the same post-receipt READY mutation and the same no-grant +assertion. Its distinct `mutation-detected` result requires the named +`post-receipt-grant-forbidden` failure and a complete canonical grant observed +with `pidfd_getfd` and nonconsuming `MSG_PEEK` on the stopped receiver's exact +authenticated socket. A timeout, process kill, unrelated exception, unavailable +kernel operation or log message cannot satisfy that result. Only after recording +that incorrect dispatch does cleanup end the mutant issuer; the receiver must +still reject current custody with no child/admission, and the consumed epoch, +deliberate READY mutation and unrelated workers must survive refused replay. +This control does not count as production acceptance, and it never changes the +original source checkout or resets an epoch for another attempt. diff --git a/docs/dashboard-observability.md b/docs/dashboard-observability.md index 073ea211e..af960ef9e 100644 --- a/docs/dashboard-observability.md +++ b/docs/dashboard-observability.md @@ -3611,3 +3611,56 @@ boundary. The installed release acceptance case is listed separately in the required isolated-container profile. Dashboard processing events/renderers need no new target outcome: this failure precedes serving and is a daemon/management diagnostic, not a completed Issue/PR execution. + +## Management recovery ownership evidence and one-shot issuance + +The prospective original-genesis producer, full-scope recovery-containment +observer, and `management/recovery_epoch.py` host issuer have separate authority +lifetimes. Genesis and containment diagnostics remain evidence-only. The current +observer emits `observation_only=True`, `recovery_grant=False`, and +`admission_created=False`; it never writes an operation or readiness outcome. +The issuer's actual append-only writer emits `recovery_epoch_phase=reserved`, +`intent`, `consumed`, and `receipt`, with request/target bindings and +`grant_delivered=False`. These events occur after their durable record/head +boundaries, and their Loguru source locations identify the real writer. A later +`grant-send-completed` observation means that one socket send completed; it does +not acknowledge delivery, launch a child, or establish current readiness. + +The explicit bootstrap custody origin logs `consumer-custody` only after a +verified complete offer and exact consumed grant. `consumer-unavailable` and host +`unavailable` retain `current_ready=False`. A shutdown signal while awaiting an +offer immediately fails that prelaunch endpoint. An exited issuer or lost fence +makes current custody unavailable without changing historical receipts or +inventing a recovery success. Both CLI origins use the existing Loguru stdout +and file sinks; the host file is `/var/log/auto-coder/recovery-epoch.log` and the +bootstrap file is in its separate target state's `logs` directory. + +These host commands do not enter an Issue/PR processing attempt, implementation +slot, management operation admission, dispatch, or worker lifecycle. There is +therefore no dashboard event-schema or renderer addition. Creating a dashboard +success/job for a non-authorizing observation or an unacknowledged grant would +misrepresent the boundary. The separate #2825 consumer must own any future child +readiness/admission projection. Existing unknown originals and their key/target +exclusions remain unknown and excluded. + +Run `bash scripts/test.sh --ci-tests-only + tests/test_management_recovery_epoch_observability.py + tests/test_management_recovery_epoch_state.py + tests/test_management_recovery_epoch_authorization.py + tests/test_management_recovery_epoch_join.py + tests/test_management_recovery_epoch_peer.py + tests/test_management_recovery_epoch_consumer.py + tests/test_management_recovery_containment_resources.py`. +The observability test drives the real durable writer in a separate process and +checks the ordered structured phases, source locations, stdout and file output, +and absent grant/readiness claims. The real-signal consumer cases observe the +pre-offer barrier, cancellation, no custody-acknowledgment success, no target +store/child creation and released target exclusion. Dataclass join fixtures are +component scope checks, not evidence of native containment. The independent +systemd/Docker VM workflow `Management Recovery Epoch Acceptance` must traverse +actual original producers, controller/kernel observations, connected peer +custody and at-most-one issuance for integrated acceptance. See +`client-features/management-recovery-epoch.md`, +`client-features/management-original-genesis.md`, and +`client-features/management-recovery-containment-observation.md` for the fixed +profiles, interfaces and source-of-truth limits. diff --git a/scripts/prepare_management_test_image.py b/scripts/prepare_management_test_image.py index c7659fc28..fdc015187 100644 --- a/scripts/prepare_management_test_image.py +++ b/scripts/prepare_management_test_image.py @@ -80,7 +80,7 @@ def prepare_image(repository: Path, github_env: Path | None = None) -> str: encoded = base64.b64encode(serialize_build_identity(identity).encode()).decode() tag = f"auto-coder:management-fixture-{identity.source_revision[:12]}" dockerfile = (repository / "Dockerfile").read_text() + ( - "\nFROM python-environment AS management-test-fixture\n" 'ENV PATH="/opt/auto-coder/.venv/bin:${PATH}"\n' "ARG AUTO_CODER_SOURCE_REVISION\n" "RUN AUTO_CODER_SOURCE_REVISION=$AUTO_CODER_SOURCE_REVISION /opt/auto-coder/.venv/bin/python -m auto_coder.build_provenance embed\n" + "\nFROM python-environment AS management-test-fixture\n" 'ENV PATH="/opt/auto-coder/.venv/bin:${PATH}"\n' "ARG AUTO_CODER_SOURCE_REVISION\n" "RUN AUTO_CODER_SOURCE_REVISION=$AUTO_CODER_SOURCE_REVISION /opt/auto-coder/.venv/bin/python -B -m auto_coder.build_provenance embed\n" ) subprocess.run( ["docker", "build", "--file", "-", "--target", "management-test-fixture", "--build-arg", f"AUTO_CODER_BUILD_IDENTITY_B64={encoded}", "--build-arg", f"AUTO_CODER_SOURCE_REVISION={identity.source_revision}", "--tag", tag, str(repository)], diff --git a/scripts/recovery_epoch_containment_fixture.py b/scripts/recovery_epoch_containment_fixture.py new file mode 100644 index 000000000..b9b1913a1 --- /dev/null +++ b/scripts/recovery_epoch_containment_fixture.py @@ -0,0 +1,189 @@ +"""Real-object helpers for the separately authorized disposable recovery VM. + +The calling harness first runs require_disposable_host on the initially empty +VM. These helpers retain live original namespaces, capture actual records and +install an explicitly reviewed fixture scope; the production observer never +imports this module and never performs these fixture effects. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import stat +from dataclasses import asdict, dataclass, replace +from pathlib import Path +from typing import Protocol + +from scripts.test_management_containment_closed_host import MARKER, WatcherFixture, read_authorization +from scripts.test_management_containment_enrollment_host import _private_directory, _write_private + +from auto_coder.management import proxy_watch_cli +from auto_coder.management.containment_enrollment import OriginalNetworkPin +from auto_coder.management.containment_kernel_subobservation import _holder_start_token, _require_capture_host +from auto_coder.management.containment_observation import observe_recovery_docker +from auto_coder.management.containment_route_manifest import observe_recovery_route_manifest, observe_selected_bind_roots, propose_unapproved_route_draft +from auto_coder.management.fresh_host import _write_evidence +from auto_coder.management.fresh_host_observation import _engine_id +from auto_coder.management.installed_bootstrap_mount_profile import _inspect +from auto_coder.management.original_attempts import canonical, source_identity +from auto_coder.management.recovery_containment import capture_recovery_original_pair, load_enrollment, route_selection +from auto_coder.management.recovery_containment_model import RECOVERY_ENROLLMENT_ROOT, RECOVERY_PROFILE, RecoveryEnrollment, RecoveryEnrollmentObservation, RecoveryFile, RecoveryLock, RecoveryOriginal, RecoveryTarget, RecoveryWatcher +from auto_coder.management.recovery_containment_profile import _mapped_file, read_bound_file + + +class Generation(Protocol): + name: str + root: Path + container: str + proxy: str | None + + def path(self, name: str) -> Path: ... + + +class OriginalChain(Protocol): + image: str + proxy_image: str + + +def selected_file(path: Path, *, source: bool = False) -> RecoveryFile: + resolved = path.resolve(strict=True) + info = resolved.lstat() + result = RecoveryFile(str(resolved), hashlib.sha256(resolved.read_bytes()).hexdigest(), info.st_dev, info.st_ino) + read_bound_file(result, source=source) + return result + + +def selected_lock(path: Path) -> RecoveryLock: + if path != path.resolve(strict=True): + raise ValueError("fixture lock is not canonical") + info = path.lstat() + if not stat.S_ISREG(info.st_mode) or info.st_uid != 0 or stat.S_IMODE(info.st_mode) != 0o600 or info.st_nlink != 1: + raise ValueError("fixture lock is not the actual private original object") + return RecoveryLock(str(path), info.st_dev, info.st_ino) + + +@dataclass +class CapturedGeneration: + original: RecoveryOriginal + namespace_fd: int + + def close(self) -> None: + if self.namespace_fd >= 0: + os.close(self.namespace_fd) + self.namespace_fd = -1 + + +def capture_generation(chain: OriginalChain, generation: Generation, *, generation_id: str | None = None) -> CapturedGeneration: + """Capture while the genuine manager and paired Caddy are still alive.""" + _require_capture_host() + if generation.proxy is None: + raise ValueError("the original paired Caddy must already exist") + mounts = observe_selected_bind_roots((generation.container, generation.proxy)) + bootstrap = selected_file(generation.path("bootstrap") / "config.json") + config = json.loads(read_bound_file(bootstrap)) + management_destination = config.get("management_config") + if management_destination is None: + runtime = next(item for item in config["runtimes"] if item["reference"] == config["initial_runtime"]) + arguments = runtime["arguments"] + management_destination = arguments[arguments.index("--config") + 1] + manager_mounts = tuple(item for item in mounts if item.container_id == generation.container) + management = selected_file(Path(_mapped_file(management_destination, manager_mounts, writable=False))) + owner = selected_lock(generation.path("state") / ".supervisor-owner.lock") + boot = Path("/proc/sys/kernel/random/boot_id").read_text().strip() + preliminary = RecoveryOriginal(generation_id or generation.name, config["instance_id"], generation.container, chain.image, generation.proxy, chain.proxy_image, "started", boot, bootstrap, management, selected_file(generation.path("Caddyfile")), mounts, owner, None, None) + actual = _inspect(generation.container) + state = actual.get("State") + if not isinstance(state, dict) or state.get("Running") is not True or type(state.get("Pid")) is not int or state["Pid"] <= 1: + raise ValueError("fixture original is not currently running") + descriptor = os.open(f"/proc/{state['Pid']}/ns/net", os.O_RDONLY | os.O_CLOEXEC) + try: + namespace = os.readlink(f"/proc/self/fd/{descriptor}") + info = os.fstat(descriptor) + captured_boot, manager, proxy = capture_recovery_original_pair(preliminary, source=source_identity()) + if captured_boot != boot or manager.net_namespace != namespace: + raise ValueError("fixture original namespace changed during capture") + directory = generation.path("recovery-original-capture") + directory.mkdir(mode=0o700) + _write_evidence(directory, {"management": asdict(manager), "proxy": asdict(proxy)}) + pin = OriginalNetworkPin(os.getpid(), _holder_start_token(os.getpid()), boot, descriptor, namespace, info.st_dev, info.st_ino) + complete = replace(preliminary, prior_evidence=selected_file(directory / "prior-evidence.json"), namespace_pin=pin) + parsed = RecoveryOriginal.parse(json.loads(json.dumps(asdict(complete)))) + return CapturedGeneration(parsed, descriptor) + except BaseException: + os.close(descriptor) + raise + + +def start_watcher(generation: Generation, original: RecoveryOriginal, *, first: bool = False) -> tuple[WatcherFixture, RecoveryWatcher]: + """Run, observe and inhibit the real watcher before retaining its EX lock.""" + unit = "auto-coder-management-proxy-watch" + ("" if first else "-" + generation.name) + ".service" + fixture = WatcherFixture.start(generation.root, original.management_id, original.management_image_id, original.proxy_id, original.proxy_image_id, Path(original.proxy_config.path), unit=unit) + selected = RecoveryWatcher(unit, selected_file(fixture.config), selected_file(Path(proxy_watch_cli.__file__), source=True), selected_file(fixture.source, source=True), selected_lock(fixture.state / "proxy-watch.lock")) + return fixture, selected + + +def target_from_container(container_id: str, *, bootstrap_config: Path, management_config: Path, published_port: int) -> RecoveryTarget: + """Select the real already running prelaunch target and its created owner.""" + _require_capture_host() + inspected = _inspect(container_id) + config = json.loads(bootstrap_config.read_bytes()) + mounts = observe_selected_bind_roots((container_id,)) + state = next(item.source for item in mounts if item.destination == config["state_root"]) + image = inspected.get("Image") + if not isinstance(image, str): + raise ValueError("fixture target image is absent") + return RecoveryTarget.parse(json.loads(json.dumps(asdict(RecoveryTarget(config["instance_id"], container_id, image, selected_file(bootstrap_config), selected_file(management_config), mounts, selected_lock(Path(state) / ".supervisor-owner.lock"), published_port))))) + + +def enroll_recovery_scope( + captures: tuple[CapturedGeneration, ...], + target: RecoveryTarget, + watchers: tuple[RecoveryWatcher, ...], + *, + deployment_id: str, + enrollment_id: str, + authorization: dict[str, object], +) -> RecoveryEnrollmentObservation: + """Install fixture approval only under the still-current separate VM marker. + + The caller owns complete controller setup/inhibition. The route draft is + inspected here and marked approved only after all detected candidate and + explicitly selected watcher routes already have their real inhibited facts. + This helper neither stops controllers nor treats an unapproved list as proof. + """ + _require_capture_host() + actual_authorization = read_authorization(MARKER) + if actual_authorization != authorization or authorization["boot_id"] != Path("/proc/sys/kernel/random/boot_id").read_text().strip() or authorization["machine_id"] != Path("/etc/machine-id").read_text().strip() or authorization["engine_id"] != _engine_id(): + raise PermissionError("fixture authorization no longer selects this exact VM") + selected = RecoveryEnrollment( + "approved-management-recovery-containment-v1", + deployment_id, + enrollment_id, + str(authorization["authorization_id"]), + RECOVERY_PROFILE, + str(authorization["engine_id"]), + source_identity(), + tuple(sorted((item.original for item in captures), key=lambda item: item.generation_id)), + target, + tuple(sorted(watchers, key=lambda item: item.unit)), + True, + ) + RecoveryEnrollment.parse(json.loads(json.dumps(asdict(selected)))) + _private_directory(RECOVERY_ENROLLMENT_ROOT) + path = RECOVERY_ENROLLMENT_ROOT / (enrollment_id + ".json") + route_path = RECOVERY_ENROLLMENT_ROOT / (enrollment_id + ".routes.json") + if path.exists() or path.is_symlink() or route_path.exists() or route_path.is_symlink(): + raise FileExistsError("fixture recovery approval already exists") + _write_private(path, canonical(asdict(selected))) + approved = load_enrollment(enrollment_id) + managers = tuple(item.management_id for item in selected.originals) + (selected.target.management_id,) + docker = observe_recovery_docker(managers, tuple(item.proxy_id for item in selected.originals)) + draft = propose_unapproved_route_draft(route_selection(approved), docker, closed_profile=True) + # The isolated host harness has already audited its stock routes. This is + # the explicit separately authorized fixture approval, not observer output. + draft["complete_ordinary_routes_attested"] = True + _write_private(route_path, canonical(draft)) + observe_recovery_route_manifest(route_selection(approved), docker) + return approved diff --git a/scripts/recovery_epoch_fault_tracer.py b/scripts/recovery_epoch_fault_tracer.py new file mode 100644 index 000000000..10ddaa553 --- /dev/null +++ b/scripts/recovery_epoch_fault_tracer.py @@ -0,0 +1,351 @@ +"""Real Linux syscall barriers for the disposable recovery acceptance fixture. + +This observer never changes syscall arguments/results or supplies production +observations. It stops the actual issuer at an exclusive publication open or +at the successful directory fsync following its real file fsync. The caller +then kills the pinned process or resumes it after an external fixture mutation. +Linux ptrace ABI: https://man7.org/linux/man-pages/man2/ptrace.2.html +""" + +from __future__ import annotations + +import ctypes +import hashlib +import os +import platform +import signal +import threading +from dataclasses import dataclass +from pathlib import Path +from typing import Protocol + +from auto_coder.logger_config import get_logger + +logger = get_logger(__name__) +PHASES = ("reserved", "intent", "consumed", "receipt") +PUBLICATIONS = tuple(phase + suffix for phase in PHASES for suffix in (".json", "-head.json")) +BOUNDARY_SCENARIOS = tuple(f"{when}-{name.removesuffix('.json')}" for name in PUBLICATIONS for when in ("before", "after")) + +_PTRACE_SEIZE = 0x4206 +_PTRACE_INTERRUPT = 0x4207 +_PTRACE_GET_SYSCALL_INFO = 0x420E +_PTRACE_SYSCALL = 24 +_PTRACE_DETACH = 17 +_PTRACE_O_TRACESYSGOOD = 1 +_PTRACE_O_EXITKILL = 1 << 20 +_WAIT_ALL = 0x40000000 +_AUDIT_ARCH_X86_64 = 0xC000003E +_SYS_OPENAT = 257 +_SYS_FSYNC = 74 +_BARRIER_ACTION_TIMEOUT = 1800 + + +class _Entry(ctypes.Structure): + _fields_ = [("number", ctypes.c_uint64), ("arguments", ctypes.c_uint64 * 6)] + + +class _Exit(ctypes.Structure): + _fields_ = [("result", ctypes.c_int64), ("is_error", ctypes.c_uint8)] + + +class _Call(ctypes.Union): + _fields_ = [("entry", _Entry), ("exit", _Exit), ("space", ctypes.c_uint8 * 64)] + + +class _Information(ctypes.Structure): + _fields_ = [("operation", ctypes.c_uint8), ("padding", ctypes.c_uint8 * 3), ("architecture", ctypes.c_uint32), ("instruction", ctypes.c_uint64), ("stack", ctypes.c_uint64), ("call", _Call)] + + +def parse_boundary(scenario: str) -> tuple[str, str]: + if scenario not in BOUNDARY_SCENARIOS: + raise ValueError("a reviewed publication boundary scenario is required") + when, name = scenario.split("-", 1) + return name + ".json", when + + +def _token(pid: int) -> str: + return Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split()[19] + + +@dataclass(frozen=True) +class PublicationObservation: + process_id: int + start_token: str + boot_id: str + name: str + when: str + directory_device: int + directory_inode: int + file_device: int | None + file_inode: int | None + file_sha256: str | None + file_fsync_observed: bool + directory_fsync_observed: bool + + +class TraceProcess(Protocol): + """Actual process identity with a caller-owned termination observer.""" + + pid: int + returncode: int | None + + def wait(self, timeout: float | None = None) -> int: ... + + +class KernelPublicationBarrier: + """Attach to one actual child and retain its exact syscall stop. + + Do not call Popen.poll/wait between start and completion: this dedicated + tracer thread exclusively consumes waitpid notifications during that time. + start returns with the issuer stopped. resume_initial releases that initial + stop. wait returns with the selected native publication still stopped. + kill or release finishes tracing; close kills an unfinished tracee. + """ + + def __init__(self, issuer: TraceProcess, *, root: Path, name: str, when: str): + if platform.system() != "Linux" or platform.machine() != "x86_64": + raise ValueError("the native publication fixture requires Linux x86_64") + if name not in PUBLICATIONS or when not in {"before", "after"}: + raise ValueError("unsupported publication boundary") + if not root.is_absolute() or root.resolve(strict=True) != root or not root.is_dir(): + raise ValueError("the publication directory must be original and canonical") + self.issuer = issuer + self.root = root + self.name = name + self.when = when + self.root_identity = (root.stat().st_dev, root.stat().st_ino) + self.start_token = _token(issuer.pid) + self.boot_id = Path("/proc/sys/kernel/random/boot_id").read_text().strip() + self._pidfd = os.pidfd_open(issuer.pid) + self._ready = threading.Event() + self._resume = threading.Event() + self._hit = threading.Event() + self._action = threading.Event() + self._done = threading.Event() + self._decision = "kill" + self._error: BaseException | None = None + self._observation: object | None = None + self._thread: threading.Thread | None = None + self._file_identity: tuple[int, int] | None = None + self._file_synced = False + self._pending: str | None = None + self._libc = ctypes.CDLL(None, use_errno=True) + self._libc.ptrace.argtypes = [ctypes.c_ulong, ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p] + self._libc.ptrace.restype = ctypes.c_long + + def _ptrace(self, request: int, address: int = 0, data: int | ctypes.c_void_p = 0) -> int: + result = self._libc.ptrace(request, self.issuer.pid, ctypes.c_void_p(address), ctypes.c_void_p(data) if isinstance(data, int) else data) + if result == -1: + error = ctypes.get_errno() + raise OSError(error, os.strerror(error)) + return result + + def _require_lifetime(self) -> None: + if _token(self.issuer.pid) != self.start_token or Path("/proc/sys/kernel/random/boot_id").read_text().strip() != self.boot_id: + raise AssertionError("the traced issuer lifetime changed") + if (self.root.stat().st_dev, self.root.stat().st_ino) != self.root_identity: + raise AssertionError("the traced publication directory changed") + + def _descriptor_identity(self, descriptor: int) -> tuple[int, int]: + info = Path(f"/proc/{self.issuer.pid}/fd/{descriptor}").stat() + return info.st_dev, info.st_ino + + def _cstring(self, address: int) -> str: + with Path(f"/proc/{self.issuer.pid}/mem").open("rb", buffering=0) as memory: + value = bytearray() + for offset in range(4096): + byte = os.pread(memory.fileno(), 1, address + offset) + if byte == b"\0": + return value.decode("utf-8", "strict") + if len(byte) != 1: + break + value.extend(byte) + raise AssertionError("the native publication pathname was not bounded") + + def _observe_call(self) -> bool: + info = _Information() + length = self._ptrace(_PTRACE_GET_SYSCALL_INFO, ctypes.sizeof(info), ctypes.cast(ctypes.byref(info), ctypes.c_void_p)) + if length < 24 or info.architecture != _AUDIT_ARCH_X86_64: + raise AssertionError("the native syscall information is unavailable") + if info.operation == 1: + self._pending = None + call = info.call.entry + if call.number == _SYS_OPENAT and call.arguments[2] & (os.O_CREAT | os.O_EXCL) == os.O_CREAT | os.O_EXCL: + directory = ctypes.c_int(call.arguments[0]).value + if directory >= 0 and self._descriptor_identity(directory) == self.root_identity and self._cstring(call.arguments[1]) == self.name: + if self._file_identity is not None: + raise AssertionError("the same native publication was attempted twice") + if (self.root / self.name).exists(): + raise AssertionError("the selected publication already existed before exclusive creation") + self._pending = "open" + return self.when == "before" + elif call.number == _SYS_FSYNC and self._file_identity is not None: + identity = self._descriptor_identity(call.arguments[0]) + if identity == self._file_identity: + self._pending = "file-fsync" + elif identity == self.root_identity and self._file_synced: + self._pending = "directory-fsync" + elif info.operation == 2 and self._pending is not None: + pending, self._pending = self._pending, None + if info.call.exit.is_error or info.call.exit.result < 0: + raise AssertionError("the selected real publication syscall failed") + if pending == "open": + self._file_identity = self._descriptor_identity(info.call.exit.result) + actual = (self.root / self.name).stat() + if self._file_identity != (actual.st_dev, actual.st_ino): + raise AssertionError("the created publication differs from its actual descriptor") + elif pending == "file-fsync": + self._file_synced = True + elif pending == "directory-fsync": + return self.when == "after" + return False + + def _record(self) -> None: + self._require_lifetime() + path = self.root / self.name + device = inode = digest = None + if self.when == "after": + actual = path.stat() + device, inode = actual.st_dev, actual.st_ino + if self._file_identity != (device, inode) or not self._file_synced: + raise AssertionError("actual publication/file fsync identity is missing") + digest = hashlib.sha256(path.read_bytes()).hexdigest() + elif path.exists(): + raise AssertionError("the before-publication barrier was crossed") + self._observation = PublicationObservation(self.issuer.pid, self.start_token, self.boot_id, self.name, self.when, *self.root_identity, device, inode, digest, self._file_synced, self.when == "after") + logger.bind(native_publication=self.name, native_boundary=self.when, issuer_pid=self.issuer.pid).info("Actual issuer stopped at native publication boundary") + self._hit.set() + + def _wait_status(self) -> int: + pid, status = os.waitpid(self.issuer.pid, _WAIT_ALL) + if pid != self.issuer.pid: + raise AssertionError("a different process supplied the native trace status") + if os.WIFEXITED(status) or os.WIFSIGNALED(status): + self.issuer.returncode = os.waitstatus_to_exitcode(status) + return status + + def _run(self) -> None: + attached = False + try: + self._require_lifetime() + self._ptrace(_PTRACE_SEIZE, data=_PTRACE_O_TRACESYSGOOD | _PTRACE_O_EXITKILL) + attached = True + self._ptrace(_PTRACE_INTERRUPT) + status = self._wait_status() + if not os.WIFSTOPPED(status): + raise AssertionError("actual issuer exited before the initial trace barrier") + self._ready.set() + if not self._resume.wait(_BARRIER_ACTION_TIMEOUT): + raise TimeoutError("the actual target probe did not release the initial barrier") + self._ptrace(_PTRACE_SYSCALL) + while True: + status = self._wait_status() + if not os.WIFSTOPPED(status): + raise AssertionError("actual issuer exited before the selected publication boundary") + delivered = os.WSTOPSIG(status) + if delivered == signal.SIGTRAP | 0x80: + if self._observe_call(): + self._record() + if not self._action.wait(_BARRIER_ACTION_TIMEOUT): + raise TimeoutError("the observed publication was not explicitly released or killed") + if self._decision == "release": + self._ptrace(_PTRACE_DETACH) + attached = False + return + signal.pidfd_send_signal(self._pidfd, signal.SIGKILL) + while self.issuer.returncode is None: + self._wait_status() + return + delivered = 0 + elif delivered == signal.SIGTRAP and status >> 16: + delivered = 0 + self._ptrace(_PTRACE_SYSCALL, data=delivered) + except BaseException as error: + self._error = error + finally: + if attached and self.issuer.returncode is None: + try: + signal.pidfd_send_signal(self._pidfd, signal.SIGKILL) + while self.issuer.returncode is None: + self._wait_status() + except (OSError, ChildProcessError): + pass + self._ready.set() + self._hit.set() + self._done.set() + + def _raise_error(self) -> None: + if self._error is not None: + raise self._error + + def start(self) -> None: + if self._thread is not None: + raise ValueError("a native publication tracer cannot be restarted") + self._thread = threading.Thread(target=self._run, name="recovery-native-publication", daemon=True) + self._thread.start() + if not self._ready.wait(15): + self.close() + raise TimeoutError("the native initial trace barrier was not observed") + self._raise_error() + + def resume_initial(self) -> None: + if not self._ready.is_set() or self._done.is_set(): + self._raise_error() + raise ValueError("no active initial publication barrier") + self._resume.set() + + def wait_observation(self, timeout: float = 900) -> object: + if not self._hit.wait(timeout): + self.close() + raise TimeoutError("the actual issuer never reached the selected publication") + self._raise_error() + if self._observation is None: + raise AssertionError("the native publication observation is missing") + return self._observation + + def wait(self, timeout: float = 900) -> PublicationObservation: + observation = self.wait_observation(timeout) + if not isinstance(observation, PublicationObservation): + raise AssertionError("the native stop is not a publication observation") + return observation + + def _finish(self, action: str) -> None: + self._raise_error() + if self._observation is None or self._done.is_set(): + raise ValueError("the selected native publication is not currently stopped") + self._decision = action + self._action.set() + if not self._done.wait(15): + self.close() + raise TimeoutError("the native publication trace did not finish") + self._raise_error() + + def kill(self) -> None: + self._finish("kill") + + def release(self) -> None: + self._finish("release") + + def close(self) -> None: + if self._pidfd < 0: + return + if self._thread is not None and not self._done.is_set(): + self._decision = "kill" + self._resume.set() + self._action.set() + try: + signal.pidfd_send_signal(self._pidfd, signal.SIGKILL) + except ProcessLookupError: + pass + if not self._done.wait(15): + raise TimeoutError("the actual traced issuer did not exit during cleanup") + if self._thread is not None and self._done.is_set() and self._error is not None and self.issuer.returncode is None: + # A refused initial SEIZE never transfers waitpid ownership to the + # tracer. End the caller's exact attempted issuer here as well. + try: + signal.pidfd_send_signal(self._pidfd, signal.SIGKILL) + except ProcessLookupError: + pass + self.issuer.wait(timeout=15) + os.close(self._pidfd) + self._pidfd = -1 diff --git a/scripts/recovery_epoch_mutation_control.py b/scripts/recovery_epoch_mutation_control.py new file mode 100644 index 000000000..6a16a55b2 --- /dev/null +++ b/scripts/recovery_epoch_mutation_control.py @@ -0,0 +1,253 @@ +"""One explicit native mutation control, separate from production acceptance. + +The control starts from a private project copy before any original fixture is +created. It removes only the post-consumption final verifier, so the real +package identity, Docker images, history and enrollment all agree on that +intentionally incorrect copy. No production source or authority provider is +patched at runtime. +""" + +from __future__ import annotations + +import argparse +import ast +import copy +import hashlib +import json +import os +import re +import shutil +import sys +from pathlib import Path + +from loguru import logger + +CONTROL = "omit-post-consumption-verifier" +CONTROL_SCENARIO = "post-receipt-ready-mutation" +CONTROL_SCHEMA = "auto-coder-native-recovery-mutation-control-result-v1" +SOURCE_SCHEMA = "auto-coder-recovery-mutation-source-v1" +ORACLE = "post-receipt-grant-forbidden" +ISSUER_SOURCE = Path("src/auto_coder/management/recovery_epoch.py") +_IGNORED = frozenset({".git", ".venv", ".agent-tmp", "__pycache__", ".mypy_cache", ".pytest_cache", ".ruff_cache"}) + + +def _canonical(value: object) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False).encode() + + +def omit_post_consumption_verifier(source: bytes) -> tuple[bytes, int]: + """Delete exactly the second standalone call, proving the full AST delta.""" + original = ast.parse(source) + definitions = [node for node in original.body if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == "issue_epoch_once"] + if len(definitions) != 1: + raise ValueError("the actual source has no unique issue_epoch_once function") + function = definitions[0] + calls = sorted((node for node in ast.walk(function) if isinstance(node, ast.Expr) and isinstance(node.value, ast.Call) and isinstance(node.value.func, ast.Name) and node.value.func.id == "final_verification"), key=lambda node: node.lineno) + if len(calls) != 2 or any(node.value.args or node.value.keywords for node in calls): + raise ValueError("the actual issuer does not contain exactly two standalone final verifier calls") + first, removed = calls + parents = [node for node in ast.walk(function) if isinstance(getattr(node, "body", None), list) and removed in node.body] + if len(parents) != 1: + raise ValueError("the second final verifier has no unique original statement list") + statements = parents[0].body + index = statements.index(removed) + previous = statements[index - 1] if index else None + if not ( + isinstance(previous, ast.Assign) + and len(previous.targets) == 1 + and isinstance(previous.targets[0], ast.Name) + and previous.targets[0].id == "receipt" + and isinstance(previous.value, ast.Call) + and isinstance(previous.value.func, ast.Attribute) + and isinstance(previous.value.func.value, ast.Name) + and (previous.value.func.value.id, previous.value.func.attr) == ("state", "consume") + and len(previous.value.args) == 1 + and isinstance(previous.value.args[0], ast.Name) + and previous.value.args[0].id == "binding" + and not previous.value.keywords + and first.lineno < previous.lineno < removed.lineno + ): + raise ValueError("the selected second verifier does not directly follow actual durable consumption") + lines = source.splitlines(keepends=True) + if removed.end_lineno != removed.lineno or lines[removed.lineno - 1].strip() != b"final_verification()": + raise ValueError("the second verifier is not one exact removable source statement") + mutant = b"".join(lines[: removed.lineno - 1] + lines[removed.lineno :]) + expected = copy.deepcopy(original) + expected_function = next(node for node in expected.body if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == "issue_epoch_once") + expected_removed = next(node for node in ast.walk(expected_function) if isinstance(node, ast.Expr) and node.lineno == removed.lineno) + expected_parent = next(node for node in ast.walk(expected_function) if isinstance(getattr(node, "body", None), list) and expected_removed in node.body) + expected_parent.body.remove(expected_removed) + if ast.dump(ast.parse(mutant), include_attributes=False) != ast.dump(expected, include_attributes=False): + raise ValueError("the native mutation changed more than the second final verifier") + return mutant, removed.lineno + + +def _files(root: Path) -> dict[str, str]: + result = {} + for directory, names, files in os.walk(root): + names[:] = sorted(name for name in names if name not in _IGNORED) + if any((Path(directory) / name).is_symlink() for name in names): + raise ValueError("the mutation source contains a linked directory") + for name in sorted(files): + path = Path(directory) / name + if path.is_symlink() or not path.is_file(): + raise ValueError("the mutation source contains a nonregular file: " + str(path)) + result[path.relative_to(root).as_posix()] = hashlib.sha256(path.read_bytes()).hexdigest() + return result + + +def prepare(source: Path, destination: Path, evidence: Path) -> dict[str, object]: + """Create one distinct source copy; never overwrite or modify the original.""" + source = source.resolve(strict=True) + if not source.is_dir() or not destination.is_absolute() or destination.exists() or destination.is_symlink() or destination.is_relative_to(source): + raise ValueError("a new separate absolute mutation-copy directory is required") + if not evidence.is_absolute() or evidence.exists() or evidence.is_symlink() or evidence.is_relative_to(source) or evidence.is_relative_to(destination): + raise ValueError("new mutation evidence must be outside both source trees") + original_files = _files(source) + original = (source / ISSUER_SOURCE).read_bytes() + mutant, line = omit_post_consumption_verifier(original) + shutil.copytree(source, destination, ignore=shutil.ignore_patterns(*_IGNORED), copy_function=shutil.copy2) + selected = destination / ISSUER_SOURCE + with selected.open("wb") as stream: + stream.write(mutant) + stream.flush() + os.fsync(stream.fileno()) + copied_files = _files(destination) + changed = [path for path in original_files.keys() | copied_files.keys() if original_files.get(path) != copied_files.get(path)] + if changed != [ISSUER_SOURCE.as_posix()] or _files(source) != original_files: + raise AssertionError("the copied project differs outside the single approved verifier statement") + result = { + "schema": SOURCE_SCHEMA, + "control": CONTROL, + "scenario": CONTROL_SCENARIO, + "source_root": str(source), + "control_root": str(destination), + "changed_file": ISSUER_SOURCE.as_posix(), + "removed_call_lineno": line, + "original_source_sha256": hashlib.sha256(original).hexdigest(), + "mutant_source_sha256": hashlib.sha256(mutant).hexdigest(), + "original_project_sha256": hashlib.sha256(_canonical(original_files)).hexdigest(), + "mutant_project_sha256": hashlib.sha256(_canonical(copied_files)).hexdigest(), + "package_files": {path.removeprefix("src/auto_coder/"): digest for path, digest in copied_files.items() if path.startswith("src/auto_coder/") and path.endswith(".py")}, + "fixture_files": {path: digest for path, digest in copied_files.items() if path.startswith(("scripts/", "tests/")) and path.endswith(".py")}, + "only_second_verifier_removed": True, + "first_verifier_retained": True, + } + descriptor = os.open(evidence, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + with os.fdopen(descriptor, "wb") as stream: + stream.write(_canonical(result)) + stream.flush() + os.fsync(stream.fileno()) + logger.info("Prepared the isolated source mutation at original line {}: {}", line, selected) + return result + + +def verify_source_evidence(path: Path, *, executing: bool = False) -> tuple[dict[str, object], str]: + encoded = path.read_bytes() + value = json.loads(encoded) + if _canonical(value) != encoded or value.get("schema") != SOURCE_SCHEMA or value.get("control") != CONTROL or value.get("scenario") != CONTROL_SCENARIO: + raise ValueError("the mutation source evidence is not the exact supported canonical record") + original = Path(value["source_root"]) / ISSUER_SOURCE + changed = Path(value["control_root"]) / ISSUER_SOURCE + mutant, line = omit_post_consumption_verifier(original.read_bytes()) + if changed.read_bytes() != mutant or value["removed_call_lineno"] != line or value["original_source_sha256"] != hashlib.sha256(original.read_bytes()).hexdigest() or value["mutant_source_sha256"] != hashlib.sha256(mutant).hexdigest(): + raise AssertionError("the retained original or exact single-call mutation changed") + if value.get("only_second_verifier_removed") is not True or value.get("first_verifier_retained") is not True: + raise AssertionError("the mutation omitted its explicit single-verifier proof") + if executing: + from auto_coder.management import recovery_epoch + + package = Path(recovery_epoch.__file__).resolve(strict=True).parents[1] + actual = {item.relative_to(package).as_posix(): hashlib.sha256(item.read_bytes()).hexdigest() for item in sorted(package.rglob("*.py")) if not item.is_symlink()} + if actual != value["package_files"]: + raise AssertionError("the executing issuer package is not the complete original mutation copy") + if Path(__file__).resolve(strict=True).parents[1] != Path(value["control_root"]): + raise AssertionError("the actual fixture scripts are not running from the private copied project") + if any(hashlib.sha256((Path(value["control_root"]) / name).read_bytes()).hexdigest() != digest for name, digest in value["fixture_files"].items()): + raise AssertionError("the copied native fixture scripts changed after the single-call mutation") + return value, hashlib.sha256(encoded).hexdigest() + + +def _validate_queued_grant(queued: dict[str, object]) -> None: + if queued.get("schema") != "actual-recovery-queued-grant-v1" or queued.get("grant_schema") != "management-recovery-epoch-grant-v1": + raise AssertionError("the mutation did not observe the actual canonical recovery grant") + grant = queued["grant"] + assert set(grant) == {"schema", "granted", "request_id", "authorization_id", "target_id", "challenge", "receipt_nonce", "inventory_sha256", "binding_sha256", "consumed_sha256", "receipt_sha256", "host_boot_id", "host_pid", "host_start_token"} + assert grant["schema"] == queued["grant_schema"] and grant["granted"] == "recovery-epoch-consumed" + assert all(re.fullmatch(r"[0-9a-f]{64}", grant[name]) for name in ("challenge", "receipt_nonce", "inventory_sha256", "binding_sha256", "consumed_sha256", "receipt_sha256")) + encoded = _canonical(grant) + b"\n" + assert queued["bytes_observed"] == len(encoded) and queued["message_sha256"] == hashlib.sha256(encoded).hexdigest() + assert all(queued.get(name) is True for name in ("peek_only", "socket_flags_unchanged", "bytes_still_queued", "receiver_stopped", "issuer_alive")) + assert queued["socket_flags_before"] == queued["socket_flags_after"] and queued["socket_inode"] > 0 + assert queued["peer_credentials"] == {"pid": queued["issuer_pid"], "uid": 0, "gid": 0} + assert (queued["grant"]["target_id"], queued["grant"]["host_pid"], queued["grant"]["host_start_token"]) == (queued["target_id"], queued["issuer_pid"], queued["issuer_start_token"]) + + +class PostReceiptGrantDispatched(AssertionError): + """The same native oracle observed the precise incorrect dispatched frame.""" + + +def assert_post_receipt_refusal(observation: dict[str, object]) -> None: + assert observation["schema"] == "actual-recovery-post-receipt-dispatch-v1" + assert observation["receiver_stopped"] is True and observation["queue_empty_before_resume"] is True and observation["socket_flags_unchanged"] is True + queued = observation["queued_grant"] + if queued is not None: + _validate_queued_grant(queued) + assert observation["queue_bytes_observed"] == queued["bytes_observed"] + assert all(observation[name] == queued[name] for name in ("issuer_pid", "issuer_start_token", "target_pid", "target_start_token", "peer_credentials")) + raise PostReceiptGrantDispatched(ORACLE) + assert observation["queue_bytes_observed"] == 0 and observation["issuer_returncode"] == 2 + + +def validate_control_result(value: dict[str, object], evidence: Path) -> None: + source, digest = verify_source_evidence(evidence) + assert value["schema"] == CONTROL_SCHEMA and value["status"] == "mutation-detected" and value["control"] == CONTROL and value["scenario"] == CONTROL_SCENARIO + assert value["source_evidence_sha256"] == digest and value["source_mutation"] == source + assert value["no_child_started"] is True and value["preserved_originals"] is True and value["preserved_workers"] is True + assert value["original_outcomes"]["D"] == "never-created" and value["original_outcomes"]["C"] in {"launched", "unknown"} + oracle = value["oracle"] + assert oracle["name"] == ORACLE and oracle["detected"] is True + _validate_queued_grant(oracle["queued_grant"]) + durable = value["durable"] + assert durable["phase"] == "receipt" and durable["consumer_grant_acknowledged"] is False and durable["grant_send_completed"] is True + assert durable["mutation"]["scenario"] == CONTROL_SCENARIO and durable["mutation"]["boundary"] == "after-receipt-head" + assert durable["dispatch"]["queued_grant"] == oracle["queued_grant"] + assert durable["dispatch"]["issuer_death_before_receiver_resumed"] is True and durable["replay_returncode"] == 2 + assert durable["retained_publications"] == ["reserved.json", "reserved-head.json", "intent.json", "intent-head.json", "consumed.json", "consumed-head.json", "receipt.json", "receipt-head.json"] + assert durable["epoch_sha256_before_resume"] == durable["epoch_sha256_after_refusal"] == durable["epoch_sha256_after_replay"] + assert durable["originals_after_injection_sha256"] == value["preserved_originals_sha256"] + assert value["preservation_baseline"] == durable["mutation"]["preservation_baseline"] == "after-deliberate-fixture-mutation" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + create = commands.add_parser("prepare") + create.add_argument("--source", type=Path, required=True) + create.add_argument("--destination", type=Path, required=True) + create.add_argument("--evidence", type=Path, required=True) + run = commands.add_parser("run") + run.add_argument("--evidence", type=Path, required=True) + run.add_argument("arguments", nargs=argparse.REMAINDER) + validate = commands.add_parser("validate") + validate.add_argument("--evidence", type=Path, required=True) + validate.add_argument("--result", type=Path, required=True) + args = parser.parse_args(argv) + logger.remove() + logger.add(sys.stdout, colorize=True) + logger.add(str(args.evidence) + ".log", rotation="10 MB") + if args.command == "prepare": + prepare(args.source, args.destination, args.evidence) + elif args.command == "validate": + validate_control_result(json.loads(args.result.read_bytes()), args.evidence) + else: + verify_source_evidence(args.evidence, executing=True) + from scripts.test_management_recovery_epoch_host import main as run_fixture + + arguments = args.arguments[1:] if args.arguments[:1] == ["--"] else args.arguments + return run_fixture([*arguments, "--mutation-control-manifest", str(args.evidence)]) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/recovery_epoch_native_negatives.py b/scripts/recovery_epoch_native_negatives.py new file mode 100644 index 000000000..86c20a336 --- /dev/null +++ b/scripts/recovery_epoch_native_negatives.py @@ -0,0 +1,842 @@ +"""Actual refusal injections for the separately authorized recovery epoch VM. + +The calling driver supplies only process orchestration and a preservation +projection. Every authority decision still comes from the original production +issuer or the separate read-only containment CLI. No injection runs after an +original epoch publication, and an unexpected publication is never repaired. +""" + +from __future__ import annotations + +import fcntl +import hashlib +import json +import os +import re +import select +import shutil +import signal +import socket +import stat +import subprocess +import sys +import time +import uuid +from contextlib import contextmanager, nullcontext +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Callable, ContextManager, Iterator + +from scripts.test_management_containment_closed_host import _NONLEADER, MARKER, read_authorization +from scripts.test_management_containment_enrollment_host import _docker, _systemctl, _write_private + +from auto_coder.logger_config import get_logger +from auto_coder.management.containment_kernel_subobservation import _holder_start_token, _require_capture_host +from auto_coder.management.containment_route_manifest import _systemd_catalog, _systemd_unit +from auto_coder.management.fresh_host import _evidence +from auto_coder.management.fresh_host_observation import _engine_id, assert_host_listener_absent +from auto_coder.management.installed_bootstrap_mount_profile import _inspect +from auto_coder.management.original_attempts import canonical +from auto_coder.management.recovery_containment_model import RECOVERY_ENROLLMENT_ROOT, NamedDescriptor, RecoveryEnrollment, RecoveryLock, RecoveryOriginal, RecoveryResources +from auto_coder.management.recovery_containment_profile import digest +from auto_coder.management.recovery_diagnostics import safe_refusal_causes +from auto_coder.management.recovery_epoch_peer import inspect_kernel_lock +from auto_coder.management.recovery_epoch_state import EPOCH_ROOT + +logger = get_logger(__name__) +_FIXTURE_ROOT = Path("/var/lib/auto-coder/recovery-epoch-fixtures") +_UNRESERVED_ENTRIES = frozenset({"epoch.lock", "inventory.json"}) +_FACTORY_ENVIRONMENT_OVERRIDES = ( + ("factory-epoch-root-environment-refused", "AUTO_CODER_RECOVERY_EPOCH_ROOT"), + ("factory-inventory-environment-refused", "AUTO_CODER_RECOVERY_EPOCH_INVENTORY"), + ("factory-authorization-environment-refused", "AUTO_CODER_RECOVERY_AUTHORIZATION_ROOT"), + ("factory-containment-environment-refused", "AUTO_CODER_RECOVERY_CONTAINMENT_ROOT"), +) +_FACTORY_ARGUMENT_OVERRIDES = (("factory-root-argument-refused", "--root"), ("factory-inventory-argument-refused", "--inventory")) +REQUIRED_NEGATIVE_CASES = frozenset( + { + "busy-original-owner-refused", + "busy-controller-fence-refused", + "missing-original-owner-refused", + "replaced-original-owner-refused", + "replaced-epoch-lock-refused", + "unknown-docker-management-route-refused", + "omitted-installed-controller-refused", + "unmasked-installed-restarter-refused", + "retained-old-network-task-and-listener-refused", + "escaped-old-network-nonleader-refused", + "old-published-listener-refused", + *(label for label, _ in _FACTORY_ENVIRONMENT_OVERRIDES + _FACTORY_ARGUMENT_OVERRIDES), + } +) +REQUIRED_REPORT_CASES = frozenset( + { + "native-current-containment-reverified", + "forged-native-containment-refused", + "cross-deployment-containment-refused", + "stale-native-containment-refused", + "escaped-nonleader-current-containment-refused", + "measured-exited-nonleader-current-containment-reverified", + } +) + + +@dataclass(frozen=True) +class NativeNegativeContext: + enrollment: RecoveryEnrollment + fixture_root: Path + epoch_root: Path + + +class EpochAlreadyPublished(AssertionError): + """The fresh fixture has become uncertain and must never be reused.""" + + +class NativeContainmentFailure(AssertionError): + """Carry only bounded non-authorizing CLI metadata to the failure artifact.""" + + def __init__(self, message: str, *, action: str, returncode: int, causes: object): + super().__init__(message) + self.diagnostics = {"schema": "native-recovery-containment-cli-failure-v1", "action": action, "returncode": returncode, "causes": safe_refusal_causes(causes)} + + +def assert_unreserved_epoch(root: Path) -> None: + """Reject every record, head, partial name or unexpected epoch object.""" + if not root.is_absolute() or root != root.resolve(strict=True): + raise EpochAlreadyPublished("the fixed epoch directory is no longer canonical; never repair or retry") + directory = root.lstat() + if not stat.S_ISDIR(directory.st_mode) or directory.st_uid != os.geteuid() or stat.S_IMODE(directory.st_mode) != 0o700 or {path.name for path in root.iterdir()} != _UNRESERVED_ENTRIES: + raise EpochAlreadyPublished("the epoch has an original publication, partial state or changed root; never repair or retry") + for name in _UNRESERVED_ENTRIES: + info = (root / name).lstat() + if not stat.S_ISREG(info.st_mode) or info.st_uid != os.geteuid() or stat.S_IMODE(info.st_mode) != 0o600 or info.st_nlink != 1: + raise EpochAlreadyPublished("an original epoch input is missing or replaced by an unsupported object; never repair or retry") + + +def _projection(path: Path) -> tuple[object, ...]: + info = path.lstat() + if not stat.S_ISREG(info.st_mode): + raise AssertionError("the fixture selected a nonregular original") + return (info.st_dev, info.st_ino, info.st_mode, info.st_uid, info.st_gid, info.st_nlink, info.st_size, info.st_mtime_ns, hashlib.sha256(path.read_bytes()).hexdigest()) + + +def _epoch_snapshot(root: Path) -> str: + assert_unreserved_epoch(root) + info = root.lstat() + return digest(((info.st_dev, info.st_ino, info.st_mode, info.st_uid, info.st_gid), tuple((name, _projection(root / name)) for name in sorted(_UNRESERVED_ENTRIES)))) + + +def require_native_context(context: NativeNegativeContext, *, unreserved: bool = True) -> None: + """Verify the original VM approval before a separately bounded scenario. + + Late interruption fixtures select ``unreserved=False`` only when they also + verify their exact original publication prefix; it never permits a reset. + """ + if context.epoch_root != EPOCH_ROOT or context.fixture_root == _FIXTURE_ROOT or not context.fixture_root.is_relative_to(_FIXTURE_ROOT) or context.fixture_root != context.fixture_root.resolve(strict=True): + raise PermissionError("native refusal injections require the fixed disposable fixture roots") + _require_capture_host() + authority = read_authorization(MARKER) + selected = context.enrollment + if ( + authority["authorization_id"] != selected.authorization_id + or authority["boot_id"] != Path("/proc/sys/kernel/random/boot_id").read_text().strip() + or authority["machine_id"] != Path("/etc/machine-id").read_text().strip() + or authority["engine_id"] != selected.engine_id + or authority["engine_id"] != _engine_id() + ): + raise PermissionError("the original separate VM authorization no longer selects this fixture") + RecoveryEnrollment.parse(json.loads(json.dumps(asdict(selected)))) + if not any(original.state == "started" for original in selected.originals) or not selected.watchers: + raise AssertionError("native negatives require actual original and controller resources") + for lock in (*(original.owner for original in selected.originals if original.owner is not None), *(watcher.lock for watcher in selected.watchers), selected.target.owner): + if not Path(lock.path).is_relative_to(context.fixture_root): + raise PermissionError("a selected negative resource is outside the authorized fixture") + if unreserved: + assert_unreserved_epoch(context.epoch_root) + + +@contextmanager +def _replacement(epoch_root: Path, path: Path, backup: Path, contents: bytes) -> Iterator[None]: + """Retain the original inode; restore only our still-unchanged injection.""" + assert_unreserved_epoch(epoch_root) + original = _projection(path) + if path != path.resolve(strict=True) or backup.exists() or backup.is_symlink() or backup.parent.stat().st_dev != path.parent.stat().st_dev: + raise AssertionError("fixture replacement requires a unique same-filesystem backup") + path.rename(backup) + replacement: tuple[object, ...] | None = None + try: + _write_private(path, contents) + replacement = _projection(path) + if replacement[:2] == original[:2]: + raise AssertionError("fixture replacement did not select a different inode") + yield + finally: + # A publication makes even an epoch-lock restoration an impermissible + # retry/reset. Keep all original and injected objects for diagnosis. + assert_unreserved_epoch(epoch_root) + if _projection(backup) != original: + raise AssertionError("the retained original changed during the negative fixture") + if replacement is not None: + if _projection(path) != replacement: + raise AssertionError("the replacement changed; refusing fixture cleanup") + path.unlink() + elif path.exists() or path.is_symlink(): + raise AssertionError("an incomplete replacement is retained for diagnosis") + backup.rename(path) + if _projection(path) != original: + raise AssertionError("fixture cleanup did not restore the exact original object") + + +@contextmanager +def _missing_original(epoch_root: Path, path: Path, backup: Path) -> Iterator[None]: + """Remove only the selected fixture path while retaining its original inode.""" + assert_unreserved_epoch(epoch_root) + original = _projection(path) + if path != path.resolve(strict=True) or backup.exists() or backup.is_symlink() or backup.parent.stat().st_dev != path.parent.stat().st_dev: + raise AssertionError("fixture absence requires a unique same-filesystem backup") + path.rename(backup) + try: + yield + finally: + assert_unreserved_epoch(epoch_root) + if _projection(backup) != original: + raise AssertionError("the absent original changed during the negative fixture") + if path.exists() or path.is_symlink(): + raise AssertionError("the missing original was recreated; refusing fixture cleanup") + backup.rename(path) + if _projection(path) != original: + raise AssertionError("fixture absence cleanup did not restore the exact original") + + +@contextmanager +def _busy_fence(selected: RecoveryLock) -> Iterator[None]: + descriptor = os.open(selected.path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC) + try: + info = os.fstat(descriptor) + if (info.st_dev, info.st_ino) != (selected.device, selected.inode) or not stat.S_ISREG(info.st_mode): + raise AssertionError("fixture contention did not open the selected original") + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + lock = inspect_kernel_lock(os.getpid(), descriptor, selected.device, selected.inode) + if lock is None or lock.mode != "WRITE" or lock.pid != os.getpid(): + raise AssertionError("fixture contender lacks its real independently acquired EX") + yield + if inspect_kernel_lock(os.getpid(), descriptor, selected.device, selected.inode) != lock: + raise AssertionError("the actual fixture contention was released during refusal") + finally: + os.close(descriptor) + + +@contextmanager +def _live_process(command: tuple[str, ...], *, pass_fds: tuple[int, ...] = ()) -> Iterator[subprocess.Popen[bytes]]: + """Bound a real injection process to its original pidfd and ready message.""" + process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, pass_fds=pass_fds) + lifetime = -1 + try: + lifetime = os.pidfd_open(process.pid, 0) + if process.stdout is None: + raise AssertionError("fixture process has no readiness pipe") + message = b"" + deadline = time.monotonic() + 15 + while b"\n" not in message and len(message) <= 64: + left = deadline - time.monotonic() + if left <= 0 or not select.select((process.stdout,), (), (), left)[0]: + raise AssertionError("actual negative process never reached its bounded readiness barrier") + chunk = os.read(process.stdout.fileno(), 65 - len(message)) + if not chunk: + raise AssertionError("actual negative process exited before readiness") + message += chunk + if message != b"READY\n" or process.poll() is not None: + raise AssertionError("actual negative process did not report exact readiness") + yield process + if process.poll() is not None: + raise AssertionError("the actual injected process exited during refusal") + finally: + if process.poll() is None: + if lifetime >= 0: + signal.pidfd_send_signal(lifetime, signal.SIGTERM) + else: + process.terminate() + try: + process.communicate(timeout=10) + except subprocess.TimeoutExpired: + if lifetime >= 0: + signal.pidfd_send_signal(lifetime, signal.SIGKILL) + else: + process.kill() + process.communicate(timeout=10) + if lifetime >= 0: + os.close(lifetime) + + +_NAMESPACE_LISTENER = """ +import os +import socket +import sys +os.setns(int(sys.argv[1]), os.CLONE_NEWNET) +os.close(int(sys.argv[1])) +listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +listener.bind(('127.0.0.1', 8080)) +listener.listen(1) +print('READY', flush=True) +sys.stdin.buffer.read(1) +""" + + +@contextmanager +def _retained_namespace_listener(original: RecoveryOriginal) -> Iterator[None]: + pin = original.namespace_pin + if pin is None or original.prior_evidence is None: + raise AssertionError("a genuine retained original namespace is required") + if _holder_start_token(pin.holder_pid) != pin.holder_start_token: + raise AssertionError("the original namespace holder changed before injection") + descriptor = os.open(f"/proc/{pin.holder_pid}/fd/{pin.holder_fd}", os.O_RDONLY | os.O_CLOEXEC) + try: + info = os.fstat(descriptor) + if (info.st_dev, info.st_ino) != (pin.device, pin.inode) or os.readlink(f"/proc/self/fd/{descriptor}") != pin.namespace: + raise AssertionError("the injected process did not receive the exact retained namespace") + with _live_process((sys.executable, "-I", "-B", "-c", _NAMESPACE_LISTENER, str(descriptor)), pass_fds=(descriptor,)) as process: + proc = Path("/proc") / str(process.pid) + prior, _, _ = _evidence(Path(original.prior_evidence.path)) + if os.readlink(proc / "ns/net") != pin.namespace or "0::" + prior.cgroup_path in (proc / "cgroup").read_text().splitlines(): + raise AssertionError("the real task did not remain outside the old container in its retained network") + rows = (proc / "net/tcp").read_text().splitlines()[1:] + if not any(row.split()[1].endswith(":1F90") and row.split()[3] == "0A" for row in rows): + raise AssertionError("the actual retained namespace has no injected listening socket") + yield + finally: + os.close(descriptor) + + +@dataclass(frozen=True) +class NativeTaskObservation: + pid: int + tid: int + start_token: str + state: str + network_namespace: str | None + cgroups: tuple[str, ...] + + +@dataclass(frozen=True) +class EscapedNonleaderObservation: + generation_id: str + original_namespace: str + original_cgroup: str + leader: NativeTaskObservation + nonleader: NativeTaskObservation + + +@dataclass(frozen=True) +class ExitedNonleaderObservation: + leader: NativeTaskObservation + nonleader_tid: int + nonleader_absent: bool + remaining_tids: tuple[int, ...] + wait_code: int + wait_status: int + pidfd_readable: bool + + +def _task_observation(pid: int, tid: int) -> NativeTaskObservation: + """Read one actual kernel task twice; missing namespace alone is not exit.""" + if type(pid) is not int or type(tid) is not int or min(pid, tid) <= 1: + raise AssertionError("native task observation requires exact positive identities") + task = Path(f"/proc/{pid}/task/{tid}") + + def identity() -> tuple[str, str]: + value = (task / "stat").read_text(encoding="ascii") + prefix, separator, tail = value.rpartition(")") + fields = tail.split() + if len(value) > 4096 or not separator or prefix.partition(" (")[0] != str(tid) or len(fields) < 20 or not fields[19].isdecimal() or int(fields[19]) <= 0: + raise AssertionError("the actual task lifetime is incomplete") + return fields[0], fields[19] + + before = identity() + try: + namespace = os.readlink(task / "ns/net") + except FileNotFoundError: + namespace = None + groups = tuple((task / "cgroup").read_text(encoding="ascii").splitlines()) + after = identity() + if before[1] != after[1] or (before[0] == "Z") != (after[0] == "Z") or len(groups) != 1 or not groups[0].startswith("0::/") or namespace is not None and re.fullmatch(r"net:\[[0-9]+\]", namespace) is None: + raise AssertionError("the actual task changed or left the measured kernel profile") + return NativeTaskObservation(pid, tid, after[1], after[0], namespace, groups) + + +def _escaped_task_observation(original: RecoveryOriginal, pid: int, tid: int) -> EscapedNonleaderObservation: + pin = original.namespace_pin + if pin is None or original.prior_evidence is None or tid == pid: + raise AssertionError("a real nonleader and original namespace are required") + prior, _, _ = _evidence(Path(original.prior_evidence.path)) + leader, nonleader = _task_observation(pid, pid), _task_observation(pid, tid) + if leader.network_namespace != os.readlink("/proc/self/ns/net") or leader.network_namespace == pin.namespace or nonleader.network_namespace != pin.namespace: + raise AssertionError("the escaped nonleader lacks the independently observed namespace split") + for task in (leader, nonleader): + group = task.cgroups[0][3:] + if task.state in {"Z", "X", "x"} or group == prior.cgroup_path or group.startswith(prior.cgroup_path + "/"): + raise AssertionError("the live escaped task is not outside the original cgroup") + return EscapedNonleaderObservation(original.generation_id, pin.namespace, prior.cgroup_path, leader, nonleader) + + +def _same_live_task(left: NativeTaskObservation, right: NativeTaskObservation) -> bool: + return (left.pid, left.tid, left.start_token, left.network_namespace, left.cgroups) == (right.pid, right.tid, right.start_token, right.network_namespace, right.cgroups) and right.state not in {"Z", "X", "x"} + + +def _wait_unreaped_exit(pid: int, descriptor: int, start_token: str, nonleader_tid: int) -> ExitedNonleaderObservation: + """Keep the exact zombie available for a separate production verifier.""" + poller = select.poll() + poller.register(descriptor, select.POLLIN) + if not poller.poll(15000): + raise AssertionError("the exact nonleader process did not exit at its pidfd") + waited = os.waitid(os.P_PIDFD, descriptor, os.WEXITED | os.WNOWAIT | os.WNOHANG) + if waited is None or waited.si_pid != pid or waited.si_code not in {os.CLD_EXITED, os.CLD_KILLED, os.CLD_DUMPED}: + raise AssertionError("the original child has no independently observed unreaped exit") + leader = _task_observation(pid, pid) + tasks = tuple(sorted(int(path.name) for path in Path(f"/proc/{pid}/task").iterdir())) + absent = not Path(f"/proc/{pid}/task/{nonleader_tid}").exists() + if nonleader_tid == pid or leader.start_token != start_token or leader.state != "Z" or leader.network_namespace is not None or not absent or tasks != (pid,): + raise AssertionError("the original task is not a measured namespace-free zombie with its nonleader gone") + if _task_observation(pid, pid) != leader or not poller.poll(0): + raise AssertionError("the exact exited task changed during independent observation") + return ExitedNonleaderObservation(leader, nonleader_tid, absent, tasks, waited.si_code, waited.si_status, True) + + +@dataclass +class NativeNonleaderProcess: + process: subprocess.Popen[bytes] + descriptor: int + original: RecoveryOriginal + before: EscapedNonleaderObservation + exited: ExitedNonleaderObservation | None = None + + def observe_live(self) -> EscapedNonleaderObservation: + poller = select.poll() + poller.register(self.descriptor, select.POLLIN) + if self.exited is not None or poller.poll(0): + raise AssertionError("the original escaped nonleader ended before refusal") + actual = _escaped_task_observation(self.original, self.process.pid, self.before.nonleader.tid) + if not _same_live_task(self.before.leader, actual.leader) or not _same_live_task(self.before.nonleader, actual.nonleader): + raise AssertionError("the exact escaped nonleader lifetime or custody changed") + return actual + + def exit_unreaped(self) -> ExitedNonleaderObservation: + self.observe_live() + signal.pidfd_send_signal(self.descriptor, signal.SIGTERM) + self.exited = _wait_unreaped_exit(self.process.pid, self.descriptor, self.before.leader.start_token, self.before.nonleader.tid) + return self.exited + + def observe_exited(self) -> ExitedNonleaderObservation: + if self.exited is None: + raise AssertionError("no measured task exit precedes the positive control") + actual = _wait_unreaped_exit(self.process.pid, self.descriptor, self.before.leader.start_token, self.before.nonleader.tid) + if actual != self.exited: + raise AssertionError("the unreaped original lifetime changed during the positive control") + return actual + + +@contextmanager +def _escaped_namespace_nonleader(original: RecoveryOriginal) -> Iterator[NativeNonleaderProcess]: + """Reuse the real closed-host thread producer with exact original pin custody.""" + pin = original.namespace_pin + if pin is None or original.prior_evidence is None or _holder_start_token(pin.holder_pid) != pin.holder_start_token: + raise AssertionError("the original retained namespace holder is unavailable") + namespace = os.open(f"/proc/{pin.holder_pid}/fd/{pin.holder_fd}", os.O_RDONLY | os.O_CLOEXEC) + process: subprocess.Popen[bytes] | None = None + lifetime = -1 + try: + info = os.fstat(namespace) + if (info.st_dev, info.st_ino) != (pin.device, pin.inode) or os.readlink(f"/proc/self/fd/{namespace}") != pin.namespace: + raise AssertionError("the nonleader fixture did not retain the exact original namespace") + process = subprocess.Popen((sys.executable, "-I", "-B", "-c", _NONLEADER, str(namespace)), pass_fds=(namespace,), stdout=subprocess.PIPE, stderr=subprocess.PIPE) + lifetime = os.pidfd_open(process.pid, 0) + assert process.stdout is not None + message = b"" + deadline = time.monotonic() + 15 + while b"\n" not in message and len(message) <= 256: + left = deadline - time.monotonic() + if left <= 0 or not select.select((process.stdout,), (), (), left)[0]: + raise AssertionError("the real escaped nonleader did not reach its readiness barrier") + chunk = os.read(process.stdout.fileno(), 257 - len(message)) + if not chunk: + raise AssertionError("the real escaped nonleader exited before readiness") + message += chunk + ready = json.loads(message) + if not message.endswith(b"\n") or len(message) > 256 or not isinstance(ready, dict) or set(ready) != {"pid", "tid"} or ready["pid"] != process.pid or type(ready["tid"]) is not int or ready["tid"] == process.pid: + raise AssertionError("the real escaped nonleader readiness identity differs") + held = NativeNonleaderProcess(process, lifetime, original, _escaped_task_observation(original, process.pid, ready["tid"])) + held.observe_live() + yield held + if held.exited is None: + held.observe_live() + else: + held.observe_exited() + finally: + if process is not None: + if lifetime >= 0: + poller = select.poll() + poller.register(lifetime, select.POLLIN) + if not poller.poll(0): + signal.pidfd_send_signal(lifetime, signal.SIGTERM) + elif process.poll() is None: + process.terminate() + try: + process.communicate(timeout=15) + except subprocess.TimeoutExpired: + if lifetime >= 0: + signal.pidfd_send_signal(lifetime, signal.SIGKILL) + else: + process.kill() + process.communicate(timeout=15) + if lifetime >= 0: + os.close(lifetime) + os.close(namespace) + + +@contextmanager +def _record_nonleader_refusal(context: NativeNegativeContext, original: RecoveryOriginal) -> Iterator[None]: + with _escaped_namespace_nonleader(original) as held: + before = held.observe_live() + yield + after = held.observe_live() + _write_private(context.fixture_root / "native-nonleader-issuer-observation.json", canonical({"case": "escaped-old-network-nonleader-refused", "before": asdict(before), "after": asdict(after), "epoch_sha256": _epoch_snapshot(context.epoch_root)})) + + +@contextmanager +def _published_listener(original: RecoveryOriginal, target_port: int) -> Iterator[None]: + if original.prior_evidence is None: + raise AssertionError("the old listener requires actual original capture") + prior, _, _ = _evidence(Path(original.prior_evidence.path)) + if prior.published_port == target_port: + raise AssertionError("the fixture target reused an original published port") + assert_host_listener_absent(prior.published_port) + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener: + listener.bind(("127.0.0.1", prior.published_port)) + listener.listen(1) + try: + assert_host_listener_absent(prior.published_port) + except ValueError: + pass + else: + raise AssertionError("the actual host listener injection was not visible") + yield + assert_host_listener_absent(prior.published_port) + + +@contextmanager +def _extra_docker(enrollment: RecoveryEnrollment) -> Iterator[None]: + image = enrollment.originals[0].management_image_id + identifier = _docker("create", "--name", "ac-recovery-unknown-" + uuid.uuid4().hex, "--restart=no", "--network=none", "--read-only", "--entrypoint", "python", image, "-m", "auto_coder.management.bootstrap_supervisor", "--config", "/bootstrap/config.json") + if re.fullmatch(r"[0-9a-f]{64}", identifier) is None: + raise AssertionError("the actual extra Docker object has no complete identity") + try: + observed = _inspect(identifier) + state = observed.get("State") + if observed.get("Image") != image or not isinstance(state, dict) or state.get("Status") != "created" or state.get("Pid") != 0 or state.get("Running") is not False or observed.get("RestartCount") != 0: + raise AssertionError("the actual unknown manager route is not a pristine created object") + yield + if _inspect(identifier) != observed: + raise AssertionError("the issuer changed the injected Docker object") + finally: + current = _inspect(identifier) + state = current.get("State") + if not isinstance(state, dict) or state.get("Status") != "created" or state.get("Pid") != 0 or state.get("Running") is not False or current.get("Image") != image: + raise AssertionError("the injected object acquired a lifetime; refusing destructive cleanup") + _docker("rm", identifier) + + +@contextmanager +def _omitted_controller(context: NativeNegativeContext, backups: Path) -> Iterator[None]: + path = RECOVERY_ENROLLMENT_ROOT / (context.enrollment.enrollment_id + ".routes.json") + previous = json.loads(path.read_bytes()) + routes = previous.get("systemd_routes") + selected = context.enrollment.watchers[0].unit + if not isinstance(routes, list) or sum(isinstance(route, dict) and route.get("unit") == selected for route in routes) != 1: + raise AssertionError("the approved full manifest does not contain the actual selected watcher") + previous["systemd_routes"] = [route for route in routes if route["unit"] != selected] + with _replacement(context.epoch_root, path, backups / "approved-routes.original", canonical(previous)): + yield + + +@contextmanager +def _unmasked_restarter(context: NativeNegativeContext) -> Iterator[None]: + """Add an actual runnable restarter without touching original mask inodes.""" + catalog = _systemd_catalog() + executable = shutil.which("docker") + if executable is None: + raise AssertionError("the original local Docker executable is unavailable") + executable = str(Path(executable).resolve(strict=True)) + if any(character in executable for character in ' \t\n\r%"\\'): + raise AssertionError("the native Docker executable is outside the finite systemd profile") + name = "auto-coder-recovery-negative-" + uuid.uuid4().hex + ".service" + path = Path("/run/systemd/system") / name + contents = f"[Unit]\nDescription=Disposable management recovery restarter negative\n[Service]\nType=oneshot\nExecStart={executable} start {context.enrollment.originals[0].management_id}\n".encode() + _write_private(path, contents) + identity = _projection(path) + try: + _systemctl("daemon-reload") + # Match the enrolled container identity, as the production route scan does. + actual = _systemd_unit(name, extra_cues=(context.enrollment.originals[0].management_id,)) + if actual.inhibited or not actual.management_candidate or actual.main_pid != 0 or actual.active_state != "inactive" or name not in _systemd_catalog(): + raise AssertionError("the actual installed restarter is not the intended unmasked inactive route") + yield + finally: + if _projection(path) != identity: + raise AssertionError("the fixture restarter source changed; refusing cleanup") + path.unlink() + _systemctl("daemon-reload") + deadline = time.monotonic() + 15 + while _systemd_catalog() != catalog: + if time.monotonic() >= deadline: + raise AssertionError("the original systemd catalog did not return after exact fixture removal") + time.sleep(0.1) + + +def _run_case(context: NativeNegativeContext, label: str, injection: Callable[[], ContextManager[None]], run_refusal: Callable[[str], None], preserved_snapshot: Callable[[], object]) -> None: + assert_unreserved_epoch(context.epoch_root) + baseline = preserved_snapshot() + epoch_before = _epoch_snapshot(context.epoch_root) + logger.info("Starting native prepublication refusal case: {}", label) + with injection(): + injected = preserved_snapshot() + epoch_injected = _epoch_snapshot(context.epoch_root) + try: + run_refusal(label) + finally: + assert_unreserved_epoch(context.epoch_root) + if _epoch_snapshot(context.epoch_root) != epoch_injected: + raise AssertionError("the refused issuer changed an original epoch input") + if preserved_snapshot() != injected: + raise AssertionError("the refused issuer changed original/OAuth/worker state during " + label) + if _epoch_snapshot(context.epoch_root) != epoch_before or preserved_snapshot() != baseline: + raise AssertionError("the exact injected fixture state did not restore after " + label) + logger.info("Native prepublication refusal and preservation verified: {}", label) + + +@dataclass(frozen=True) +class FactoryOverrideObservation: + case: str + argument: str + environment_variable: str + selected_path: str + returncode: int + refusal: str + alternate_inputs_sha256: str + epoch_sha256: str + + +def _alternate_input_snapshot(root: Path) -> str: + """Bind the real alternate directory, including new or removed entries.""" + if root != root.resolve(strict=True): + raise AssertionError("alternate fixture inputs are no longer canonical") + paths = (root, *sorted(root.rglob("*"))) + if len(paths) > 32: + raise AssertionError("alternate fixture input inventory exceeds its bound") + entries = [] + for path in paths: + info = path.lstat() + if stat.S_ISREG(info.st_mode): + contents = hashlib.sha256(path.read_bytes()).hexdigest() + elif stat.S_ISDIR(info.st_mode): + contents = "directory" + else: + raise AssertionError("alternate fixture input has an unsupported object") + entries.append((str(path.relative_to(root)), info.st_dev, info.st_ino, info.st_mode, info.st_uid, info.st_gid, info.st_nlink, info.st_size, info.st_mtime_ns, info.st_ctime_ns, contents)) + return digest(entries) + + +def _run_factory_overrides(context: NativeNegativeContext, preserved_snapshot: Callable[[], object]) -> list[str]: + """Reject real CLI selectors before either fixed or alternate authority moves.""" + if any(name in os.environ for _, name in _FACTORY_ENVIRONMENT_OVERRIDES): + raise AssertionError("the native fixture environment already contains an authority override") + directory = context.fixture_root / "native-factory-overrides" + directory.mkdir(mode=0o700) + alternate = directory / "alternate-authority" + alternate.mkdir(mode=0o700) + for name in sorted(_UNRESERVED_ENTRIES): + _write_private(alternate / name, (context.epoch_root / name).read_bytes()) + observed: list[FactoryOverrideObservation] = [] + selectors = tuple((label, argument, "") for label, argument in _FACTORY_ARGUMENT_OVERRIDES) + tuple((label, "", variable) for label, variable in _FACTORY_ENVIRONMENT_OVERRIDES) + for label, argument, variable in selectors: + selected = alternate / "inventory.json" if argument == "--inventory" or variable == "AUTO_CODER_RECOVERY_EPOCH_INVENTORY" else alternate + + def refuse(actual_label: str) -> None: + before = _alternate_input_snapshot(alternate) + environment = dict(os.environ, PYTHONDONTWRITEBYTECODE="1") + command = [sys.executable, "-B", "-m", "auto_coder.management.recovery_epoch"] + if argument: + command.extend((argument, str(selected))) + else: + environment[variable] = str(selected) + refusal = "unrecognized arguments: " + argument if argument else "request/environment recovery inventory overrides are unsupported" + try: + completed = subprocess.run(command, env=environment, capture_output=True, text=True, timeout=30) + output = completed.stdout + completed.stderr + _write_private(directory / (actual_label + ".log"), output.encode()) + if completed.returncode != 2 or refusal not in output or "Fixed recovery issuer endpoint is listening" in output or "One consumed recovery epoch grant send completed" in output: + raise AssertionError("the actual production issuer accepted or misclassified a factory override: " + actual_label) + finally: + assert_unreserved_epoch(context.epoch_root) + if _alternate_input_snapshot(alternate) != before: + raise AssertionError("the actual production issuer changed alternate authority inputs") + observed.append(FactoryOverrideObservation(actual_label, argument, variable, str(selected), completed.returncode, refusal, before, _epoch_snapshot(context.epoch_root))) + + _run_case(context, label, nullcontext, refuse, preserved_snapshot) + _write_private(context.fixture_root / "native-factory-override-observations.json", canonical({"schema": "actual-recovery-factory-overrides-v1", "cases": [asdict(item) for item in observed]})) + return [item.case for item in observed] + + +def run_native_negatives(context: NativeNegativeContext, run_refusal: Callable[[str], None], preserved_snapshot: Callable[[], object]) -> list[str]: + """Exercise actual issuer refusals before its first irreversible publication.""" + require_native_context(context) + backups = context.fixture_root / "native-negative-originals" + backups.mkdir(mode=0o700) + original = next(item for item in context.enrollment.originals if item.state == "started") + if original.owner is None: + raise AssertionError("the original started owner identity is absent") + selected_owner = original.owner + owner = Path(selected_owner.path) + epoch = context.epoch_root / "epoch.lock" + injections: tuple[tuple[str, Callable[[], ContextManager[None]]], ...] = ( + ("busy-original-owner-refused", lambda: _busy_fence(selected_owner)), + ("busy-controller-fence-refused", lambda: _busy_fence(context.enrollment.watchers[0].lock)), + ("missing-original-owner-refused", lambda: _missing_original(context.epoch_root, owner, backups / "absent-owner.original")), + ("replaced-original-owner-refused", lambda: _replacement(context.epoch_root, owner, backups / "owner.original", owner.read_bytes())), + ("replaced-epoch-lock-refused", lambda: _replacement(context.epoch_root, epoch, backups / "epoch.original", epoch.read_bytes())), + ("unknown-docker-management-route-refused", lambda: _extra_docker(context.enrollment)), + ("omitted-installed-controller-refused", lambda: _omitted_controller(context, backups)), + ("unmasked-installed-restarter-refused", lambda: _unmasked_restarter(context)), + ("retained-old-network-task-and-listener-refused", lambda: _retained_namespace_listener(original)), + ("escaped-old-network-nonleader-refused", lambda: _record_nonleader_refusal(context, original)), + ("old-published-listener-refused", lambda: _published_listener(original, context.enrollment.target.published_port)), + ) + cases = [] + for label, injection in injections: + _run_case(context, label, injection, run_refusal, preserved_snapshot) + cases.append(label) + cases.extend(_run_factory_overrides(context, preserved_snapshot)) + if set(cases) != REQUIRED_NEGATIVE_CASES: + raise AssertionError("the native refusal matrix is incomplete") + return cases + + +def _issuer_resources(enrollment: RecoveryEnrollment, issuer_pid: int) -> RecoveryResources: + """Read actual stopped issuer descriptors; the production verifier rechecks.""" + if type(issuer_pid) is not int or issuer_pid <= 1: + raise AssertionError("an exact actual issuer process is required") + proc = Path("/proc") / str(issuer_pid) + fields = (proc / "stat").read_text().rsplit(")", 1)[1].split() + if fields[0] not in {"T", "t"}: + raise AssertionError("native report probes require the actual prepublication SIGSTOP/ptrace barrier") + start = _holder_start_token(issuer_pid) + descriptors = tuple((proc / "fd").iterdir()) + if len(descriptors) > 4096 or any(not item.name.isdecimal() for item in descriptors): + raise AssertionError("the actual issuer descriptor inventory exceeds the fixture bound") + + def find(name: str, path: str, device: int, inode: int) -> NamedDescriptor: + matches = [] + for candidate in descriptors: + info = candidate.stat() + if (info.st_dev, info.st_ino) == (device, inode) and os.readlink(candidate) == path: + matches.append(int(candidate.name)) + if len(matches) != 1: + raise AssertionError("the actual issuer resource is absent or ambiguous: " + name) + return NamedDescriptor(name, matches[0]) + + owners = tuple(find(item.generation_id, item.owner.path, item.owner.device, item.owner.inode) for item in enrollment.originals if item.owner is not None) + controllers = tuple(find(item.unit, item.lock.path, item.lock.device, item.lock.inode) for item in enrollment.watchers) + namespaces = tuple(find(item.generation_id, item.namespace_pin.namespace, item.namespace_pin.device, item.namespace_pin.inode) for item in enrollment.originals if item.namespace_pin is not None) + return RecoveryResources(issuer_pid, start, Path("/proc/sys/kernel/random/boot_id").read_text().strip(), owners, controllers, namespaces) + + +def _containment_cli(directory: Path, label: str, enrollment: str, resources: Path, *, previous: Path | None = None, refusal: str | None = None) -> dict[str, object]: + command = [sys.executable, "-B", "-m", "auto_coder.management.recovery_containment", "verify" if previous is not None else "observe", "--enrollment", enrollment, "--resources", str(resources)] + if previous is not None: + command.extend(("--previous", str(previous))) + completed = subprocess.run(command, capture_output=True, text=True, timeout=300, env=dict(os.environ, PYTHONDONTWRITEBYTECODE="1")) + _write_private(directory / (label + ".log"), (completed.stdout + completed.stderr).encode()) + value = json.loads(completed.stdout.splitlines()[-1]) + if not isinstance(value, dict) or any(value.get(name) is not False for name in ("grant_issued", "history_asserted", "admission_created")): + raise AssertionError("the actual containment CLI returned an incomplete or authorizing result") + if refusal is None: + if completed.returncode != 0 or value.get("status") != "observed-current-recovery-containment": + raise NativeContainmentFailure("the genuine native containment positive control was unavailable: " + str(value.get("reason")), action="verify" if previous is not None else "observe", returncode=completed.returncode, causes=value.get("causes")) + elif completed.returncode != 2 or value.get("status") != "unavailable" or refusal not in str(value.get("reason")): + raise NativeContainmentFailure("the actual containment verifier did not reject the injected condition: " + label + " " + str(value.get("reason")), action="verify" if previous is not None else "observe", returncode=completed.returncode, causes=value.get("causes")) + return value + + +def verify_native_containment_reports(context: NativeNegativeContext, issuer_pid: int) -> list[str]: + """Reobserve saved/forged reports under genuine paused issuer custody.""" + require_native_context(context) + directory = context.fixture_root / "native-containment-report-probes" + directory.mkdir(mode=0o700) + lifetime = os.pidfd_open(issuer_pid, 0) + try: + epoch = _epoch_snapshot(context.epoch_root) + resources = _issuer_resources(context.enrollment, issuer_pid) + resource_path = directory / "actual-issuer-resources.json" + _write_private(resource_path, canonical(asdict(resources))) + enrollment = context.enrollment.enrollment_id + poller = select.poll() + poller.register(lifetime, select.POLLIN) + + def check_barrier() -> None: + if poller.poll(0) or _epoch_snapshot(context.epoch_root) != epoch or _issuer_resources(context.enrollment, issuer_pid) != resources: + raise AssertionError("the actual issuer advanced or lost custody during native report probes") + + def probe(label: str, *, previous: Path | None = None, refusal: str | None = None) -> dict[str, object]: + check_barrier() + try: + return _containment_cli(directory, label, enrollment, resource_path, previous=previous, refusal=refusal) + finally: + check_barrier() + + try: + report = probe("original-observer") + previous = directory / "original-observation.json" + _write_private(previous, canonical(report)) + probe("separate-current-verifier", previous=previous) + cases = ["native-current-containment-reverified"] + for label, field in (("forged-native-containment-refused", "target"), ("cross-deployment-containment-refused", "deployment")): + forged = json.loads(json.dumps(report)) + if field == "target": + forged["facts"]["target"]["pid"] += 1 + else: + forged["facts"]["enrollment"]["enrollment"]["deployment_id"] = "foreign-" + uuid.uuid4().hex + forged["evidence_sha256"] = digest(forged["facts"]) + path = directory / (label + ".json") + _write_private(path, canonical(forged)) + probe(label, previous=path, refusal="previous recovery scope/source/resources/current facts differ") + cases.append(label) + original = next(item for item in context.enrollment.originals if item.state == "started") + check_barrier() + with _retained_namespace_listener(original): + probe("stale-native-containment-refused", previous=previous, refusal="prior network namespace still has a task") + cases.append("stale-native-containment-refused") + restored = probe("restored-current-verifier", previous=previous) + restored_path = directory / "pre-nonleader-observation.json" + _write_private(restored_path, canonical(restored)) + issuer_observation = json.loads((context.fixture_root / "native-nonleader-issuer-observation.json").read_bytes()) + if not isinstance(issuer_observation, dict) or issuer_observation.get("case") != "escaped-old-network-nonleader-refused": + raise AssertionError("the actual full-issuer nonleader observation is absent") + measurements = [issuer_observation] + with _escaped_namespace_nonleader(original) as held: + before = held.observe_live() + refused = probe("escaped-nonleader-current-containment-refused", previous=restored_path, refusal="prior network namespace still has a task") + after = held.observe_live() + measurements.append({"case": "escaped-nonleader-current-containment-refused", "before": asdict(before), "after": asdict(after), "status": refused["status"]}) + cases.append("escaped-nonleader-current-containment-refused") + exited = held.exit_unreaped() + held.observe_exited() + verified = probe("measured-exited-nonleader-current-containment-reverified", previous=restored_path) + retained_exit = held.observe_exited() + measurements.append({"case": "measured-exited-nonleader-current-containment-reverified", "before_exit": asdict(after), "after_exit": asdict(exited), "after_verification": asdict(retained_exit), "status": verified["status"]}) + cases.append("measured-exited-nonleader-current-containment-reverified") + if set(cases) != REQUIRED_REPORT_CASES: + raise AssertionError("the actual report-verifier case matrix is incomplete") + _write_private(context.fixture_root / "native-task-observations.json", canonical({"schema": "actual-recovery-task-observations-v1", "cases": measurements})) + return cases + finally: + check_barrier() + finally: + os.close(lifetime) diff --git a/scripts/recovery_epoch_native_peer.py b/scripts/recovery_epoch_native_peer.py new file mode 100644 index 000000000..8dda9f0d9 --- /dev/null +++ b/scripts/recovery_epoch_native_peer.py @@ -0,0 +1,1135 @@ +"""Real peer and late-mutation faults in the separately authorized recovery VM. + +Only the original production issuer and bootstrap can produce authority. This +fixture observes real syscalls, retained process lifetimes and received kernel +objects. It may stop/kill its exact target or mutate an explicitly selected +fixture object; it never changes syscall arguments/results or production code. +""" + +from __future__ import annotations + +import array +import ctypes +import errno +import fcntl +import hashlib +import json +import os +import re +import select +import signal +import socket +import stat +import struct +import subprocess +import sys +import time +from contextlib import ExitStack, contextmanager +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Callable, Iterator + +from scripts.recovery_epoch_fault_tracer import _AUDIT_ARCH_X86_64, _PTRACE_GET_SYSCALL_INFO, KernelPublicationBarrier, TraceProcess, _Information, _token +from scripts.recovery_epoch_native_negatives import NativeNegativeContext, _epoch_snapshot, _projection, assert_unreserved_epoch, require_native_context +from scripts.test_management_containment_enrollment_host import _docker, _systemctl + +from auto_coder.logger_config import get_logger +from auto_coder.management.bootstrap_supervisor import BootstrapConfig +from auto_coder.management.containment_route_manifest import _systemd_unit +from auto_coder.management.original_attempt_evidence import OriginalLineageEvidence, derive_lineage, observe, verify_lineage +from auto_coder.management.original_attempts import canonical, read_original +from auto_coder.management.recovery_epoch_authorization import AUTHORIZATION_FILE, PEER_SOCKET, EpochAuthorization +from auto_coder.management.recovery_epoch_consumer import GRANT_SCHEMA, ConsumerConfiguration, parse_consumer_configuration +from auto_coder.management.recovery_epoch_peer import CHALLENGE_SCHEMA, MAX_MESSAGE_BYTES, MAX_RESOURCES, OFFER_SCHEMA, RECEIPT_SCHEMA, inspect_kernel_lock + +logger = get_logger(__name__) +REQUIRED_PEER_CASES = frozenset({"queued-rights-without-receipt-refused", "received-rights-without-receipt-refused", "changed-target-lifetime-refused", "foreign-acquirer-sh-refused"}) +LATE_MUTATION_SCENARIOS = ( + "late-management-config", + "late-ready-receipt", + "late-installed-source", + "late-original-journal", + "late-original-attempt", + "late-watcher-config", + "late-controller-restarter", + "late-original-fence", + "late-target-fence", + "late-controller-fence", + "late-target-loss", +) +POST_RECEIPT_MUTATION_SCENARIOS = ("post-receipt-ready-mutation",) +MUTATION_SCENARIOS = (*LATE_MUTATION_SCENARIOS, *POST_RECEIPT_MUTATION_SCENARIOS) +_SYS_SENDTO = 44 +_SYS_SENDMSG = 46 +_SYS_RECVMSG = 47 +_SYS_PIDFD_GETFD = 438 + + +def _wait(predicate: Callable[[], bool], description: str, *, timeout: float = 30) -> None: + deadline = time.monotonic() + timeout + while not predicate(): + if time.monotonic() >= deadline: + raise AssertionError("native peer barrier timed out: " + description) + time.sleep(0.02) + + +def _target_state(context: NativeNegativeContext) -> dict[str, object]: + value = json.loads(_docker("inspect", "--format", "{{json .State}}", context.enrollment.target.management_id)) + if not isinstance(value, dict): + raise AssertionError("actual target Docker state is unavailable") + return value + + +class DockerTraceProcess: + """Pin one actual Docker PID without inventing a parent/child relationship.""" + + def __init__(self, context: NativeNegativeContext): + self.context = context + state = _target_state(context) + if state.get("Running") is not True or type(state.get("Pid")) is not int or state["Pid"] <= 1: + raise AssertionError("the exact production target has no current lifetime") + self.pid = int(state["Pid"]) + self.start_token = _token(self.pid) + self.docker_started_at = state.get("StartedAt") + self.returncode: int | None = None + self.pidfd = os.pidfd_open(self.pid, 0) + self.paused = False + self.check() + + def dead(self) -> bool: + poll = select.poll() + poll.register(self.pidfd, select.POLLIN) + return bool(poll.poll(0)) + + def check(self) -> None: + if self.pidfd < 0 or self.dead() or _token(self.pid) != self.start_token: + raise AssertionError("the exact selected target lifetime ended or changed") + state = _target_state(self.context) + if state.get("Running") is not True or state.get("Pid") != self.pid or state.get("StartedAt") != self.docker_started_at: + raise AssertionError("the selected Docker object now names another lifetime") + + def wait(self, timeout: float | None = None) -> int: + """Observe Docker's actual termination after tracer wait ownership ends.""" + timeout = 30 if timeout is None else timeout + poll = select.poll() + poll.register(self.pidfd, select.POLLIN) + if not poll.poll(max(1, int(timeout * 1000))): + raise subprocess.TimeoutExpired("actual target pidfd", timeout) + if self.returncode is None: + _wait(lambda: _target_state(self.context).get("Running") is False, "Docker records the original target's exit", timeout=timeout) + state = _target_state(self.context) + if state.get("StartedAt") != self.docker_started_at or type(state.get("ExitCode")) is not int: + raise AssertionError("Docker no longer retains the original target exit code") + self.returncode = int(state["ExitCode"]) + return self.returncode + + def pause(self) -> None: + self.check() + signal.pidfd_send_signal(self.pidfd, signal.SIGSTOP) + _wait(lambda: Path(f"/proc/{self.pid}/stat").read_text().rsplit(")", 1)[1].split()[0] in {"T", "t"}, "exact target is kernel-stopped") + self.check() + self.paused = True + + def resume(self) -> None: + if self.paused and not self.dead(): + self.check() + signal.pidfd_send_signal(self.pidfd, signal.SIGCONT) + self.paused = False + + def kill(self) -> None: + self.check() + signal.pidfd_send_signal(self.pidfd, signal.SIGKILL) + + def close(self) -> None: + if self.pidfd >= 0: + self.resume() + os.close(self.pidfd) + self.pidfd = -1 + + +def _container_ids(encoded: str) -> tuple[str, ...]: + """Require complete real Docker identities before interpreting absence.""" + values = tuple(encoded.splitlines()) + if len(values) > 4096 or len(values) != len(set(values)) or any(re.fullmatch(r"[0-9a-f]{64}", value) is None for value in values): + raise AssertionError("the actual Docker inventory has incomplete or conflicting identities") + return tuple(sorted(values)) + + +def _verify_target_loss(pidfd: int, identifier: str, before: tuple[str, ...], after: tuple[str, ...]) -> None: + """Require both kernel death and removal of exactly the selected object.""" + if before.count(identifier) != 1 or after != tuple(value for value in before if value != identifier): + raise AssertionError("the exact target was not removed without changing other Docker objects") + poll = select.poll() + poll.register(pidfd, select.POLLIN) + if not poll.poll(0): + raise AssertionError("the original target pidfd has not independently reported death") + + +@contextmanager +def _remove_target(context: NativeNegativeContext) -> Iterator[dict[str, object]]: + """Remove only the pinned disposable target and retain that loss on exit.""" + target = DockerTraceProcess(context) + try: + identifier = context.enrollment.target.management_id + before = _container_ids(_docker("ps", "--all", "--quiet", "--no-trunc")) + if identifier not in before: + raise AssertionError("the exact target is missing before its deliberate removal") + target.check() + # _docker checks the actual command return code. Neither a failed + # removal nor Docker's stopped flag supplies kernel lifetime evidence. + removed = _docker("rm", "--force", identifier, timeout=60) + if removed != identifier: + raise AssertionError("the actual Docker removal did not identify the selected target") + _wait(target.dead, "the original target pidfd reports actual death") + after = _container_ids(_docker("ps", "--all", "--quiet", "--no-trunc")) + _verify_target_loss(target.pidfd, identifier, before, after) + observation = { + "target_id": identifier, + "pid": target.pid, + "start_token": target.start_token, + "boot_id": Path("/proc/sys/kernel/random/boot_id").read_text().strip(), + "docker_started_at": target.docker_started_at, + "docker_rm_returncode": 0, + "pidfd_death_observed": True, + "exact_docker_id_absent": True, + "other_docker_ids_preserved": True, + "container_ids_before_sha256": hashlib.sha256(canonical(before)).hexdigest(), + "container_ids_after_sha256": hashlib.sha256(canonical(after)).hexdigest(), + } + yield observation + # The driver must retain the injected loss through the original + # refusal and fixed-root replay; it must never recreate this target. + _verify_target_loss(target.pidfd, identifier, before, _container_ids(_docker("ps", "--all", "--quiet", "--no-trunc"))) + finally: + target.close() + + +def _consumer_configuration(context: NativeNegativeContext) -> ConsumerConfiguration: + selected = [item for item in context.enrollment.target.mounts if item.destination == "/bootstrap/recovery-epoch.json"] + if len(selected) != 1 or selected[0].writable: + raise AssertionError("the actual consumer lacks its fixed read-only configuration") + result = parse_consumer_configuration(Path(selected[0].source).read_bytes()) + if (result.authorization_id, result.target_id) != (context.enrollment.authorization_id, context.enrollment.target.management_id): + raise AssertionError("the selected native consumer belongs to another fixture") + return result + + +def _fd_for_object(pid: int, device: int, inode: int, mode: str) -> int: + matches = [] + for path in Path(f"/proc/{pid}/fd").iterdir(): + if not path.name.isdecimal(): + continue + try: + info = path.stat() + if (info.st_dev, info.st_ino) == (device, inode): + lock = inspect_kernel_lock(pid, int(path.name), device, inode) + if lock is not None and lock.mode == mode: + matches.append(int(path.name)) + except FileNotFoundError: + continue + if len(matches) != 1: + raise AssertionError("the actual selected process does not hold one exact fence descriptor") + return matches[0] + + +def _duplicate(pid: int, descriptor: int, device: int, inode: int) -> int: + """Duplicate the real OFD; unsupported pidfd_getfd is a fixture failure.""" + lifetime = _token(pid) + pidfd = os.pidfd_open(pid, 0) + duplicate = -1 + try: + libc = ctypes.CDLL(None, use_errno=True) + libc.syscall.restype = ctypes.c_long + duplicate = int(libc.syscall(ctypes.c_long(_SYS_PIDFD_GETFD), ctypes.c_int(pidfd), ctypes.c_int(descriptor), ctypes.c_uint(0))) + if duplicate < 0: + error = ctypes.get_errno() + raise OSError(error, os.strerror(error)) + info = os.fstat(duplicate) + if (info.st_dev, info.st_ino) != (device, inode) or _token(pid) != lifetime: + raise AssertionError("the duplicated native descriptor or process changed") + os.set_inheritable(duplicate, False) + return duplicate + except BaseException: + if duplicate >= 0: + os.close(duplicate) + raise + finally: + os.close(pidfd) + + +@contextmanager +def _foreign_acquirer(pid: int, device: int, inode: int, mode: str) -> Iterator[dict[str, object]]: + descriptor = _fd_for_object(pid, device, inode, mode) + before = inspect_kernel_lock(pid, descriptor, device, inode) + if before is None or before.pid != pid: + raise AssertionError("the selected fence was not independently held before the real fault") + duplicate = _duplicate(pid, descriptor, device, inode) + try: + fcntl.flock(duplicate, fcntl.LOCK_UN) + fcntl.flock(duplicate, (fcntl.LOCK_SH if mode == "READ" else fcntl.LOCK_EX) | fcntl.LOCK_NB) + after = inspect_kernel_lock(pid, descriptor, device, inode) + if after is None or after.pid != os.getpid() or after.mode != mode or after == before: + raise AssertionError("the real shared OFD did not record the foreign acquiring process") + yield {"pid": pid, "start_token": _token(pid), "descriptor": descriptor, "device": device, "inode": inode, "before": asdict(before), "after": asdict(after)} + if inspect_kernel_lock(os.getpid(), duplicate, device, inode) != after: + raise AssertionError("the injected native OFD custody changed during refusal") + finally: + os.close(duplicate) + + +def _require_publication_prefix(context: NativeNegativeContext, issuer_pid: int, through: str) -> None: + require_native_context(context, unreserved=False) + phases = ("reserved", "intent", "consumed", "receipt") + if through not in phases: + raise ValueError("an exact existing publication prefix is required") + chosen = phases[: phases.index(through) + 1] + expected = {"inventory.json", "epoch.lock", *(name + suffix for name in chosen for suffix in (".json", "-head.json"))} + if {path.name for path in context.epoch_root.iterdir()} != expected: + raise AssertionError("the actual issuer is not stopped at the selected original publication prefix") + inventory_sha = hashlib.sha256((context.epoch_root / "inventory.json").read_bytes()).hexdigest() + previous, binding = "", None + for sequence, phase in enumerate(chosen): + encoded = (context.epoch_root / (phase + ".json")).read_bytes() + record = json.loads(encoded) + head_bytes = (context.epoch_root / (phase + "-head.json")).read_bytes() + head = json.loads(head_bytes) + if canonical(record) != encoded or canonical(head) != head_bytes or record.get("phase") != phase or record.get("sequence") != sequence or record.get("previous") != previous or record.get("inventory_sha256") != inventory_sha: + raise AssertionError("the actual durable publication chain is malformed") + if head != {"schema_version": 1, "phase": phase, "record_sha256": hashlib.sha256(encoded).hexdigest(), "previous": previous, "inventory_sha256": inventory_sha}: + raise AssertionError("the actual publication head differs from its original record") + binding = record["binding"] if binding is None else binding + if record.get("binding") != binding: + raise AssertionError("the stopped publication prefix changed authority") + previous = hashlib.sha256(head_bytes).hexdigest() + if binding is None or (binding["host_pid"], binding["host_start_token"], binding["host_boot_id"]) != (issuer_pid, _token(issuer_pid), Path("/proc/sys/kernel/random/boot_id").read_text().strip()): + raise AssertionError("the durable original prefix belongs to another issuer lifetime") + + +def _failed_terminal(lineage: OriginalLineageEvidence) -> tuple[Path, dict[str, object]]: + """Select a failed original only from the complete, independently read set.""" + branches = [generation for generation in lineage.generations if generation.outcome == "never-created"] + if lineage.unavailable or len(branches) != 1: + raise AssertionError("the actual fixture has no unique complete never-created failed branch") + branch = branches[0] + matches = [] + for original in lineage.originals: + path = Path(original.path) + if path.parent != Path(lineage.binding.root) or path.suffix != ".json": + continue + encoded, file_id = read_original(path) + if file_id != (original.device, original.inode) or hashlib.sha256(encoded).hexdigest() != original.sha256: + raise AssertionError("the selected failed-branch original changed after independent derivation") + value = json.loads(encoded) + if isinstance(value, dict) and value.get("kind") == "never-created" and value.get("attempt") == branch.request_id: + matches.append((path, original)) + if len(matches) != 1: + raise AssertionError("the actual failed branch has no unique matched original terminal record") + path, original = matches[0] + return path, {"instance_id": branch.instance_id, "request_id": branch.request_id, "outcome": branch.outcome, "frontier": lineage.frontier, "terminal_kind": "never-created", "original_file": asdict(original)} + + +def _failed_branch_original(context: NativeNegativeContext) -> tuple[Path, dict[str, object]]: + """Reverify real L evidence and bind the fault to the issuer's saved input.""" + authorization = EpochAuthorization.parse(json.loads(read_original(AUTHORIZATION_FILE)[0])) + if authorization.target_id != context.enrollment.target.management_id: + raise AssertionError("the current authorization selects another target") + with ExitStack() as held: + descriptors = [] + for original in context.enrollment.originals: + owner = original.owner + if owner is None: + continue + _, identity = read_original(Path(owner.path)) + descriptor = os.open(owner.path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC) + held.callback(os.close, descriptor) + if identity != (owner.device, owner.inode) or (os.fstat(descriptor).st_dev, os.fstat(descriptor).st_ino) != identity: + raise AssertionError("the actual original owner changed before independent lineage verification") + fcntl.flock(descriptor, fcntl.LOCK_SH | fcntl.LOCK_NB) + descriptors.append(descriptor) + history = held.enter_context(observe(authorization.history)) + current = derive_lineage(authorization.history, history, tuple(descriptors)) + verify_lineage(current, authorization.history, history, tuple(descriptors)) + intent = json.loads(read_original(context.epoch_root / "intent.json")[0]) + saved = json.loads(intent["binding"]["evidence_json"])["lineage"] + if canonical(asdict(current)) != canonical(saved): + raise AssertionError("the independently verified failed-branch scope differs from the actual retained intent") + path, observation = _failed_terminal(current) + observation.update(independent_lineage_verified=True, retained_intent_lineage_matched=True) + return path, observation + + +def _mutation_file(context: NativeNegativeContext, scenario: str) -> Path: + target = context.enrollment.target + if scenario == "late-management-config": + return Path(target.management_config.path) + if scenario == "late-watcher-config": + return Path(context.enrollment.watchers[0].configuration.path) + if scenario == "late-original-journal": + original = context.enrollment.originals[0] + config = BootstrapConfig.read(Path(original.bootstrap_config.path)) + mounts = [item for item in original.mounts if item.container_id == original.management_id and item.destination == str(config.journal_root) and item.writable] + if len(mounts) != 1: + raise AssertionError("the original native journal mapping is ambiguous") + return Path(mounts[0].source) / "management-operations.sqlite3" + authorization = EpochAuthorization.parse(json.loads(AUTHORIZATION_FILE.read_bytes())) + if authorization.target_id != target.management_id: + raise AssertionError("the actual fixed authorization selects a different target") + if scenario in {"late-ready-receipt", "post-receipt-ready-mutation"}: + return Path(authorization.ready_receipt.path) + if scenario == "late-installed-source": + receipt = json.loads(Path(authorization.ready_receipt.path).read_bytes()) + return Path(receipt["environment_path"]) / "lib/python3.12/site-packages/auto_coder/__init__.py" + raise ValueError("the selected late fault is not a file mutation") + + +@contextmanager +def mutate_after_publication(context: NativeNegativeContext, scenario: str, issuer_pid: int) -> Iterator[dict[str, object]]: + """Retain the deliberate fault through refusal and replay; never repair it.""" + if scenario not in MUTATION_SCENARIOS: + raise ValueError("a reviewed late mutation scenario is required") + phase = "receipt" if scenario in POST_RECEIPT_MUTATION_SCENARIOS else "intent" + _require_publication_prefix(context, issuer_pid, phase) + observation: dict[str, object] = {"scenario": scenario, "issuer_pid": issuer_pid, "issuer_start_token": _token(issuer_pid), "boundary": "after-" + phase + "-head", "preservation_baseline": "after-deliberate-fixture-mutation"} + with ExitStack() as held: + if scenario == "late-target-loss": + observation["target_fault"] = held.enter_context(_remove_target(context)) + elif scenario.endswith("-fence"): + pid = issuer_pid + if scenario == "late-controller-fence": + lock, mode = context.enrollment.watchers[0].lock, "WRITE" + else: + target = DockerTraceProcess(context) + held.callback(target.close) + pid = target.pid + if scenario == "late-original-fence": + lock, mode = next(item.owner for item in context.enrollment.originals if item.owner is not None), "READ" + else: + lock, mode = context.enrollment.target.owner, "WRITE" + observation["kernel_fault"] = held.enter_context(_foreign_acquirer(pid, lock.device, lock.inode, mode)) + elif scenario == "late-controller-restarter": + unit = context.enrollment.watchers[0].unit + before = _systemd_unit(unit) + if not before.inhibited: + raise AssertionError("the actual controller was not inhibited at the original boundary") + _systemctl("unmask", unit) + _systemctl("daemon-reload") + after = _systemd_unit(unit) + if after.inhibited or after.main_pid != 0: + raise AssertionError("the real controller did not become startable without starting a process") + observation["controller_fault"] = {"unit": unit, "before": asdict(before), "after": asdict(after)} + else: + if scenario == "late-original-attempt": + path, observation["original_attempt"] = _failed_branch_original(context) + else: + path = _mutation_file(context, scenario) + if not path.is_relative_to(context.fixture_root) or path != path.resolve(strict=True): + raise PermissionError("late mutation escaped the separately authorized fixture root") + before = _projection(path) + descriptor = os.open(path, os.O_WRONLY | os.O_APPEND | os.O_NOFOLLOW | os.O_CLOEXEC) + try: + if (os.fstat(descriptor).st_dev, os.fstat(descriptor).st_ino) != before[:2]: + raise AssertionError("the selected original changed before deliberate mutation") + os.write(descriptor, b"\n# deliberate native fixture source mutation\n" if scenario == "late-installed-source" else b"\x00") + os.fsync(descriptor) + finally: + os.close(descriptor) + after = _projection(path) + if before == after or before[:2] != after[:2]: + raise AssertionError("the selected native original bytes did not change in place") + observation["file_fault"] = {"path": str(path), "before": before, "after": after} + logger.bind(native_peer_fault=scenario, issuer_pid=issuer_pid).info("Injected the explicit native fault after the original {} head", phase) + yield observation + if "file_fault" in observation and _projection(path) != after: + raise AssertionError("production refusal or replay changed the deliberately injected file") + if "controller_fault" in observation and _systemd_unit(unit) != after: + raise AssertionError("production refusal or replay changed the deliberately unmasked controller") + + +def _socket_option(descriptor: int, option: int, size: int) -> bytes: + """Read kernel socket metadata without constructing or reconfiguring a socket.""" + libc = ctypes.CDLL(None, use_errno=True) + libc.getsockopt.argtypes = (ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_void_p, ctypes.POINTER(ctypes.c_uint)) + libc.getsockopt.restype = ctypes.c_int + value = ctypes.create_string_buffer(size) + length = ctypes.c_uint(size) + result = libc.getsockopt(descriptor, socket.SOL_SOCKET, option, value, ctypes.byref(length)) + if result < 0: + error = ctypes.get_errno() + raise OSError(error, os.strerror(error)) + if length.value != size: + raise AssertionError("the actual socket metadata has an incomplete kernel representation") + return value.raw + + +def _local_socket_peer(descriptor: int) -> dict[str, int] | None: + """Return the real AF_UNIX stream peer's host-visible credentials.""" + domain = struct.unpack("i", _socket_option(descriptor, socket.SO_DOMAIN, struct.calcsize("i")))[0] + kind = struct.unpack("i", _socket_option(descriptor, socket.SO_TYPE, struct.calcsize("i")))[0] + if domain != socket.AF_UNIX or kind != socket.SOCK_STREAM: + return None + pid, uid, gid = struct.unpack("iII", _socket_option(descriptor, socket.SO_PEERCRED, struct.calcsize("iII"))) + return {"pid": pid, "uid": uid, "gid": gid} + + +def _peek_queued_packet(descriptor: int, expected: bytes) -> tuple[bytes, int, int]: + """Observe the actual queued frame twice, without consuming or changing flags.""" + if not expected.endswith(b"\n") or not 0 < len(expected) <= MAX_MESSAGE_BYTES: + raise AssertionError("a bounded complete expected grant frame is required") + libc = ctypes.CDLL(None, use_errno=True) + libc.recv.argtypes = (ctypes.c_int, ctypes.c_void_p, ctypes.c_size_t, ctypes.c_int) + libc.recv.restype = ctypes.c_ssize_t + before = fcntl.fcntl(descriptor, fcntl.F_GETFL) + + def peek() -> bytes: + buffer = ctypes.create_string_buffer(MAX_MESSAGE_BYTES + 1) + result = libc.recv(descriptor, buffer, len(buffer), socket.MSG_PEEK | socket.MSG_DONTWAIT) + if result < 0: + error = ctypes.get_errno() + raise AssertionError("the actual receiver has no observable complete queued grant: " + os.strerror(error)) + if result <= 0 or result > MAX_MESSAGE_BYTES: + raise AssertionError("the actual receiver has an empty or oversized queued grant") + return buffer.raw[:result] + + encoded = peek() + if encoded != expected: + raise AssertionError("the actual queued bytes differ from the exact consumed grant") + if peek() != encoded: + raise AssertionError("the actual grant bytes did not remain queued after MSG_PEEK") + after = fcntl.fcntl(descriptor, fcntl.F_GETFL) + if after != before: + raise AssertionError("observing the actual grant changed shared socket status flags") + return encoded, before, after + + +def _peek_pending_frame(descriptor: int) -> bytes: + """Observe even an empty real stream queue without changing its shared OFD.""" + libc = ctypes.CDLL(None, use_errno=True) + libc.recv.argtypes = (ctypes.c_int, ctypes.c_void_p, ctypes.c_size_t, ctypes.c_int) + libc.recv.restype = ctypes.c_ssize_t + before = fcntl.fcntl(descriptor, fcntl.F_GETFL) + buffer = ctypes.create_string_buffer(MAX_MESSAGE_BYTES + 1) + result = libc.recv(descriptor, buffer, len(buffer), socket.MSG_PEEK | socket.MSG_DONTWAIT) + error = ctypes.get_errno() + if fcntl.fcntl(descriptor, fcntl.F_GETFL) != before: + raise AssertionError("pending grant observation changed the receiver's shared status flags") + if result < 0: + if error in {errno.EAGAIN, errno.EWOULDBLOCK}: + return b"" + raise OSError(error, os.strerror(error)) + if result > MAX_MESSAGE_BYTES: + raise AssertionError("the actual pending receiver frame exceeds the protocol bound") + return buffer.raw[:result] + + +def _issuer_socket(target: DockerTraceProcess, issuer_pid: int, issuer_uid: int, held: ExitStack) -> tuple[int, int, int, int, dict[str, int]]: + before = _descriptor_snapshot(target.pid) + if len(before) > 4096: + raise AssertionError("the actual receiver descriptor inventory exceeds the fixture bound") + matches = [] + for descriptor, device, inode, _ in before: + if not stat.S_ISSOCK(Path(f"/proc/{target.pid}/fd/{descriptor}").stat().st_mode): + continue + duplicate = _duplicate(target.pid, descriptor, device, inode) + held.callback(os.close, duplicate) + credentials = _local_socket_peer(duplicate) + if credentials is not None and credentials["pid"] == issuer_pid: + if credentials["uid"] != issuer_uid: + raise AssertionError("the actual queued-grant sender has the wrong Unix owner") + matches.append((descriptor, device, inode, duplicate, credentials)) + if len(matches) != 1: + raise AssertionError("the stopped receiver does not retain one exact authenticated issuer socket") + return matches[0] + + +def _expected_queued_grant(context: NativeNegativeContext, issuer_pid: int, issuer_start_token: str, target_pid: int, target_start_token: str) -> dict[str, object]: + """Bind the expected packet to fixed consumer inputs and original durable custody.""" + configuration = _consumer_configuration(context) + inventory_bytes = read_original(context.epoch_root / "inventory.json")[0] + consumed_bytes = read_original(context.epoch_root / "consumed.json")[0] + receipt_bytes = read_original(context.epoch_root / "receipt.json")[0] + receipt = json.loads(receipt_bytes) + binding = receipt["binding"] + evidence_bytes = binding["evidence_json"].encode() + evidence = json.loads(evidence_bytes) + if canonical(evidence) != evidence_bytes or hashlib.sha256(evidence_bytes).hexdigest() != binding["evidence_sha256"]: + raise AssertionError("the actual retained custody evidence is not canonical and digest-bound") + boot = Path("/proc/sys/kernel/random/boot_id").read_text().strip() + if (binding["host_pid"], binding["host_start_token"], binding["host_boot_id"]) != (issuer_pid, issuer_start_token, boot): + raise AssertionError("the original consumed evidence names another issuer lifetime") + custody = evidence["custody"] + peer = custody["peer"] + if (peer["pid"], peer["start_token"], peer["boot_id"], peer["uid"]) != (target_pid, target_start_token, boot, 0): + raise AssertionError("the original consumed custody names another target lifetime") + selected = {name: getattr(configuration, name) for name in ("request_id", "authorization_id", "target_id")} + if any(binding[name] != value or peer[name] != value for name, value in selected.items()): + raise AssertionError("the original consumed custody differs from the actual fixed consumer binding") + for name in ("challenge", "receipt_nonce"): + if not isinstance(custody[name], str) or re.fullmatch(r"[0-9a-f]{64}", custody[name]) is None: + raise AssertionError("the actual original custody challenge or receipt nonce is incomplete") + return { + "schema": GRANT_SCHEMA, + "granted": "recovery-epoch-consumed", + **selected, + "challenge": custody["challenge"], + "receipt_nonce": custody["receipt_nonce"], + "inventory_sha256": hashlib.sha256(inventory_bytes).hexdigest(), + "binding_sha256": hashlib.sha256(canonical(binding)).hexdigest(), + "consumed_sha256": hashlib.sha256(consumed_bytes).hexdigest(), + "receipt_sha256": hashlib.sha256(receipt_bytes).hexdigest(), + "host_boot_id": boot, + "host_pid": issuer_pid, + "host_start_token": issuer_start_token, + } + + +def observe_queued_grant(context: NativeNegativeContext, issuer_pid: int, *, target_pid: int, target_start_token: str, issuer_start_token: str) -> dict[str, object]: + """Prove a full grant reached the stopped receiver before the issuer dies.""" + _require_publication_prefix(context, issuer_pid, "receipt") + with ExitStack() as held: + target = DockerTraceProcess(context) + held.callback(target.close) + issuer_lifetime = os.pidfd_open(issuer_pid, 0) + held.callback(os.close, issuer_lifetime) + issuer_dead = select.poll() + issuer_dead.register(issuer_lifetime, select.POLLIN) + + def check_lifetimes() -> None: + target.check() + if (target.pid, target.start_token) != (target_pid, target_start_token) or issuer_dead.poll(0) or _token(issuer_pid) != issuer_start_token: + raise AssertionError("a queued-grant observer input differs from the original pinned lifetimes") + + check_lifetimes() + before = _descriptor_snapshot(target.pid) + _stopped_receiver(target.pid, target.start_token, target.pidfd, before) + expected = _expected_queued_grant(context, issuer_pid, issuer_start_token, target.pid, target.start_token) + configuration = _consumer_configuration(context) + descriptor, device, inode, duplicate, credentials = _issuer_socket(target, issuer_pid, configuration.issuer_uid, held) + encoded, flags_before, flags_after = _peek_queued_packet(duplicate, canonical(expected) + b"\n") + check_lifetimes() + _stopped_receiver(target.pid, target.start_token, target.pidfd, before) + _assert_no_target_admission(context, running_pid=target.pid) + _require_publication_prefix(context, issuer_pid, "receipt") + if _expected_queued_grant(context, issuer_pid, issuer_start_token, target.pid, target.start_token) != expected or _local_socket_peer(duplicate) != credentials: + raise AssertionError("the original grant binding or socket sender changed during kernel observation") + logger.bind(issuer_pid=issuer_pid, target_pid=target.pid, grant_schema=GRANT_SCHEMA, bytes_observed=len(encoded)).info("Observed the complete consumed grant still queued for the stopped actual receiver") + return { + "schema": "actual-recovery-queued-grant-v1", + "target_id": configuration.target_id, + "target_pid": target.pid, + "target_start_token": target.start_token, + "issuer_pid": issuer_pid, + "issuer_start_token": issuer_start_token, + "socket_descriptor": descriptor, + "socket_device": device, + "socket_inode": inode, + "peer_credentials": credentials, + "grant_schema": GRANT_SCHEMA, + "message_sha256": hashlib.sha256(encoded).hexdigest(), + "bytes_observed": len(encoded), + "grant": expected, + "socket_flags_before": flags_before, + "socket_flags_after": flags_after, + "peek_only": True, + "socket_flags_unchanged": True, + "bytes_still_queued": True, + "receiver_stopped": True, + "issuer_alive": True, + } + + +@contextmanager +def observe_post_receipt_dispatch(context: NativeNegativeContext, issuer: subprocess.Popen, release: Callable[[], None]) -> Iterator[dict[str, object]]: + """Measure refusal or a real queued grant while the same receiver is stopped.""" + _require_publication_prefix(context, issuer.pid, "receipt") + with ExitStack() as held: + target = DockerTraceProcess(context) + held.callback(target.close) + issuer_start = _token(issuer.pid) + issuer_pidfd = os.pidfd_open(issuer.pid, 0) + held.callback(os.close, issuer_pidfd) + target.pause() + before = _descriptor_snapshot(target.pid) + _stopped_receiver(target.pid, target.start_token, target.pidfd, before) + configuration = _consumer_configuration(context) + descriptor, device, inode, duplicate, credentials = _issuer_socket(target, issuer.pid, configuration.issuer_uid, held) + flags = fcntl.fcntl(duplicate, fcntl.F_GETFL) + if _peek_pending_frame(duplicate): + raise AssertionError("the receiver already had a grant before the post-receipt issuer resumed") + observation: dict[str, object] = { + "schema": "actual-recovery-post-receipt-dispatch-v1", + "issuer_pid": issuer.pid, + "issuer_start_token": issuer_start, + "target_pid": target.pid, + "target_start_token": target.start_token, + "socket_descriptor": descriptor, + "socket_device": device, + "socket_inode": inode, + "peer_credentials": credentials, + "receiver_stopped": True, + "queue_empty_before_resume": True, + "queue_bytes_observed": 0, + "queued_grant": None, + "issuer_returncode": None, + "issuer_death_before_receiver_resumed": False, + } + try: + release() + poll = select.poll() + poll.register(issuer_pidfd, select.POLLIN) + poll.register(duplicate, select.POLLIN | select.POLLHUP) + deadline = time.monotonic() + 1800 + while True: + if time.monotonic() >= deadline: + raise AssertionError("the original issuer neither refused nor queued a complete post-receipt grant") + poll.poll(1000) + target.check() + _stopped_receiver(target.pid, target.start_token, target.pidfd, before) + queued = _peek_pending_frame(duplicate) + returncode = issuer.poll() + if queued: + observation["queued_grant"] = observe_queued_grant(context, issuer.pid, target_pid=target.pid, target_start_token=target.start_token, issuer_start_token=issuer_start) + observation["queue_bytes_observed"] = len(queued) + break + if returncode is not None: + if _peek_pending_frame(duplicate): + raise AssertionError("the original issuer left a grant before its observed exit") + observation["issuer_returncode"] = returncode + break + if fcntl.fcntl(duplicate, fcntl.F_GETFL) != flags or _local_socket_peer(duplicate) != credentials: + raise AssertionError("the exact receiver socket changed during the post-receipt observation") + observation["socket_flags_unchanged"] = True + _assert_no_target_admission(context, running_pid=target.pid) + yield observation + finally: + # A detected mutant's live issuer is terminated only after the + # actual unsafe dispatch evidence has been captured. No retry or + # original epoch repair is performed. + if issuer.poll() is None: + signal.pidfd_send_signal(issuer_pidfd, signal.SIGKILL) + issuer.wait(timeout=30) + dead = select.poll() + dead.register(issuer_pidfd, select.POLLIN) + if not dead.poll(0): + raise AssertionError("the original issuer remains alive before receiver resumption") + observation["issuer_death_before_receiver_resumed"] = True + + +@contextmanager +def pause_target_before_grant(context: NativeNegativeContext, issuer_pid: int) -> Iterator[dict[str, object]]: + """Hold the actual receiver while the original issuer sends, then dies.""" + _require_publication_prefix(context, issuer_pid, "receipt") + target = DockerTraceProcess(context) + issuer_lifetime = os.pidfd_open(issuer_pid, 0) + try: + target.pause() + observation = {"target_pid": target.pid, "target_start_token": target.start_token, "issuer_pid": issuer_pid, "issuer_start_token": _token(issuer_pid), "boundary": "after-receipt-head-before-target-grant-observation", "issuer_death_verified_before_receiver_resumed": False} + _assert_no_target_admission(context, running_pid=target.pid) + yield observation + target.check() + issuer_dead = select.poll() + issuer_dead.register(issuer_lifetime, select.POLLIN) + if not issuer_dead.poll(0): + raise AssertionError("the exact original issuer still lives before the receiver resumes") + observation["issuer_death_verified_before_receiver_resumed"] = True + _assert_no_target_admission(context, running_pid=target.pid) + finally: + target.close() + os.close(issuer_lifetime) + + +class _Iovec(ctypes.Structure): + _fields_ = [("base", ctypes.c_void_p), ("length", ctypes.c_size_t)] + + +class _MessageHeader(ctypes.Structure): + _fields_ = [("name", ctypes.c_void_p), ("name_length", ctypes.c_uint32), ("vectors", ctypes.c_void_p), ("vector_count", ctypes.c_size_t), ("control", ctypes.c_void_p), ("control_length", ctypes.c_size_t), ("flags", ctypes.c_int)] + + +class _ControlHeader(ctypes.Structure): + _fields_ = [("length", ctypes.c_size_t), ("level", ctypes.c_int), ("kind", ctypes.c_int)] + + +@dataclass(frozen=True) +class PeerMessageObservation: + process_id: int + start_token: str + boot_id: str + schema: str + direction: str + when: str + socket_descriptor: int + socket_inode: int + message_sha256: str + bytes_observed: int + syscall_result: int | None + transferred_descriptors: tuple[int, ...] + descriptor_objects: tuple[tuple[int, int, int, str], ...] + + +def _decode_rights(control: bytes) -> tuple[int, ...]: + """Decode the bounded native Linux cmsghdr layout, never claimed JSON FDs.""" + result: list[int] = [] + offset = 0 + header_size = ctypes.sizeof(_ControlHeader) + alignment = ctypes.sizeof(ctypes.c_size_t) + while offset < len(control): + if len(control) - offset < header_size: + raise AssertionError("native SCM_RIGHTS control header is truncated") + header = _ControlHeader.from_buffer_copy(control[offset : offset + header_size]) + if header.length < header_size or offset + header.length > len(control) or (header.level, header.kind) != (socket.SOL_SOCKET, socket.SCM_RIGHTS): + raise AssertionError("native message control is not exact SCM_RIGHTS") + payload = control[offset + header_size : offset + header.length] + if len(payload) % array.array("i").itemsize: + raise AssertionError("native SCM_RIGHTS descriptor bytes are truncated") + descriptors = array.array("i") + descriptors.frombytes(payload) + result.extend(descriptors) + if len(result) > MAX_RESOURCES or any(item < 0 for item in result) or len(set(result)) != len(result): + raise AssertionError("native SCM_RIGHTS descriptor set is invalid") + end = offset + int(header.length) + aligned = (end + alignment - 1) // alignment * alignment + # Linux may include or omit the final cmsghdr alignment padding. + offset = min(aligned, len(control)) if end <= len(control) < aligned else aligned + return tuple(result) + + +class KernelPeerMessageBarrier(KernelPublicationBarrier): + """Stop one real protocol syscall with no ptrace register or memory writes.""" + + def __init__( + self, + process: TraceProcess, + *, + context: NativeNegativeContext, + schema: str, + direction: str, + when: str = "after", + before_send: Callable[[], None] | None = None, + ): + if schema not in {CHALLENGE_SCHEMA, OFFER_SCHEMA, RECEIPT_SCHEMA, GRANT_SCHEMA} or direction not in {"send", "receive"} or when not in {"before", "after"} or when == "before" and direction != "send": + raise ValueError("an exact reviewed native message boundary is required") + super().__init__(process, root=context.epoch_root, name="reserved.json", when="before") + self.context = context + self.schema = schema + self.direction = direction + self.message_when = when + self.before_send = before_send + configuration = _consumer_configuration(context) + self.binding = {name: getattr(configuration, name) for name in ("request_id", "authorization_id", "target_id")} + self._message_call: tuple[int, int, int, int] | None = None + self._selected_message: tuple[bytes, tuple[int, ...], int, int | None] | None = None + self._before_called = False + + def _memory(self, address: int, length: int) -> bytes: + if not address or type(length) is not int or not 0 <= length <= MAX_MESSAGE_BYTES + 1024: + raise AssertionError("native protocol memory read is absent or unbounded") + with Path(f"/proc/{self.issuer.pid}/mem").open("rb", buffering=0) as memory: + value = os.pread(memory.fileno(), length, address) + if len(value) != length: + raise AssertionError("native protocol memory is incomplete") + return value + + def _message(self, address: int, *, received_length: int | None) -> tuple[bytes, tuple[int, ...]]: + header = _MessageHeader.from_buffer_copy(self._memory(address, ctypes.sizeof(_MessageHeader))) + if not 1 <= header.vector_count <= 4 or header.control_length > socket.CMSG_SPACE(MAX_RESOURCES * array.array("i").itemsize) or header.flags & (socket.MSG_TRUNC | socket.MSG_CTRUNC): + raise AssertionError("native protocol message vectors or ancillary data exceed their bound") + vectors = self._memory(int(header.vectors or 0), ctypes.sizeof(_Iovec) * header.vector_count) + data = bytearray() + remaining = received_length + for index in range(header.vector_count): + item = _Iovec.from_buffer_copy(vectors[index * ctypes.sizeof(_Iovec) : (index + 1) * ctypes.sizeof(_Iovec)]) + size = int(item.length) if remaining is None else min(int(item.length), remaining) + if size < 0 or size > MAX_MESSAGE_BYTES + 1 or len(data) + size > MAX_MESSAGE_BYTES + 1: + raise AssertionError("native protocol message bytes exceed their bound") + if size: + data.extend(self._memory(int(item.base or 0), size)) + if remaining is not None: + remaining -= size + if remaining == 0: + break + if remaining not in (None, 0): + raise AssertionError("actual recvmsg result exceeds its real iovec capacity") + control = self._memory(int(header.control or 0), int(header.control_length)) if header.control_length else b"" + return bytes(data), _decode_rights(control) + + def _selected(self, encoded: bytes) -> bool: + if not encoded.endswith(b"\n") or encoded.count(b"\n") != 1 or len(encoded) > MAX_MESSAGE_BYTES: + return False + try: + value = json.loads(encoded) + except (UnicodeError, ValueError): + return False + return isinstance(value, dict) and value.get("schema") == self.schema and all(value.get(name) == selected for name, selected in self.binding.items()) + + def _observe_call(self) -> bool: + info = _Information() + length = self._ptrace(_PTRACE_GET_SYSCALL_INFO, ctypes.sizeof(info), ctypes.cast(ctypes.byref(info), ctypes.c_void_p)) + if length < 24 or info.architecture != _AUDIT_ARCH_X86_64: + raise AssertionError("actual native protocol syscall information is unavailable") + if info.operation == 1: + self._message_call = None + call = info.call.entry + number, descriptor, address = int(call.number), int(call.arguments[0]), int(call.arguments[1]) + if self.direction == "receive" and number == _SYS_RECVMSG: + self._message_call = (number, descriptor, address, 0) + elif self.direction == "send" and number in {_SYS_SENDMSG, _SYS_SENDTO}: + if number == _SYS_SENDMSG: + encoded, descriptors = self._message(address, received_length=None) + else: + size = int(call.arguments[2]) + if not 0 < size <= MAX_MESSAGE_BYTES: + return False + encoded, descriptors = self._memory(address, size), () + if self._selected(encoded): + if self.before_send is not None: + if self._before_called: + raise AssertionError("the actual selected protocol send was attempted twice") + self.before_send() + self._before_called = True + self._message_call = (number, descriptor, address, len(encoded)) + self._selected_message = (encoded, descriptors, descriptor, None) + return self.message_when == "before" + elif info.operation == 2 and self._message_call is not None: + number, descriptor, address, expected_size = self._message_call + self._message_call = None + result = int(info.call.exit.result) + if info.call.exit.is_error or result <= 0: + return False + if self.direction == "receive": + encoded, descriptors = self._message(address, received_length=result) + if not self._selected(encoded): + return False + self._selected_message = (encoded, descriptors, descriptor, result) + return True + if self._selected_message is None or result != expected_size: + raise AssertionError("the actual selected protocol send was incomplete") + encoded, descriptors, _, _ = self._selected_message + self._selected_message = (encoded, descriptors, descriptor, result) + return True + return False + + def _record(self) -> None: + self._require_lifetime() + if self._selected_message is None: + raise AssertionError("the actual selected protocol message is absent") + encoded, descriptors, descriptor, result = self._selected_message + path = Path(f"/proc/{self.issuer.pid}/fd/{descriptor}") + info = path.stat() + if not stat.S_ISSOCK(info.st_mode): + raise AssertionError("the actual protocol descriptor is not a kernel socket") + if self.schema == OFFER_SCHEMA and not descriptors or self.schema != OFFER_SCHEMA and descriptors: + raise AssertionError("the actual protocol ancillary descriptors differ from the expected message class") + objects = [] + for received in descriptors: + item = Path(f"/proc/{self.issuer.pid}/fd/{received}") + actual = item.stat() + objects.append((received, actual.st_dev, actual.st_ino, os.readlink(item))) + self._observation = PeerMessageObservation(self.issuer.pid, self.start_token, self.boot_id, self.schema, self.direction, self.message_when, descriptor, info.st_ino, hashlib.sha256(encoded).hexdigest(), len(encoded), result, descriptors, tuple(objects)) + logger.bind(native_peer_schema=self.schema, native_peer_direction=self.direction, native_peer_boundary=self.message_when, process_id=self.issuer.pid).info("Actual production protocol process stopped at its selected kernel message boundary") + self._hit.set() + + def wait(self, timeout: float = 900) -> PeerMessageObservation: + result = self.wait_observation(timeout) + if not isinstance(result, PeerMessageObservation): + raise AssertionError("the native peer barrier did not observe its actual message") + return result + + +def _descriptor_snapshot(pid: int) -> tuple[tuple[int, int, int, str], ...]: + result = [] + for path in sorted(Path(f"/proc/{pid}/fd").iterdir(), key=lambda item: int(item.name)): + info = path.stat() + result.append((int(path.name), info.st_dev, info.st_ino, os.readlink(path))) + return tuple(result) + + +def _stopped_receiver(pid: int, start_token: str, pidfd: int, descriptors: tuple[tuple[int, int, int, str], ...]) -> dict[str, object]: + """Prove the exact live receiver cannot yet observe userspace messages.""" + poll = select.poll() + poll.register(pidfd, select.POLLIN) + if poll.poll(0) or _token(pid) != start_token: + raise AssertionError("the receiver lifetime ended before the original boundary observation") + state = Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split()[0] + if state not in {"T", "t"} or _descriptor_snapshot(pid) != descriptors: + raise AssertionError("the receipt-free receiver is not still stopped with its exact descriptors") + return {"pid": pid, "start_token": start_token, "boot_id": Path("/proc/sys/kernel/random/boot_id").read_text().strip(), "state": state, "descriptor_sha256": hashlib.sha256(canonical(descriptors)).hexdigest(), "pidfd_alive": True} + + +def _assert_no_target_admission(context: NativeNegativeContext, *, running_pid: int | None = None) -> None: + state = Path(context.enrollment.target.owner.path).parent + config = BootstrapConfig.read(Path(context.enrollment.target.bootstrap_config.path)) + journals = [item for item in context.enrollment.target.mounts if item.destination == str(config.journal_root) and item.writable] + if len(journals) != 1 or set(os.listdir(state)) - {".supervisor-owner.lock", "logs"} or os.listdir(journals[0].source): + raise AssertionError("the prelaunch target created a native consumer claim or operation") + if running_pid is not None and Path(f"/proc/{running_pid}/task/{running_pid}/children").read_text().strip(): + raise AssertionError("the actual prelaunch target launched a child") + for path in (state / "logs").glob("recovery-epoch-bootstrap*.log"): + if "Recovery custody acknowledged" in path.read_text(): + raise AssertionError("the actual consumer accepted an unauthorized current grant") + + +def _assert_received_objects(context: NativeNegativeContext, observation: PeerMessageObservation, issuer_pid: int) -> None: + expected = {(item.device, item.inode): item.kind for item in _consumer_configuration(context).transferred_resources} + if len(observation.descriptor_objects) != len(expected) or {(device, inode) for _, device, inode, _ in observation.descriptor_objects} != set(expected): + raise AssertionError("actual recvmsg did not install the exact complete original object set") + for descriptor, device, inode, path in observation.descriptor_objects: + if expected[device, inode] == "network-namespace": + if path != f"net:[{inode}]": + raise AssertionError("actual recvmsg namespace object differs") + else: + lock = inspect_kernel_lock(observation.process_id, descriptor, device, inode) + if lock is None or lock.mode != "WRITE" or lock.pid != issuer_pid: + raise AssertionError("actual recvmsg did not retain the issuer's exclusive OFD") + + +def _stop_target_after_failure(context: NativeNegativeContext) -> None: + state = _target_state(context) + if state.get("Running") is True: + target = DockerTraceProcess(context) + try: + target.kill() + target.wait(30) + finally: + target.close() + + +def _run_peer_case(context: NativeNegativeContext, case: str) -> dict[str, object]: + require_native_context(context) + if case not in REQUIRED_PEER_CASES: + raise ValueError("a reviewed production peer refusal is required") + if _target_state(context).get("Running") is not False or PEER_SOCKET.exists() or PEER_SOCKET.is_symlink(): + raise AssertionError("a previous target lifetime or issuer endpoint remains; never reset it to retry") + epoch_before = _epoch_snapshot(context.epoch_root) + observations: dict[str, object] = {"case": case} + log = context.fixture_root / ("native-peer-" + case + ".log") + issuer = None + host_barrier = None + target_barrier = None + target: DockerTraceProcess | None = None + replacement: DockerTraceProcess | None = None + paused_snapshot: tuple[tuple[int, int, int, str], ...] | None = None + unacknowledged_snapshot: tuple[tuple[int, int, int, str], ...] | None = None + with ExitStack() as held: + output = held.enter_context(log.open("xb")) + try: + issuer = subprocess.Popen([sys.executable, "-B", "-m", "auto_coder.management.recovery_epoch"], stdout=output, stderr=subprocess.STDOUT, env=dict(os.environ, PYTHONDONTWRITEBYTECODE="1")) + + def stop_receiver_before_offer() -> None: + nonlocal target, paused_snapshot + target = DockerTraceProcess(context) + target.pause() + paused_snapshot = _descriptor_snapshot(target.pid) + _assert_no_target_admission(context, running_pid=target.pid) + + receiving_offer = case == "received-rights-without-receipt-refused" + challenge = case in {"changed-target-lifetime-refused", "foreign-acquirer-sh-refused"} + host_barrier = KernelPeerMessageBarrier( + issuer, + context=context, + schema=CHALLENGE_SCHEMA if challenge else OFFER_SCHEMA, + direction="receive" if challenge else "send", + when="before" if receiving_offer else "after", + before_send=stop_receiver_before_offer if case == "queued-rights-without-receipt-refused" else None, + ) + host_barrier.start() + host_barrier.resume_initial() + _wait(lambda: PEER_SOCKET.exists() and "Fixed recovery issuer endpoint is listening" in log.read_text(), "the actual fixed issuer endpoint is listening", timeout=120) + _docker("start", context.enrollment.target.management_id) + observation = host_barrier.wait(timeout=600) + observations["issuer_syscall"] = asdict(observation) + assert_unreserved_epoch(context.epoch_root) + if case == "queued-rights-without-receipt-refused": + if target is None or paused_snapshot is None or _descriptor_snapshot(target.pid) != paused_snapshot or observation.syscall_result != observation.bytes_observed: + raise AssertionError("the successful native send did not precede actual receiver custody") + observations["receiver_before_and_after_enqueue"] = paused_snapshot + _assert_no_target_admission(context, running_pid=target.pid) + unacknowledged_snapshot = paused_snapshot + host_barrier.release() + elif receiving_offer: + target = DockerTraceProcess(context) + target_barrier = KernelPeerMessageBarrier(target, context=context, schema=OFFER_SCHEMA, direction="receive") + target_barrier.start() + target_barrier.resume_initial() + host_barrier.release() + received = target_barrier.wait(timeout=90) + _assert_received_objects(context, received, issuer.pid) + observations["receiver_syscall"] = asdict(received) + assert_unreserved_epoch(context.epoch_root) + _assert_no_target_admission(context, running_pid=target.pid) + unacknowledged_snapshot = _descriptor_snapshot(target.pid) + elif case == "changed-target-lifetime-refused": + target = DockerTraceProcess(context) + observations["original_target_lifetime"] = {"pid": target.pid, "start_token": target.start_token} + target.kill() + target.wait(30) + _docker("start", context.enrollment.target.management_id) + replacement = DockerTraceProcess(context) + if (replacement.pid, replacement.start_token) == (target.pid, target.start_token): + raise AssertionError("the actual target restart did not create a new lifetime") + observations["replacement_target_lifetime"] = {"pid": replacement.pid, "start_token": replacement.start_token} + _assert_no_target_admission(context, running_pid=replacement.pid) + host_barrier.release() + else: + target = DockerTraceProcess(context) + original = next(item for item in _consumer_configuration(context).original_fences if item.name.startswith("owner-")) + observations["foreign_original_custody"] = held.enter_context(_foreign_acquirer(target.pid, original.device, original.inode, "READ")) + host_barrier.release() + # Trace ownership has ended before ordinary Popen wait/poll. + issuer.wait(timeout=120) + if issuer.returncode != 2: + raise AssertionError("the actual issuer did not refuse the incomplete native custody") + if unacknowledged_snapshot is not None: + assert target is not None + # Keep the exact live receiver blocked until the issuer's + # genuine receipt timeout refuses. Killing it beforehand would + # let a dead-peer check hide missing acknowledgment enforcement. + target.check() + observations["live_receiver_after_issuer_refusal"] = _stopped_receiver(target.pid, target.start_token, target.pidfd, unacknowledged_snapshot) + assert_unreserved_epoch(context.epoch_root) + _assert_no_target_admission(context, running_pid=target.pid) + observations.update(receiver_alive_until_issuer_refusal=True, issuer_refused_before_receiver_termination=True) + if target_barrier is not None: + target_barrier.kill() + else: + target.kill() + target.wait(30) + _wait(lambda: _target_state(context).get("Running") is False, "the actual refused consumer exits", timeout=90) + _assert_no_target_admission(context) + if PEER_SOCKET.exists() or PEER_SOCKET.is_symlink(): + raise AssertionError("the original refusing issuer left its endpoint; never unlink it to retry") + if _epoch_snapshot(context.epoch_root) != epoch_before or "One consumed recovery epoch grant send completed" in log.read_text(): + raise AssertionError("incomplete native peer custody published or dispatched an epoch") + observations.update(issuer_returncode=issuer.returncode, epoch_unreserved=True, issuer_endpoint_removed_by_original=True, consumer_grant_accepted=False, no_child_or_admission=True) + return observations + finally: + if target_barrier is not None: + target_barrier.close() + if host_barrier is not None: + host_barrier.close() + if issuer is not None and issuer.poll() is None: + issuer.kill() + issuer.wait(timeout=30) + _stop_target_after_failure(context) + for process in (replacement, target): + if process is not None: + process.close() + + +def run_native_peer_cases(context: NativeNegativeContext, preserved_snapshot: Callable[[], object]) -> list[str]: + """Require actual peer refusal before every original epoch publication.""" + require_native_context(context) + observations = [] + for case in sorted(REQUIRED_PEER_CASES): + preserved = preserved_snapshot() + observation = _run_peer_case(context, case) + if preserved_snapshot() != preserved: + raise AssertionError("actual peer refusal changed original records, OAuth or unrelated workers") + observations.append(observation) + logger.bind(native_peer_case=case, epoch_unreserved=True, consumer_grant_accepted=False).info("Actual production peer refusal preserved original state") + destination = context.fixture_root / "native-peer-observations.json" + descriptor = os.open(destination, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + with os.fdopen(descriptor, "wb") as stream: + stream.write(canonical({"schema": "actual-recovery-native-peer-faults-v1", "cases": observations})) + stream.flush() + os.fsync(stream.fileno()) + return [str(item["case"]) for item in observations] diff --git a/scripts/recovery_vm/Dockerfile b/scripts/recovery_vm/Dockerfile new file mode 100644 index 000000000..140742c15 --- /dev/null +++ b/scripts/recovery_vm/Dockerfile @@ -0,0 +1,10 @@ +FROM ubuntu:24.04 +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ca-certificates cloud-image-utils curl python3 qemu-system-x86 qemu-utils \ + && rm -rf /var/lib/apt/lists/* +ARG CLOUD_IMAGE_URL=https://cloud-images.ubuntu.com/minimal/releases/noble/release-20261105/ubuntu-24.04-minimal-cloudimg-amd64.img +ARG CLOUD_IMAGE_SHA256=406630c08b5e545fcc7209854b129d47e3af822db128c9f19336593b5a2d3fe0 +RUN curl --fail --location --retry 3 "$CLOUD_IMAGE_URL" -o /opt/base.img \ + && echo "$CLOUD_IMAGE_SHA256 /opt/base.img" | sha256sum --check --strict +COPY runner.sh /opt/runner.sh +ENTRYPOINT ["/bin/bash", "/opt/runner.sh"] diff --git a/scripts/recovery_vm/README.md b/scripts/recovery_vm/README.md new file mode 100644 index 000000000..2d9f6c73a --- /dev/null +++ b/scripts/recovery_vm/README.md @@ -0,0 +1,183 @@ +# Native recovery epoch fixture + +This runner boots a fresh Ubuntu VM with its own Linux kernel, initial namespaces, +systemd, Docker engine and boot identity. It does not run the host issuer on the +outer CI runner or use a privileged Docker container as a substitute for the VM. +The driver exercises original provenance, current containment, genuine installed +READY and the production issuer/consumer entrypoints. A run is acceptance only +when its exact scenario oracle succeeds and the VM exits successfully. + +Preparation selects KVM when the outer host exposes `/dev/kvm`; its separate +Compose override passes that exact device and its actual group to the unprivileged +QEMU process. The preparing shell does not need membership in the device group. +If that device is absent, preparation explicitly selects QEMU TCG CPU emulation +with the `max` CPU model. The selected accelerator is bound in the input manifest. +Both modes boot the same real kernel and execute the same native oracles. A KVM +permission or execution failure is an error; the runner does not silently switch +an already prepared KVM input to another mode. + +The outer QEMU container has a read-only root, no capabilities, no new privileges, +2 CPUs, 6 GiB memory and a 256-process limit. Only prepared input, the new guest +disk and bounded output are mounted. Each scenario gets a new directory and new +32 GiB sparse guest disk; an existing disk or original epoch is never reset. + +Preparation also binds the actual source Docker image store (`overlay2` or the +`overlayfs` containerd snapshotter). The fresh guest configures the same store +before its first daemon start and verifies both imported images retain their +exact approved IDs. Importing between these stores can change the meaning of +Docker's image ID; a changed ID is a preparation failure, not a reason to select +another image or weaken the production source checks. +Docker documents the [fresh-install storage default](https://docs.docker.com/engine/storage/containerd/) +and the [cross-store image-ID behavior](https://github.com/docker/cli/issues/6670). + +The selected guest service profile is ordinary system Docker with its standard +daemon-managed, separate-process containerd and `PrivateMounts=yes`. Before package +installation can start an engine, preparation masks Docker and the standalone +containerd service. It then installs the reviewed Docker override, unmasks Docker +only, and checks the actual daemon command, PID/start identity, systemd cgroup +driver, empty engine and managed containerd child. Both daemons must share one +mount namespace distinct from PID 1 while retaining the host PID, user and network +namespaces. These observations are retained in `docker-profile.json`, and the +effective service configuration is included in `guest-platform.txt`. + +This mount profile keeps transient container mounts out of the host systemd unit +catalog throughout target starts and restarts; production still requires exact +equality with the approved catalog and refuses unknown routes. Docker documents +[automatic containerd startup](https://docs.docker.com/reference/cli/dockerd/#run-containerd-standalone), +and systemd 255 documents +[PrivateMounts propagation](https://www.freedesktop.org/software/systemd/man/255/systemd.exec.html#PrivateMounts=). +This is provisioning of the fresh synthetic VM before any original history or +recovery approval exists. It does not change an operational host, configure the +production issuer, or establish acceptance for every stock Docker service profile +or an incident with missing originals. + +Before original producers or recovery approval, the guest stops and masks these +six explicitly enumerated timer/service pairs from its observed stock timer +inventory: + +| Timer | Service | +| --- | --- | +| `systemd-tmpfiles-clean.timer` | `systemd-tmpfiles-clean.service` | +| `apt-daily.timer` | `apt-daily.service` | +| `apt-daily-upgrade.timer` | `apt-daily-upgrade.service` | +| `dpkg-db-backup.timer` | `dpkg-db-backup.service` | +| `e2scrub_all.timer` | `e2scrub_all.service` | +| `fstrim.timer` | `fstrim.service` | + +The existing stock-route fixture separately inhibits `motd-news.timer` and its +service. There is no wildcard timer discovery or inhibition. + +A scheduled tmpfiles execution previously changed its approved service readback. +The [later native after-receipt-head job](https://github.com/kitamura-tetsuo/auto-coder/actions/runs/43321049334/job/131543555073) +recorded target startup at `2026-11-12 23:58:08.1399639 UTC`, after route approval, +then `dpkg-db-backup.service` starting at `2026-11-13 00:00:00.2218320 UTC` and +finishing at `00:00:00.3340939 UTC`. The issuer refused at `00:01:07.623 UTC` +in the per-unit effective-route comparison, before the selected publication +boundary. Its bounded cause metadata did not export the exact mismatching unit +name. The observed post-approval backup execution changes the full `ExecStart` +readback included in that unit's strict digest even after the service returns +inactive. The finite preparation profile also covers the other observed stock +housekeeping timers, so their calendar schedules do not determine acceptance. + +Preparation stops all six timers before waiting for any selected service to +stop. It uses the production systemd reader to require real protected +`/dev/null` masks, inactive state, zero service PIDs and no pending daemon reload. +It reobserves every inhibited fact and retains the matching result, +initial/stopped facts and mask identities in `maintenance-profile.json`, with +stdout and file logging. An unavailable unit, failed state or stop, preexisting +mask path or changed final state fails preparation. This finite fresh-VM setup +leaves the production whole-route catalog and digest comparisons unchanged; it +never refreshes an approval after maintenance or changes an operational recovery +host. + +The guest installs the exact `uv==0.13.6` release and validates its reported release +number. The complete observed `--version` banner, including its bounded build +descriptor, is retained in `uv-version.txt` and supplied to staging and recovery +authorization. The production installer continues to require exact banner +equality; the fixture does not strip build identity from that comparison. + +## Reproduce a reviewed scenario + +Use a clean, committed checkout with the locked test dependencies installed. The +read credential is used only to verify the actual immutable tested-source tag and +successful GitHub test run for the installed READY. It needs repository contents +and Actions read access, not publication authority. Keep it in an existing private +file; do not put its value on the command line. + +```sh +uv sync --locked --dev --extra test +uv run python scripts/prepare_management_test_image.py +docker pull caddy:2 +export APPROVED_AUTHORIZATION_ID=reviewed-recovery-fixture-unique +RECOVERY_FIXTURE=/var/tmp/ac-recovery-vm-unique +MANAGER_IMAGE_ID=$(docker image inspect --format '{{.Id}}' auto-coder:management-fixture-$(git rev-parse --short=12 HEAD)) +CADDY_IMAGE_ID=$(docker image inspect --format '{{.Id}}' caddy:2) +READ_TOKEN_FILE=/absolute/path/to/private-read-token +bash scripts/recovery_vm/prepare.sh "$RECOVERY_FIXTURE" "$APPROVED_AUTHORIZATION_ID" "$MANAGER_IMAGE_ID" "$CADDY_IMAGE_ID" "$READ_TOKEN_FILE" grant +docker compose --env-file "$RECOVERY_FIXTURE/compose.env" build +docker compose --env-file "$RECOVERY_FIXTURE/compose.env" run --rm fixture +``` + +The prepared `compose.env` selects the exact reviewed Compose files, including the +KVM-only device override when applicable. Do not add a separate `-f` argument +that would discard this selection. The VM-only authorization marker binds the +actual new machine, boot and engine. The driver verifies it before provisioning +any management fixture. It refuses a populated engine or earlier fixture roots. + +`ACCEPTANCE_SCENARIOS` and `required_cases()` in +`scripts/test_management_recovery_epoch_host.py` define the complete finite +scenario oracle. CI derives its matrix directly from this list and requires +every matrix job to pass. The two `kill-after-*` debug aliases select boundaries +already present in the matrix and do not add independent cases. All before/after +publication cases and all eleven mutations after the original intent head run on +independent fresh VMs. A distinct `post-receipt-ready-mutation` scenario changes +the exact READY receipt after consumption and receipt-head publication, then +resumes that same issuer and independently proves refusal with no queued grant. +They +use real ptrace syscall stops around the actual exclusive record/head creation +and successful file/directory fsync, with a pinned SIGKILL and no syscall-result +replacement. The native process oracles also inspect real peer custody, original +store preservation, unrelated worker lifetimes and effect requests, and absence +of a replacement child/admission. Component guard tests cannot substitute for +these executions. + +The `grant` scenario first requires all four actual protocol refusals implemented +in `scripts/recovery_epoch_native_peer.py`: queued rights without a receipt, +received rights without a receipt, a replaced target lifetime, and an SH lock +whose acquiring process differs from the target. Each must preserve an entirely +unreserved epoch before a later independent invocation can grant. Its successful +path also holds the original issuer at the durable reservation head while an +overlapping contender is refused. The separate `lost-reply` scenario requires an +actual successful grant send while the receiver is stopped, then verifies the +issuer's pidfd is dead before resuming the receiver; consumption must remain and +the receiver must refuse current custody. The companion helper tests validate +the fixture's native message parsing and guards; they are not VM acceptance. + +## Verifier-omission control + +A separate CI job proves that the post-receipt scenario detects omission of only +the second final verification. It uses another fresh VM, copies the reviewed +project into a private control directory, and removes exactly that one call with +structural and byte-level checks. Its original producers, source identities, +derived images and issuer all use the same copied package before any history is +created. The canonical checkout and installed package remain unchanged. + +The control must reach the real receipt-head barrier and fail the unchanged +native oracle because an exact canonical grant is actually queued on the stopped +receiver's socket. An unrelated error, timeout, process exit or log message cannot +pass this control. The result has a separate `mutation-detected` status, binds the +exact original/mutant source evidence and preserves consumed state, the deliberate +READY mutation and unrelated workers. It supplies test-sensitivity evidence and +does not count as a successful production acceptance scenario. + +To prepare this explicitly selected control, use the reproduction command above +with `post-receipt-ready-mutation omit-post-consumption-verifier` as the final two +arguments to `prepare.sh`. Every ordinary preparation records `mutation_control` +as `none`. Both the complete unmodified production matrix and this separate +control must succeed for the aggregate CI gate. + +The guest read credential is deleted on exit. The outer input credential must +also be deleted after the run; the CI workflow does this even on failure. +Artifacts contain the result, bounded diagnostics and guest/serial logs, not the +original OAuth stores or a reusable authority inventory. The fixture is not an +incident-recovery procedure and cannot fill missing historical evidence. diff --git a/scripts/recovery_vm/compose.kvm.yaml b/scripts/recovery_vm/compose.kvm.yaml new file mode 100644 index 000000000..f581518b3 --- /dev/null +++ b/scripts/recovery_vm/compose.kvm.yaml @@ -0,0 +1,6 @@ +services: + fixture: + group_add: + - "${RECOVERY_KVM_GID:?Set the actual KVM device group}" + devices: + - /dev/kvm:/dev/kvm:rwm diff --git a/scripts/recovery_vm/compose.yaml b/scripts/recovery_vm/compose.yaml new file mode 100644 index 000000000..dbe2bf4dc --- /dev/null +++ b/scripts/recovery_vm/compose.yaml @@ -0,0 +1,30 @@ +services: + fixture: + build: + context: ${RECOVERY_VM_TOOLING:?Set the reviewed tooling directory} + environment: + APPROVED_AUTHORIZATION_ID: ${APPROVED_AUTHORIZATION_ID:?Set the matching disposable fixture authorization} + user: "${RECOVERY_VM_UID:?Set the preparation UID}:${RECOVERY_VM_GID:?Set the preparation GID}" + init: true + cpus: 2.0 + mem_limit: 6g + pids_limit: 256 + read_only: true + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + tmpfs: + - /tmp:size=128m + volumes: + - type: bind + source: ${RECOVERY_VM_ROOT:?Set the prepared fixture directory}/input + target: /input + read_only: true + - type: bind + source: ${RECOVERY_VM_ROOT:?Set the prepared fixture directory}/disk + target: /disk + - type: bind + source: ${RECOVERY_VM_ROOT:?Set the prepared fixture directory}/output + target: /output + stop_grace_period: 30s diff --git a/scripts/recovery_vm/guest.sh b/scripts/recovery_vm/guest.sh new file mode 100644 index 000000000..687e02edd --- /dev/null +++ b/scripts/recovery_vm/guest.sh @@ -0,0 +1,313 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +umask 077 +input=/mnt/fixture-input +output=/mnt/fixture-output +export PYTHONDONTWRITEBYTECODE=1 +exec > >(tee -a "$output/guest.log") 2>&1 + +finish() { + result=$? + trap - EXIT + set +e + rm -f /run/recovery-fixture-source-token + { + systemctl cat cloud-final.service cloud-config.service cloud-init.service cloud-init-local.service + systemctl show cloud-final.service cloud-config.service cloud-init.service cloud-init-local.service -p ExecStart -p ActiveState -p UnitFileState + systemctl cat docker.service docker.socket containerd.service + systemctl show docker.service containerd.service -p ExecStart -p PrivateMounts -p MainPID -p ActiveState -p LoadState -p UnitFileState -p FragmentPath -p DropInPaths + docker version + docker compose version + } > "$output/guest-platform.txt" 2>&1 + printf '%s\n' "$result" | tee "$output/guest-exit-code" > /dev/null + sync + systemctl poweroff --no-block + exit "$result" +} +trap finish EXIT +python3 - "$input" <<'PY' +import hashlib +import json +import pathlib +import sys + +root = pathlib.Path(sys.argv[1]) +manifest = json.loads((root / "manifest.json").read_text()) +if manifest.get("schema") != "auto-coder-recovery-vm-input-v1": + raise ValueError("Unsupported recovery fixture input") +for name, digest in manifest["inputs_sha256"].items(): + with (root / name).open("rb") as stream: + if hashlib.file_digest(stream, "sha256").hexdigest() != digest: + raise ValueError("Recovery fixture input changed: " + name) +PY +export DEBIAN_FRONTEND=noninteractive +# Match the source store before the new daemon starts; cross-store imports can +# change Docker's image IDs even when their layers are identical. +mkdir -p /etc/docker +test ! -e /etc/docker/daemon.json +test ! -L /etc/docker/daemon.json +install -m 600 "$input/docker-daemon.json" /etc/docker/daemon.json +# This fresh VM has no previous engine. Prevent package auto-start before the +# selected ordinary service profile exists; no original history exists yet. +test ! -e /usr/bin/dockerd +test ! -e /run/docker.pid +test ! -e /var/lib/docker +for fixture_unit in docker.service docker.socket containerd.service; do + test ! -e "/etc/systemd/system/$fixture_unit" + test ! -L "/etc/systemd/system/$fixture_unit" + ln -s /dev/null "/etc/systemd/system/$fixture_unit" +done +systemctl daemon-reload +apt-get update +apt-get install -y --no-install-recommends ca-certificates cron curl docker.io docker-compose-v2 git python3-venv +mkdir -p /etc/systemd/system/docker.service.d +test ! -e /etc/systemd/system/docker.service.d/recovery-fixture.conf +tee /etc/systemd/system/docker.service.d/recovery-fixture.conf > /dev/null <<'UNIT' +[Service] +ExecStart= +ExecStart=/usr/bin/dockerd -H fd:// +PrivateMounts=yes +UNIT +# Docker starts its normal managed containerd process in this same namespace. +# A standalone containerd in a different mount namespace could not see its +# overlay/netns objects. Keep that unused packaged service masked throughout. +for fixture_unit in docker.service docker.socket; do + test "$(readlink "/etc/systemd/system/$fixture_unit")" = /dev/null + rm "/etc/systemd/system/$fixture_unit" +done +systemctl daemon-reload +systemctl start docker +python3 - "$output/docker-profile.json" <<'PY' +import json +import os +from pathlib import Path +import subprocess +import sys + + +def properties(unit): + names = ("MainPID", "ActiveState", "LoadState", "PrivateMounts") + raw = subprocess.check_output(["systemctl", "show", unit, "--property=" + ",".join(names)], text=True) + result = dict(line.split("=", 1) for line in raw.splitlines()) + assert set(result) == set(names) + return result + + +def process(pid): + proc = Path("/proc") / str(pid) + fields = (proc / "stat").read_text().rsplit(")", 1)[1].split() + assert fields[0] not in ("Z", "X", "x") and int(fields[19]) > 0 + return { + "pid": pid, + "parent": int(fields[1]), + "start": fields[19], + "namespaces": {name: os.readlink(proc / "ns" / name) for name in ("pid", "user", "net", "mnt")}, + } + + +service, standalone = properties("docker.service"), properties("containerd.service") +assert service["ActiveState"] == "active" and service["PrivateMounts"] == "yes" +assert standalone["LoadState"] == "masked" and standalone["ActiveState"] == "inactive" and standalone["MainPID"] == "0" +pid = int(service["MainPID"]) +assert pid > 1 and Path("/run/docker.pid").read_text().strip() == str(pid) +assert os.path.samefile(f"/proc/{pid}/exe", "/usr/bin/dockerd") +assert Path(f"/proc/{pid}/cmdline").read_bytes().split(b"\0") == [b"/usr/bin/dockerd", b"-H", b"fd://", b""] +daemon = process(pid) +host = process(1) +assert all(daemon["namespaces"][name] == host["namespaces"][name] for name in ("pid", "user", "net")) +assert daemon["namespaces"]["mnt"] != host["namespaces"]["mnt"] +children = {int(value) for task in Path(f"/proc/{pid}/task").iterdir() for value in (task / "children").read_text().split()} +managed = [child for child in children if os.path.samefile(f"/proc/{child}/exe", "/usr/bin/containerd")] +assert len(managed) == 1 +containerd = process(managed[0]) +assert containerd["parent"] == pid and containerd["namespaces"] == daemon["namespaces"] +assert subprocess.check_output(["docker", "info", "--format", "{{.CgroupDriver}}"], text=True).strip() == "systemd" +assert subprocess.check_output(["docker", "ps", "--all", "--quiet"], text=True).strip() == "" +assert process(pid) == daemon and process(managed[0]) == containerd +result = {"schema": "auto-coder-recovery-vm-docker-profile-v1", "profile": "ordinary-system-docker-managed-containerd-private-mounts", "daemon": daemon, "managed_containerd": containerd, "standalone_service": standalone} +with Path(sys.argv[1]).open("x") as stream: + json.dump(result, stream, sort_keys=True) +print("Verified ordinary Docker with its managed containerd in one private mount namespace before original fixture creation.", flush=True) +PY +docker info --format 'Actual guest image store: {{.Driver}} {{json .DriverStatus}}' +python3 -m venv /opt/fixture-tools +/opt/fixture-tools/bin/pip install uv==0.13.6 +mkdir -m 700 /opt/recovery-fixture /opt/reviewed-source +tar --extract --gzip --file "$input/source.tar.gz" --directory /opt/recovery-fixture +cd /opt/recovery-fixture +/opt/fixture-tools/bin/uv sync --locked --no-editable --no-dev --extra test --link-mode copy +verify_installed_watcher() { +.venv/bin/python - <<'PY' +from pathlib import Path +import auto_coder.management.proxy_watch_cli as watcher + +source = Path(watcher.__file__) +source.chmod(0o600) +identity = source.stat() +assert identity.st_uid == 0 and identity.st_nlink == 1 +print("Installed watcher source is a private, independently copied file.") +PY +} +verify_installed_watcher +.venv/bin/python - "$output/maintenance-profile.json" <<'PY' +import json +import os +from dataclasses import asdict +from pathlib import Path +import subprocess +import sys + +from auto_coder.logger_config import get_logger, setup_logger +from auto_coder.management.containment_route_manifest import _systemd_unit + +MAINTENANCE_PAIRS = ( + ("systemd-tmpfiles-clean.timer", "systemd-tmpfiles-clean.service"), + ("apt-daily.timer", "apt-daily.service"), + ("apt-daily-upgrade.timer", "apt-daily-upgrade.service"), + ("dpkg-db-backup.timer", "dpkg-db-backup.service"), + ("e2scrub_all.timer", "e2scrub_all.service"), + ("fstrim.timer", "fstrim.service"), +) +MAINTENANCE_TIMERS = tuple(timer for timer, _ in MAINTENANCE_PAIRS) +MAINTENANCE_SERVICES = tuple(service for _, service in MAINTENANCE_PAIRS) +MAINTENANCE_UNITS = tuple(unit for pair in MAINTENANCE_PAIRS for unit in pair) +MASK_ROOT = Path("/etc/systemd/system") +logger = get_logger(__name__) + + +def inhibit_maintenance(output): + """Stop and mask this fresh VM's selected maintenance pairs before originals.""" + evidence = {"schema": "auto-coder-recovery-vm-maintenance-profile-v1", "status": "preparing"} + try: + masks = tuple(MASK_ROOT / unit for unit in MAINTENANCE_UNITS) + if any(path.exists() or path.is_symlink() for path in masks): + raise ValueError("The fresh maintenance mask paths are not empty") + initial = tuple(_systemd_unit(unit) for unit in MAINTENANCE_UNITS) + evidence["initial"] = [asdict(fact) for fact in initial] + if any(not fact.unit_file_state or fact.active_state not in {"active", "inactive"} or fact.needs_daemon_reload for fact in initial): + raise ValueError("The original installed maintenance units are unavailable") + # Stop every selected timer before waiting for any selected service. + # Never reset failed units or change an approved recovery environment. + for unit in (*MAINTENANCE_TIMERS, *MAINTENANCE_SERVICES): + subprocess.run(("systemctl", "stop", "--", unit), check=True, timeout=30) + stopped = tuple(_systemd_unit(unit) for unit in MAINTENANCE_UNITS) + evidence["stopped"] = [asdict(fact) for fact in stopped] + if any(fact.active_state != "inactive" or fact.main_pid != 0 or fact.needs_daemon_reload for fact in stopped): + raise ValueError("The actual maintenance units did not stop") + subprocess.run(("systemctl", "mask", "--", *MAINTENANCE_UNITS), check=True, timeout=30) + subprocess.run(("systemctl", "daemon-reload"), check=True, timeout=30) + observed = tuple(_systemd_unit(unit) for unit in MAINTENANCE_UNITS) + evidence["observed"] = [asdict(fact) for fact in observed] + if any(not fact.inhibited or fact.unit_file_state != "masked" or fact.active_state != "inactive" or fact.main_pid != 0 or fact.needs_daemon_reload for fact in observed): + raise ValueError("The actual maintenance units are not masked and inactive") + evidence["masks"] = [] + for path in masks: + info = path.lstat() + if not path.is_symlink() or os.readlink(path) != "/dev/null": + raise ValueError("The maintenance units lack their actual /dev/null masks") + evidence["masks"].append({"unit": path.name, "path": str(path), "device": info.st_dev, "inode": info.st_ino, "ctime_ns": info.st_ctime_ns, "mode": info.st_mode, "uid": info.st_uid, "target": os.readlink(path)}) + if tuple(_systemd_unit(unit) for unit in MAINTENANCE_UNITS) != observed: + raise ValueError("The inhibited maintenance units changed during verification") + evidence["status"] = "inhibited" + logger.info("Verified the fixed maintenance timer/service pairs are masked and inactive before original fixture creation") + except Exception as error: + evidence.update(status="failed", failure_type=type(error).__name__) + logger.error("Fresh recovery VM maintenance preparation failed: {}", type(error).__name__) + raise + finally: + with output.open("x") as stream: + json.dump(evidence, stream, sort_keys=True) + + +if __name__ == "__main__": + output = Path(sys.argv[1]) + setup_logger(stream=sys.stdout, log_file=str(output.with_suffix(".log"))) + inhibit_maintenance(output) + logger.complete() +PY +docker compose version +install -m 600 "$input/source-token" /run/recovery-fixture-source-token +mapfile -t selected < <(python3 - "$input" <<'PY' +import json +import pathlib +import sys + +manifest = json.loads((pathlib.Path(sys.argv[1]) / "manifest.json").read_text()) +for key in ("authorization_id", "manager_image", "proxy_image", "ready_source_sha", "scenario", "mutation_control"): + print(manifest[key]) +PY +) +driver_module=scripts.test_management_recovery_epoch_host +driver_prefix=() +case "${selected[5]}" in + none) ;; + omit-post-consumption-verifier) + test "${selected[4]}" = post-receipt-ready-mutation + .venv/bin/python -m scripts.recovery_epoch_mutation_control prepare \ + --source /opt/recovery-fixture \ + --destination /opt/recovery-mutation-control \ + --evidence "$output/mutation-source.json" + cd /opt/recovery-mutation-control + /opt/fixture-tools/bin/uv sync --locked --no-editable --no-dev --extra test --link-mode copy + verify_installed_watcher + driver_module=scripts.recovery_epoch_mutation_control + driver_prefix=(run --evidence "$output/mutation-source.json" --) + ;; + *) echo 'Unsupported recovery mutation control.' >&2; exit 2 ;; +esac +fixture_uv_version=$(.venv/bin/python - <<'PY' +import subprocess +from auto_coder.management.recovery_epoch_authorization import validate_uv_version + +banner = subprocess.check_output(["/opt/fixture-tools/bin/uv", "--version"], text=True).strip() +if not banner.startswith("uv "): + raise ValueError("The selected fixture uv did not return its version banner") +version = validate_uv_version(banner.removeprefix("uv ")) +if version.split(" ", 1)[0] != "0.13.6": + raise ValueError("The installed fixture uv does not match the reviewed release pin") +print(version) +PY +) +printf 'uv %s\n' "$fixture_uv_version" | tee "$output/uv-version.txt" +git -C /opt/reviewed-source init +git -C /opt/reviewed-source remote add origin https://github.com/kitamura-tetsuo/auto-coder.git +git -C /opt/reviewed-source fetch --no-tags origin "${selected[3]}" +git -C /opt/reviewed-source checkout --detach FETCH_HEAD +test "$(git -C /opt/reviewed-source rev-parse --is-shallow-repository)" = false +docker image load --input "$input/images.tar" +docker image ls --no-trunc +for selected_image in "${selected[1]}" "${selected[2]}"; do + printf 'Verifying imported fixture image %s\n' "$selected_image" + test "$(docker image inspect --format '{{.Id}}' "$selected_image")" = "$selected_image" +done +python3 - "$input" <<'PY' +import json +import os +import pathlib +import subprocess +import sys + +manifest = json.loads((pathlib.Path(sys.argv[1]) / "manifest.json").read_text()) +marker = { + "schema": "auto-coder-disposable-containment-host-v1", + "authorization_id": manifest["authorization_id"], + "machine_id": pathlib.Path("/etc/machine-id").read_text().strip(), + "boot_id": pathlib.Path("/proc/sys/kernel/random/boot_id").read_text().strip(), + "engine_id": subprocess.check_output(["docker", "info", "--format", "{{.ID}}"], text=True).strip(), + "synthetic_only": True, +} +fd = os.open("/etc/auto-coder-disposable-containment-host.json", os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) +with os.fdopen(fd, "w") as stream: + json.dump(marker, stream, sort_keys=True) +PY +export AUTOCODER_FRESH_TEST_BASE_IMAGE="${selected[1]}" +export AUTOCODER_RECOVERY_CADDY_IMAGE="${selected[2]}" +export PATH="/opt/fixture-tools/bin:$PATH" +.venv/bin/python -m "$driver_module" "${driver_prefix[@]}" \ + --authorization-id "${selected[0]}" \ + --source-repository /opt/reviewed-source \ + --source-token-file /run/recovery-fixture-source-token \ + --source-sha "${selected[3]}" \ + --scenario "${selected[4]}" \ + --uv /opt/fixture-tools/bin/uv --uv-version "$fixture_uv_version" \ + --output "$output/recovery-epoch-result.json" diff --git a/scripts/recovery_vm/prepare.sh b/scripts/recovery_vm/prepare.sh new file mode 100644 index 000000000..04199ba8b --- /dev/null +++ b/scripts/recovery_vm/prepare.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +umask 077 + +# Export exact reviewed source/images; preparation does not launch a guest. +if [[ ( "$#" != 6 && "$#" != 7 ) || ! "$1" =~ ^/var/tmp/ac-recovery-vm-[a-zA-Z0-9_-]+$ || ! "$2" =~ ^[a-zA-Z0-9_.:-]{1,128}$ || ! "$3" =~ ^sha256:[0-9a-f]{64}$ || ! "$4" =~ ^sha256:[0-9a-f]{64}$ || ! "$6" =~ ^[a-z][a-z-]{1,63}$ ]]; then + echo 'Usage: prepare.sh /var/tmp/ac-recovery-vm-UNIQUE AUTHORIZATION_ID MANAGER_IMAGE_ID CADDY_IMAGE_ID READ_TOKEN_FILE SCENARIO [omit-post-consumption-verifier]' >&2 + exit 2 +fi +fixture_root=$1 +fixture_authorization=$2 +manager_image=$3 +proxy_image=$4 +source_token_file=$5 +fixture_scenario=$6 +fixture_mutation_control=${7:-none} +if [[ "$fixture_mutation_control" != none && ( "$fixture_mutation_control" != omit-post-consumption-verifier || "$fixture_scenario" != post-receipt-ready-mutation ) ]]; then + echo 'Only the explicit post-receipt verifier-omission control is supported.' >&2 + exit 2 +fi +tooling_root=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P) +repo_root=$(git -C "$tooling_root" rev-parse --show-toplevel) +cd -- "$repo_root" +"$repo_root/.venv/bin/python" - "$fixture_scenario" <<'PY' +import sys +from scripts.test_management_recovery_epoch_host import required_cases + +required_cases(sys.argv[1]) +PY +fixture_accelerator=tcg +fixture_kvm_gid=0 +fixture_compose_files="$tooling_root/compose.yaml" +if [[ -c /dev/kvm ]]; then + fixture_accelerator=kvm + fixture_kvm_gid=$(stat -c '%g' /dev/kvm) + fixture_compose_files="$fixture_compose_files:$tooling_root/compose.kvm.yaml" +fi +test -f "$source_token_file" +if [[ -e "$fixture_root" || -L "$fixture_root" ]]; then + echo 'The recovery fixture directory must not exist.' >&2 + exit 2 +fi +if [[ -n "$(git -C "$repo_root" status --porcelain --untracked-files=normal)" ]]; then + echo 'Commit reviewed source before exporting the recovery VM inputs.' >&2 + exit 2 +fi +for selected in "$manager_image" "$proxy_image"; do + test "$(docker image inspect --format '{{.Id}}' "$selected")" = "$selected" +done +fixture_driver=$(docker info --format '{{.Driver}}') +fixture_driver_status=$(docker info --format '{{json .DriverStatus}}') +if [[ "$fixture_driver" == overlayfs && "$fixture_driver_status" == *'io.containerd.snapshotter.v1'* ]]; then + fixture_image_store=containerd +elif [[ "$fixture_driver" == overlay2 && "$fixture_driver_status" != *'io.containerd.snapshotter.v1'* ]]; then + fixture_image_store=overlay2 +else + echo 'The recovery VM requires a supported, explicitly matched Docker image store.' >&2 + exit 2 +fi +mkdir -m 700 "$fixture_root" "$fixture_root/input" "$fixture_root/output" "$fixture_root/disk" +exec > >(tee -a "$fixture_root/output/preparation.log") 2>&1 +printf 'Exporting exact fixture IDs from Docker image store %s.\n' "$fixture_image_store" +if [[ "$fixture_image_store" == overlay2 ]]; then + tee "$fixture_root/input/docker-daemon.json" > /dev/null <<'JSON' +{"features":{"containerd-snapshotter":false},"storage-driver":"overlay2"} +JSON +else + tee "$fixture_root/input/docker-daemon.json" > /dev/null <<'JSON' +{"features":{"containerd-snapshotter":true},"storage-driver":"overlayfs"} +JSON +fi +git -C "$repo_root" archive --format=tar.gz HEAD | tee "$fixture_root/input/source.tar.gz" > /dev/null +docker image save "$manager_image" "$proxy_image" | tee "$fixture_root/input/images.tar" > /dev/null +cp "$tooling_root/guest.sh" "$fixture_root/input/guest.sh" +install -m 600 "$source_token_file" "$fixture_root/input/source-token" +fixture_commit=$(git -C "$repo_root" rev-parse HEAD) +python3 - "$fixture_root" "$fixture_authorization" "$fixture_commit" "$manager_image" "$proxy_image" "$fixture_scenario" "$fixture_accelerator" "$fixture_image_store" "$fixture_mutation_control" <<'PY' +import hashlib +import json +import pathlib +import sys + +root = pathlib.Path(sys.argv[1]) +inputs = root / "input" +manifest = dict(zip(("authorization_id", "commit", "manager_image", "proxy_image", "scenario", "accelerator", "image_store", "mutation_control"), sys.argv[2:])) +manifest["schema"] = "auto-coder-recovery-vm-input-v1" +manifest["ready_source_sha"] = "e742338b3f311e6d30a22a8f5341b8072513d813" +manifest["inputs_sha256"] = {} +for name in ("source.tar.gz", "images.tar", "guest.sh", "docker-daemon.json"): + with (inputs / name).open("rb") as stream: + manifest["inputs_sha256"][name] = hashlib.file_digest(stream, "sha256").hexdigest() +(inputs / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") +(inputs / "meta-data").write_text("instance-id: " + manifest["authorization_id"] + "\nlocal-hostname: ac-recovery-fixture\n") +PY +tee "$fixture_root/input/user-data" >/dev/null <<'YAML' +#cloud-config +users: [] +disable_root: true +ssh_pwauth: false +runcmd: + - [mkdir, -p, /mnt/fixture-input, /mnt/fixture-output] + - [mount, -t, 9p, -o, "trans=virtio,version=9p2000.L,ro", fixture-input, /mnt/fixture-input] + - [mount, -t, 9p, -o, "trans=virtio,version=9p2000.L", fixture-output, /mnt/fixture-output] + - [bash, /mnt/fixture-input/guest.sh] +power_state: + mode: poweroff + delay: now + timeout: 30 + condition: true +YAML +tee "$fixture_root/compose.env" >/dev/null <&2 + exit 2 +fi +if [[ -e /disk/guest.qcow2 ]]; then + echo 'A used guest disk cannot be reused as fresh recovery evidence.' >&2 + exit 2 +fi +accelerator=$(python3 -c 'import json; print(json.load(open("/input/manifest.json"))["accelerator"])') +case "$accelerator" in + kvm) acceleration=(-accel kvm -cpu host) ;; + tcg) acceleration=(-accel tcg,thread=multi -cpu max) ;; + *) echo 'The prepared guest accelerator is unsupported.' >&2; exit 2 ;; +esac +cloud-localds /disk/seed.img /input/user-data /input/meta-data +qemu-img create -f qcow2 -F qcow2 -b /opt/base.img /disk/guest.qcow2 32G +set +e +timeout --signal=TERM --kill-after=30s 120m qemu-system-x86_64 \ + "${acceleration[@]}" -smp 2 -m 4608 -nodefaults -no-reboot \ + -display none -serial stdio -monitor none \ + -drive file=/disk/guest.qcow2,if=virtio,format=qcow2 \ + -drive file=/disk/seed.img,if=virtio,format=raw,readonly=on \ + -netdev user,id=public-bootstrap -device virtio-net-pci,netdev=public-bootstrap \ + -virtfs local,path=/input,mount_tag=fixture-input,security_model=none,readonly=on \ + -virtfs local,path=/output,mount_tag=fixture-output,security_model=none \ + 2>&1 | tee /output/serial.log +qemu_status=${PIPESTATUS[0]} +set -e +if [[ "$qemu_status" != 0 || ! -f /output/guest-exit-code ]]; then + echo "The recovery guest did not complete; QEMU exit status: $qemu_status" >&2 + exit 1 +fi +result=$(cat /output/guest-exit-code) +if [[ "$result" != 0 ]]; then + echo "The recovery acceptance driver failed with status $result." >&2 + exit 1 +fi +python3 - <<'PY' +import hashlib +import json +from pathlib import Path + +result = json.loads(Path("/output/recovery-epoch-result.json").read_text()) +manifest = json.loads(Path("/input/manifest.json").read_text()) +if result.get("scenario") != manifest["scenario"]: + raise SystemExit("The recovery result does not match the prepared scenario") +control = manifest["mutation_control"] +if control == "none": + if result.get("schema") != "auto-coder-native-recovery-epoch-result-v1" or result.get("status") != "passed": + raise SystemExit("The native recovery result does not record complete acceptance") +elif control == "omit-post-consumption-verifier": + evidence = Path("/output/mutation-source.json").read_bytes() + if result.get("schema") != "auto-coder-native-recovery-mutation-control-result-v1" or result.get("status") != "mutation-detected" or result.get("control") != control or result.get("source_evidence_sha256") != hashlib.sha256(evidence).hexdigest(): + raise SystemExit("The separate mutation-control result lacks its exact source evidence") + oracle = result.get("oracle") + if not isinstance(oracle, dict) or oracle.get("name") != "post-receipt-grant-forbidden" or oracle.get("detected") is not True: + raise SystemExit("The mutation control did not detect the specified native regression") + queued = oracle.get("queued_grant") + if not isinstance(queued, dict) or queued.get("schema") != "actual-recovery-queued-grant-v1" or not isinstance(queued.get("grant"), dict): + raise SystemExit("The mutation control did not independently observe a real queued grant") + frame = json.dumps(queued["grant"], sort_keys=True, separators=(",", ":")).encode() + b"\n" + if queued.get("message_sha256") != hashlib.sha256(frame).hexdigest() or queued.get("bytes_observed") != len(frame) or queued.get("socket_flags_before") != queued.get("socket_flags_after"): + raise SystemExit("The mutation-control queued grant is incomplete or changed the socket") + for fact in ("peek_only", "socket_flags_unchanged", "bytes_still_queued", "receiver_stopped", "issuer_alive"): + if queued.get(fact) is not True: + raise SystemExit("The mutation-control observation omitted " + fact) +else: + raise SystemExit("The prepared recovery control is unsupported") +PY diff --git a/scripts/test_management_containment_closed_host.py b/scripts/test_management_containment_closed_host.py index 028a83827..0c691710e 100644 --- a/scripts/test_management_containment_closed_host.py +++ b/scripts/test_management_containment_closed_host.py @@ -462,45 +462,51 @@ class WatcherFixture: config: Path state: Path stdout: Path + unit: str = WATCHER_UNIT @classmethod - def start(cls, root: Path, manager: str, manager_image: str, proxy: str, proxy_image: str, caddyfile: Path) -> WatcherFixture: + def start(cls, root: Path, manager: str, manager_image: str, proxy: str, proxy_image: str, caddyfile: Path, *, unit: str = WATCHER_UNIT) -> WatcherFixture: + if re.fullmatch(r"auto-coder-management-proxy-watch(?:-[A-Za-z0-9_-]+)?\.service", unit) is None: + raise ValueError("fixture watcher unit must use a bounded management name") state = root / "watcher-state" state.mkdir(mode=0o700) config, output = root / "watcher-config.json", root / "watcher-output.log" _write_private(config, _json({"state_root": str(state), "manager_id": manager, "manager_image": manager_image, "proxy_id": proxy, "proxy_image": proxy_image, "config_sha256": hashlib.sha256(caddyfile.read_bytes()).hexdigest()})) _write_private(output, b"") - source = Path("/run/systemd/system") / WATCHER_UNIT + source = Path("/run/systemd/system") / unit command = f"{sys.executable} -I -B -m auto_coder.management.proxy_watch_cli --config {config}" if any(character in command for character in ("\n", "\r", "%", '"')) or " " in sys.executable or " " in str(root): raise ValueError("fixture watcher command contains unsupported systemd syntax") content = f"[Unit]\nDescription=Disposable original management proxy watcher\n[Service]\nType=simple\nExecStart={command}\nRestart=no\nStandardOutput=append:{output}\nStandardError=append:{output}\n".encode() _write_private(source, content) - result = cls(source, source.lstat().st_ino, hashlib.sha256(content).hexdigest(), config, state, output) + result = cls(source, source.lstat().st_ino, hashlib.sha256(content).hexdigest(), config, state, output, unit) try: _systemctl("daemon-reload") - _systemctl("start", WATCHER_UNIT) + _systemctl("start", unit) deadline = time.monotonic() + 30 while "ATTACHED\n" not in output.read_text(): if time.monotonic() >= deadline: raise AssertionError("the real installed watcher never observed Caddy attached") time.sleep(0.1) - pid = subprocess.run(("systemctl", "show", "--property=MainPID", "--value", WATCHER_UNIT), check=True, capture_output=True, text=True, timeout=10).stdout.strip() + pid = subprocess.run(("systemctl", "show", "--property=MainPID", "--value", unit), check=True, capture_output=True, text=True, timeout=10).stdout.strip() if not pid.isdigit() or int(pid) <= 1: raise AssertionError("the real watcher has no live unit-owned process") - _systemctl("stop", WATCHER_UNIT) - _systemctl("mask", WATCHER_UNIT) - observe_watcher_unit() + _systemctl("stop", unit) + _systemctl("mask", unit) + if unit == WATCHER_UNIT: + observe_watcher_unit() + elif not _systemd_unit(unit).inhibited: + raise AssertionError("the real additional watcher was not inhibited") return result except BaseException: result.cleanup() raise def cleanup(self) -> None: - _systemctl("stop", WATCHER_UNIT) - mask = Path("/etc/systemd/system") / WATCHER_UNIT + _systemctl("stop", self.unit) + mask = Path("/etc/systemd/system") / self.unit if mask.is_symlink() and os.readlink(mask) == "/dev/null": - _systemctl("unmask", WATCHER_UNIT) + _systemctl("unmask", self.unit) elif mask.exists() or mask.is_symlink(): raise AssertionError("fixture watcher mask was externally replaced") if self.source.lstat().st_ino != self.inode or hashlib.sha256(self.source.read_bytes()).hexdigest() != self.digest: diff --git a/scripts/test_management_recovery_epoch_host.py b/scripts/test_management_recovery_epoch_host.py new file mode 100644 index 000000000..47ccaf187 --- /dev/null +++ b/scripts/test_management_recovery_epoch_host.py @@ -0,0 +1,1398 @@ +"""Actual recovery ownership issuance on a separately authorized disposable VM. + +The driver uses original producers, the ordinary local Docker engine, systemd, +real namespace and flock custody, authenticated source publication and installed +READY verification. It never supplies an observation verdict or patches a +production function. The target stops at the explicit prelaunch custody mode; +consumer launch/recovery after this grant belongs to the separate consumer issue. +""" + +from __future__ import annotations + +import argparse +import fcntl +import hashlib +import json +import os +import re +import signal +import socket +import sqlite3 +import stat +import subprocess +import sys +import time +import uuid +from contextlib import ExitStack +from dataclasses import asdict, dataclass, replace +from pathlib import Path +from typing import Callable + +from scripts.recovery_epoch_containment_fixture import CapturedGeneration, capture_generation, enroll_recovery_scope, selected_file, start_watcher, target_from_container +from scripts.recovery_epoch_fault_tracer import BOUNDARY_SCENARIOS, PUBLICATIONS, KernelPublicationBarrier, PublicationObservation, parse_boundary +from scripts.recovery_epoch_mutation_control import CONTROL, CONTROL_SCHEMA, ORACLE, PostReceiptGrantDispatched, assert_post_receipt_refusal, validate_control_result, verify_source_evidence +from scripts.recovery_epoch_native_negatives import REQUIRED_NEGATIVE_CASES, REQUIRED_REPORT_CASES, NativeContainmentFailure, NativeNegativeContext, _replacement, assert_unreserved_epoch, run_native_negatives, verify_native_containment_reports +from scripts.recovery_epoch_native_peer import LATE_MUTATION_SCENARIOS, POST_RECEIPT_MUTATION_SCENARIOS, REQUIRED_PEER_CASES, mutate_after_publication, observe_post_receipt_dispatch, observe_queued_grant, pause_target_before_grant, run_native_peer_cases +from scripts.test_management_containment_closed_host import InhibitedStockRoutes, WatcherFixture, require_disposable_host +from scripts.test_management_containment_enrollment_host import DisposableSystemdRoutes, _private_directory, _write_private + +from auto_coder.logger_config import get_logger, setup_logger +from auto_coder.management.bootstrap_supervisor import BootstrapConfig +from auto_coder.management.bootstrap_upgrade import _source_authority +from auto_coder.management.containment_route_manifest import SystemdRoute, _systemd_catalog +from auto_coder.management.fresh_initialization import FreshInitializationIntent, claim_fresh_initialization, read_fresh_status +from auto_coder.management.operation_store import ManagementOperationStore +from auto_coder.management.operations import ActionParameter, ManagedTarget, OperationIntent, OperationPhase +from auto_coder.management.original_attempt_evidence import derive_lineage, observe, verify_lineage +from auto_coder.management.original_attempts import OriginalEvidenceUnavailable, canonical, read_original, source_identity +from auto_coder.management.recovery_containment_model import RECOVERY_ENROLLMENT_ROOT, RecoveryWatcher +from auto_coder.management.recovery_epoch_authorization import AUTHORIZATION_FILE, AUTHORIZATION_ROOT, AUTHORIZATION_SCHEMA, PEER_ROOT, PEER_SOCKET, EpochAuthorization, validate_uv_version +from auto_coder.management.recovery_epoch_consumer import GRANT_SCHEMA, ConsumerConfiguration, OriginalFence, TargetOwner, TransferredResource +from auto_coder.management.recovery_epoch_state import EPOCH_ROOT, EpochInventory +from auto_coder.management.self_update_handoff import runtime_for_receipt +from auto_coder.management.version_staging import ManagementPrepareIntent, ManagementReadyReceipt, ManagementRuntimeConfig, ManagementVersionStager +from auto_coder.operating_policy import MANAGEMENT_REPOSITORY +from tests.test_management_archived_cutover_container import Chain, Generation, diagnostic_log_lines + +logger = get_logger(__name__) +FIXTURE_ROOT = Path("/var/lib/auto-coder/recovery-epoch-fixtures") +RESULT_SCHEMA = "auto-coder-native-recovery-epoch-result-v1" +ORIGINAL_FIXTURE_DATABASES = (("state", "management-child.sqlite3"), ("state", "management-self-update.sqlite3"), ("state", "management-recovery.sqlite3"), ("journal", "management-operations.sqlite3")) +READINESS_OBSERVATION_DATABASE = ("state", "management-readiness.sqlite3") +ORIGINAL_NEGATIVE_CASES = frozenset({"original-journal-absence-refused", "changed-installed-ready-refused", "changed-target-management-config-refused"}) +LINEAGE_REPORT_CASES = frozenset({"forged-original-lineage-refused", "stale-original-lineage-refused", "cross-deployment-original-lineage-refused"}) +FACTORY_OBSERVATION_CASES = frozenset(case for case in REQUIRED_NEGATIVE_CASES if case.startswith("factory-")) +TASK_OBSERVATION_CASES = frozenset(case for case in REQUIRED_NEGATIVE_CASES | REQUIRED_REPORT_CASES if "nonleader" in case) +ACCEPTANCE_SCENARIOS = ("grant", *POST_RECEIPT_MUTATION_SCENARIOS, "lost-reply", "retarget-after-reservation", *BOUNDARY_SCENARIOS, *LATE_MUTATION_SCENARIOS) +SCENARIOS = (*ACCEPTANCE_SCENARIOS, "kill-after-intent", "kill-after-consume") +REQUIRED_CASES = frozenset( + { + "affirmative-original-genesis", + "native-successor-ready", + "native-failed-branch-retained", + "native-unresolved-successor-retained", + "missing-original-refused", + "genuine-installed-ready-in-target", + "independent-issuer-contender-refused", + "reservation-barrier-contender-refused", + "complete-native-epoch-granted", + "old-owner-and-controller-contenders-refused", + "issuer-death-ended-current-custody", + "consumed-epoch-replay-refused", + "originals-oauth-and-workers-preserved", + "no-consumer-child-or-admission", + } +) + + +def required_cases(scenario: str) -> frozenset[str]: + if scenario not in SCENARIOS: + raise ValueError("unsupported native recovery fixture scenario") + if scenario == "grant": + return REQUIRED_CASES | REQUIRED_NEGATIVE_CASES | REQUIRED_REPORT_CASES | REQUIRED_PEER_CASES | ORIGINAL_NEGATIVE_CASES | LINEAGE_REPORT_CASES + return (REQUIRED_CASES - {"complete-native-epoch-granted", "old-owner-and-controller-contenders-refused", "issuer-death-ended-current-custody", "consumed-epoch-replay-refused", "reservation-barrier-contender-refused"}) | {"native-" + scenario, "interrupted-epoch-replay-refused"} + + +def _fixture_directory() -> Path: + """Select a unique private leaf that fits the actual Linux socket paths.""" + root = FIXTURE_ROOT / uuid.uuid4().hex[:16] + sockets = (PEER_SOCKET, *(root / "deployment" / name / "guard" / "fresh-guard.sock" for name in ("A", "B", "C", "D"))) + if any(len(os.fsencode(path)) > 107 for path in sockets): + raise ValueError("the native fixture's original Unix socket pathname exceeds the Linux limit") + return root + + +class WorkerFixture(Chain): + """Reuse the real worker producer in roots unrelated to management.""" + + def __init__(self, root: Path, image: str): + self.root = root + self.image = image + self.worker_compose = None + self.worker_environment = None + self.worker_baseline = None + self.start_workers() + code = "from auto_coder.management.worker_network_client import WorkerNetworkClient,WorkerEndpointConfig;c=WorkerNetworkClient(WorkerEndpointConfig.environment('beta-agent-table-worker'));assert c.inspect_supervisor().result_code=='OK';assert c.observe_worker().result_code=='OK'" + self.docker("exec", self.worker_manager, "python", "-B", "-c", code) + + def close(self) -> None: + if self.worker_compose is not None: + self.compose("down", "--volumes", "--remove-orphans") + + def preserved(self, expected: dict[str, object], posts: dict[str, int], claims: str) -> None: + assert self.workers() == expected, "an unrelated real worker lifetime changed" + assert self.effect_posts() == posts, "an unrelated real worker received a new effect POST" + assert _worker_claims(self.root / "workers") == claims, "an unrelated worker claim changed" + + def effect_posts(self) -> dict[str, int]: + return {channel: len(path.read_text().splitlines()) if path.exists() else 0 for channel in ("beta", "release", "agent-table") for path in (self.root / "workers" / channel / "fixture-posts",)} + + +def _digest(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _snapshot(roots: tuple[Path, ...], *, semantic: bool = False) -> str: + """Fingerprint protected original bytes/objects without exporting their data.""" + entries = [] + for root in roots: + for path in (root, *sorted(root.rglob("*"))): + info = path.lstat() + if path.is_symlink(): + content = os.readlink(path) + elif path.is_file(): + content = _digest(path) + elif path.is_dir(): + content = "directory" + elif stat.S_ISSOCK(info.st_mode): + content = "socket" + else: + raise AssertionError("unsupported original fixture object") + if semantic: + entries.append((str(path), info.st_dev, info.st_ino, info.st_mode, info.st_uid, info.st_gid, info.st_size if path.is_file() else 0, info.st_mtime_ns if path.is_file() else 0, content)) + else: + entries.append((str(path), info.st_dev, info.st_ino, info.st_mode, info.st_uid, info.st_gid, info.st_size, info.st_mtime_ns, info.st_ctime_ns, content)) + return hashlib.sha256(canonical(entries)).hexdigest() + + +def _worker_claims(root: Path) -> str: + # The actual supervisor refreshes its observation timestamp and atomically + # replaces state.json during a GET. Compare its stable operation/ownership + # claims, not the expected volatile observation timestamp or file inode. + claims = [] + for channel in ("beta", "release", "agent-table"): + path = root / channel / "supervisor" / "state.json" + state = json.loads(path.read_bytes()) + worker = state.get("worker") + if state.get("schema_version") != 1 or not isinstance(worker, dict) or not worker.get("process_run_id"): + raise AssertionError("actual worker native ownership evidence is absent") + claims.append( + { + "path": str(path), + "supervisor": {name: state.get(name) for name in ("schema_version", "operations", "selected_runtime_id", "launch_intent", "stopped", "stop_verified", "last_worker")}, + "worker": {name: worker.get(name) for name in ("pid", "start_token", "process_run_id", "deployment_generation", "runtime_id", "started_at", "stop_requested", "ready", "ready_path")}, + } + ) + configurations = tuple(root / name for name in ("compose.yml", "beta-secret", "release-secret", "agent-table-secret")) + return hashlib.sha256(canonical({"claims": claims, "configuration": [(str(path), path.stat().st_dev, path.stat().st_ino, _digest(path)) for path in configurations]})).hexdigest() + + +def _wait(predicate: Callable[[], bool], description: str, *, timeout: float = 90, process: subprocess.Popen | None = None) -> None: + deadline = time.monotonic() + timeout + while not predicate(): + if process is not None and process.poll() is not None: + raise AssertionError(description + ": the original process exited with " + str(process.returncode)) + if time.monotonic() >= deadline: + raise AssertionError(description + ": the bounded native barrier timed out") + time.sleep(0.05) + + +def _command_failure(error: BaseException) -> dict[str, object] | None: + """Retain bounded child stderr/stdout without exporting command inputs.""" + if not isinstance(error, (subprocess.CalledProcessError, subprocess.TimeoutExpired)): + return None + lines = [] + for value in (error.output, error.stderr): + if isinstance(value, bytes): + value = value.decode(errors="replace") + if isinstance(value, str): + # The prefix keeps plain Compose diagnostics in the same existing + # credential/header/body filter used for all exported fixture logs. + lines.extend(diagnostic_log_lines("\n".join("RuntimeError: " + line for line in value.splitlines()[-60:]))) + return {"type": type(error).__name__, "returncode": getattr(error, "returncode", None), "diagnostics": lines[-60:]} + + +def _fixture_databases(label: str) -> tuple[tuple[str, str], ...]: + # A/B completed an authenticated readback; C deliberately did not. The + # separate readiness observation is never original lineage authority. + return ORIGINAL_FIXTURE_DATABASES + ((READINESS_OBSERVATION_DATABASE,) if label in {"A", "B"} else ()) + + +def _file_type(mode: int) -> str: + return "regular" if stat.S_ISREG(mode) else "symlink" if stat.S_ISLNK(mode) else "directory" if stat.S_ISDIR(mode) else "socket" if stat.S_ISSOCK(mode) else "other" + + +def _original_sidecar_diagnostics(root: Path, generations: tuple[Generation, ...]) -> list[dict[str, object]]: + """Inspect only fixed original sidecar metadata, without opening a database.""" + observations: list[dict[str, object]] = [] + for label in ("A", "B", "C", "D"): + selected = [generation for generation in generations if generation.name == label] + if not selected: + continue + for directory, database in _fixture_databases(label): + for suffix in ("-wal", "-shm", "-journal"): + row: dict[str, object] = {"generation": label, "database": database, "suffix": suffix} + try: + if len(selected) != 1 or selected[0].root != root / label or selected[0].root.resolve(strict=True) != root / label: + raise ValueError("the selected diagnostic generation has no exact original root") + parent = selected[0].path(directory) + if not stat.S_ISDIR(parent.lstat().st_mode): + raise ValueError("the selected diagnostic directory is not an original directory") + info = (parent / (database + suffix)).lstat() + except FileNotFoundError: + row["status"] = "absent" + except (OSError, ValueError) as error: + row.update(status="unavailable", error_type=type(error).__name__) + else: + row.update(status="observed", type=_file_type(info.st_mode), size=info.st_size, device=info.st_dev, inode=info.st_ino) + observations.append(row) + return observations + + +def _original_namespace_diagnostics(root: Path, generations: tuple[Generation, ...]) -> list[dict[str, object]]: + """Name at most sixteen unexpected SQLite entries in each original root.""" + observations: list[dict[str, object]] = [] + for label in ("A", "B", "C", "D"): + selected = [generation for generation in generations if generation.name == label] + if not selected: + continue + for directory in ("state", "journal"): + row: dict[str, object] = {"generation": label, "directory": directory} + try: + if len(selected) != 1 or selected[0].root != root / label or selected[0].root.resolve(strict=True) != root / label: + raise ValueError("the selected diagnostic generation has no exact original root") + parent = selected[0].path(directory) + info = parent.lstat() + row.update(type=_file_type(info.st_mode)) + if not stat.S_ISDIR(info.st_mode): + raise ValueError("the selected diagnostic directory is not an original directory") + known = {database + suffix for selected_directory, database in _fixture_databases(label) if selected_directory == directory for suffix in ("", "-wal", "-shm", "-journal")} + unexpected: list[dict[str, str]] = [] + for path in parent.glob("*.sqlite3*"): + if path.name in known: + continue + if len(unexpected) == 16: + row["truncated"] = True + break + unexpected.append({"basename": path.name, "type": _file_type(path.lstat().st_mode)}) + row.update(status="observed", unexpected=unexpected) + except FileNotFoundError: + row["status"] = "absent" + except (OSError, ValueError) as error: + row.update(status="unavailable", error_type=type(error).__name__) + observations.append(row) + return observations + + +def _record_failure_sidecars(result: dict[str, object], chain: Chain | None) -> None: + """Keep diagnostic failures from replacing the original native failure.""" + if chain is None: + return + for key, inspect in (("original_database_sidecars", _original_sidecar_diagnostics), ("original_database_namespaces", _original_namespace_diagnostics)): + try: + rows = inspect(chain.root, tuple(chain.generations)) + result[key] = rows + for row in rows: + logger.error("Native {} metadata: {}", key, canonical(row).decode()) + except Exception as error: + result[key] = [{"status": "unavailable", "error_type": type(error).__name__}] + + +def _record_failure_catalog(result: dict[str, object], enrollment_id: str | None) -> None: + """Record a bounded post-failure catalog delta without changing approval.""" + if enrollment_id is None: + return + try: + if not re.fullmatch(r"[A-Za-z0-9_.:-]{1,128}", enrollment_id): + raise ValueError("invalid diagnostic enrollment selector") + encoded, _ = read_original(RECOVERY_ENROLLMENT_ROOT / (enrollment_id + ".routes.json")) + if len(encoded) > 1024 * 1024: + raise ValueError("oversized diagnostic route manifest") + manifest = json.loads(encoded) + routes = manifest.get("systemd_routes") + if manifest.get("enrollment_id") != enrollment_id or not isinstance(routes, list) or not 1 <= len(routes) <= 1024: + raise ValueError("diagnostic route selection differs") + expected = {SystemdRoute.parse(row).unit for row in routes} + if len(expected) != len(routes): + raise ValueError("duplicate diagnostic route selection") + current = set(_systemd_catalog()) + if any(len(unit) > 256 for unit in expected | current): + raise ValueError("oversized diagnostic unit name") + added, removed = sorted(current - expected), sorted(expected - current) + observed = {"status": "observed-after-failure", "approved_count": len(expected), "observed_count": len(current), "added_count": len(added), "removed_count": len(removed), "added": added[:32], "removed": removed[:32], "truncated": len(added) > 32 or len(removed) > 32} + result["systemd_catalog_after_failure"] = observed + logger.error("Native systemd catalog after failure: {}", canonical(observed).decode()) + except Exception as error: + result["systemd_catalog_after_failure"] = {"status": "unavailable", "error_type": type(error).__name__} + + +def _checkpoint_stopped(generation: Generation) -> None: + """The stopped fixture producer settles its own stores before inspection.""" + for directory in (generation.path("state"), generation.path("journal")): + for database in directory.glob("*.sqlite3"): + Chain.checkpoint(database) + + +def _finalize_original_stores(chain: Chain, captures: tuple[CapturedGeneration, ...]) -> None: + """Finish known stopped fixture producers before any recovery baseline.""" + if set(os.listdir(EPOCH_ROOT)) != {"epoch.lock"} or AUTHORIZATION_FILE.exists() or AUTHORIZATION_FILE.is_symlink() or (chain.root / "target").exists(): + raise AssertionError("original fixture finalization is only permitted before recovery preparation") + if chain.root != chain.root.resolve(strict=True) or len(captures) != 3 or {captured.original.generation_id for captured in captures} != {"A", "B", "C"}: + raise AssertionError("only the three explicitly captured original generations may be finalized") + if any(process.poll() != 0 for process in chain.processes): + raise AssertionError("an original cutover reader has not completed before finalization") + originals: list[tuple[Path, bytes, os.stat_result, bool]] = [] + for captured in captures: + original = captured.original + selected = [generation for generation in chain.generations if generation.name == original.generation_id] + if len(selected) != 1 or original.state != "started" or original.instance_id != original.generation_id: + raise AssertionError("the selected original fixture generation is missing or ambiguous") + generation = selected[0] + if generation.root != chain.root / original.generation_id or (generation.container, generation.proxy) != (original.management_id, original.proxy_id): + raise AssertionError("the selected original fixture differs from its captured Docker objects") + for identifier in (original.management_id, original.proxy_id): + state = json.loads(chain.docker("inspect", "--format", "{{json .State}}", identifier)) + if state.get("Status") != "exited" or state.get("Running") is not False or state.get("Pid") != 0 or state.get("Restarting") is not False: + raise AssertionError("an original manager or proxy is not concretely stopped before finalization") + databases = _fixture_databases(generation.name) + for directory in ("state", "journal"): + parent = generation.path(directory) + known = {database + suffix for selected_directory, database in databases if selected_directory == directory for suffix in ("", "-wal", "-shm", "-journal")} + if not stat.S_ISDIR(parent.lstat().st_mode) or {path.name for path in parent.glob("*.sqlite3*")} - known: + raise AssertionError("original fixture finalization found an unknown store or directory") + for directory, name in databases: + database = generation.path(directory) / name + encoded, _ = read_original(database) + before = database.lstat() + checkpoint = (directory, name) in ORIGINAL_FIXTURE_DATABASES + for suffix in ("-wal", "-shm", "-journal"): + sidecar = database.with_name(name + suffix) + try: + info = sidecar.lstat() + except FileNotFoundError: + continue + if not checkpoint: + raise AssertionError("an original readiness observation still has a transaction sidecar") + if not stat.S_ISREG(info.st_mode) or info.st_uid != os.geteuid() or stat.S_IMODE(info.st_mode) != 0o600 or info.st_nlink != 1: + raise AssertionError("an original fixture sidecar is not its private regular object") + if suffix != "-shm" and info.st_size: + raise AssertionError("an original fixture still has nonempty transaction state") + originals.append((database, encoded, before, checkpoint)) + # A later archived read may have recreated SQLite's derived SHM index + # after an earlier producer checkpoint. Every actual reader is now gone; + # the existing checkpoint may close that index, never apply new records. + for database, _, _, checkpoint in originals: + if checkpoint: + Chain.checkpoint(database) + # The A/B readiness stores use DELETE mode and need no settlement. Keep + # their observation bytes and original identities just like every store. + for database, encoded, before, _ in originals: + current, _ = read_original(database) + after = database.lstat() + fields = ("st_dev", "st_ino", "st_ctime_ns", "st_mtime_ns", "st_mode", "st_uid", "st_gid", "st_nlink", "st_size") + if current != encoded or any(getattr(before, field) != getattr(after, field) for field in fields): + raise AssertionError("finalizing a stopped fixture changed original database bytes or identity") + if any(database.with_name(database.name + suffix).exists() or database.with_name(database.name + suffix).is_symlink() for suffix in ("-wal", "-shm", "-journal")): + raise AssertionError("original fixture sidecars remain after all tracked readers completed") + logger.info("Finalized twelve unchanged original A/B/C stores and preserved both A/B readiness observations after all cutover readers stopped, before recovery preparation") + + +def _no_target_child(target: Generation, *, running: bool) -> None: + assert not set(os.listdir(target.path("state"))) - {".supervisor-owner.lock", "logs"}, "prelaunch target created native consumer state" + assert not os.listdir(target.path("journal")), "prelaunch target admitted an operation" + if running: + state = json.loads(Chain.docker("inspect", "--format", "{{json .State}}", target.container)) + assert state["Running"] is True and state["Pid"] > 1 + pid = state["Pid"] + assert not Path(f"/proc/{pid}/task/{pid}/children").read_text().strip(), "prelaunch target has a child process" + + +def _require_target_absent(target_id: str) -> None: + """Prove exact object absence in the actual complete local inventory.""" + assert re.fullmatch(r"[0-9a-f]{64}", target_id), "the selected target must be an exact Docker ID" + identifiers = Chain.docker("ps", "--all", "--quiet", "--no-trunc").splitlines() + assert all(re.fullmatch(r"[0-9a-f]{64}", value) for value in identifiers) and len(set(identifiers)) == len(identifiers), "the complete Docker inventory is malformed" + assert target_id not in identifiers, "the deliberately removed target was recreated or remains present" + + +def _native_refused_branch(chain: Chain, prior: Generation) -> Generation: + candidate = chain.prepare("D", prior) + assert candidate.intent is not None and candidate.proxy is not None + selected = FreshInitializationIntent(**json.loads(candidate.intent.read_bytes())) + bad = replace(selected, oauth_inode=selected.oauth_inode + 1) + # This is the real original admission producer. Its failure is retained in + # the prospective history; no native store/owner or host grant is fabricated. + try: + claim_fresh_initialization(candidate.path("state"), bad) + except ValueError as error: + assert str(error) == "retained OAuth root identity differs" + else: + raise AssertionError("the original producer accepted the mismatched OAuth inode") + assert not os.listdir(candidate.path("state")) and not os.listdir(candidate.path("journal")) + for identifier in (candidate.container, candidate.proxy): + state = json.loads(chain.docker("inspect", "--format", "{{json .State}}", identifier)) + assert state["Status"] == "created" and state["Pid"] == 0 and state["StartedAt"].startswith("0001-") + # Only these never-started physical objects are removed. Their original + # attempt/terminal boundaries and exact empty roots remain authoritative. + chain.docker("rm", candidate.proxy, candidate.container) + return candidate + + +def _unresolved_successor(chain: Chain, prior: Generation) -> Generation: + candidate = chain.prepare("C", prior) + host = chain.host(candidate) + chain.status(host, "ancestors-fenced-awaiting-manual-stop") + chain.stop(prior) + _checkpoint_stopped(prior) + chain.status(host, "awaiting-replacement") + chain.docker("start", candidate.container) + out, err = host.communicate(timeout=90) + assert host.returncode == 0, (out, err) + chain.ensure_proxy(candidate) + + def launched() -> bool: + try: + return read_fresh_status(candidate.path("state"), host=True)["status"] == "launched" and chain.child(candidate)["phase"] == "running" + except (OSError, ValueError, sqlite3.Error, TypeError, KeyError): + return False + + _wait(launched, "actual unresolved first launch") + # Withholding the authenticated readiness request leaves the genuine + # native outcome unresolved. No outcome file or SQLite row is rewritten. + store = ManagementOperationStore(candidate.path("journal")) + intent = OperationIntent(ManagedTarget(candidate.name, "management", MANAGEMENT_REPOSITORY), "management-service-self-update", (ActionParameter("source_sha", "a" * 40),), "native-retained-unknown") + admitted = store.admit(intent, chain.authority).record + assert admitted is not None + changed = store.transition(admitted.operation_id, 0, OperationPhase.NEEDS_ATTENTION, safe_error_category="UNKNOWN:fixture-effect-not-retried") + assert changed.record is not None + return candidate + + +def _check_lineage(chain: Chain, captures: tuple[CapturedGeneration, ...]) -> dict[str, str]: + assert chain.history_binding is not None + with ExitStack() as stack: + descriptors = [] + for captured in captures: + assert captured.original.owner is not None + stream = stack.enter_context(Path(captured.original.owner.path).open("rb")) + fcntl.flock(stream.fileno(), fcntl.LOCK_SH | fcntl.LOCK_NB) + descriptors.append(stream.fileno()) + history = stack.enter_context(observe(chain.history_binding)) + complete = derive_lineage(chain.history_binding, history, tuple(descriptors)) + assert not complete.unavailable, complete.unavailable + assert verify_lineage(complete, chain.history_binding, history, tuple(descriptors)) == complete + try: + verify_lineage(replace(complete, generations=complete.generations[:-1]), chain.history_binding, history, tuple(descriptors)) + except OriginalEvidenceUnavailable: + pass + else: + raise AssertionError("a truncated original lineage was accepted as complete") + foreign = replace(chain.history_binding, deployment="foreign-deployment-" + uuid.uuid4().hex, history=str(uuid.uuid4())) + try: + verify_lineage(replace(complete, binding=foreign), chain.history_binding, history, tuple(descriptors)) + except OriginalEvidenceUnavailable: + pass + else: + raise AssertionError("a lineage report from another deployment/history was accepted") + outcomes = {item.instance_id: item.outcome for item in complete.generations} + assert set(outcomes) == {"A", "B", "C", "D"} + assert outcomes["B"] == "succeeded" and outcomes["C"] in {"launched", "unknown"} and outcomes["D"] == "never-created" + # A true missing original is distinct from a captured unknown. Restore + # only this disposable original pathname before taking the baseline. + database = chain.generations[0].path("journal") / "management-operations.sqlite3" + hidden = database.with_name("temporarily-absent-original") + database.rename(hidden) + try: + missing = derive_lineage(chain.history_binding, history, tuple(descriptors)) + assert missing.unavailable and any(database.name in reason for reason in missing.unavailable) + try: + verify_lineage(complete, chain.history_binding, history, tuple(descriptors)) + except OriginalEvidenceUnavailable: + pass + else: + raise AssertionError("a saved positive lineage survived a missing original") + finally: + hidden.rename(database) + restored = derive_lineage(chain.history_binding, history, tuple(descriptors)) + assert not restored.unavailable and {item.instance_id: item.outcome for item in restored.generations} == outcomes + return outcomes + + +@dataclass(frozen=True) +class PreparedTarget: + generation: Generation + receipt: ManagementReadyReceipt + python: Path + consumer: Path + request_id: str + preparation_id: str + enrollment_id: str + published_port: int + + +def _target( + chain: Chain, + captures: tuple[CapturedGeneration, ...], + watchers: tuple[RecoveryWatcher, ...], + *, + authorization_id: str, + source_repository: Path, + source_token_file: Path, + source_sha: str, + uv: Path, + uv_version: str, +) -> PreparedTarget: + root = chain.root / "target" + root.mkdir(mode=0o700) + for name in ("state", "journal", "bootstrap"): + (root / name).mkdir(mode=0o700) + _write_private(root / "state" / ".supervisor-owner.lock", b"") + # Preserve the exact original OAuth setup, including its live grant table. + management = root / "bootstrap" / "management.yaml" + _write_private(management, (chain.root / "A" / "bootstrap" / "management.yaml").read_bytes()) + versions = root / "versions" + preparation = "recovery-" + uuid.uuid4().hex + configuration = ManagementRuntimeConfig("recovery-target", MANAGEMENT_REPOSITORY, source_repository, versions, Path(sys.executable), uv, uv_version, management_config=management, source_authority=_source_authority(source_token_file)) + reservation = ManagementVersionStager(configuration) + reservation._validate_boundaries() + reservation._reserve_root() + environment = dict(os.environ, UV_PYTHON_INSTALL_DIR=str(versions / ".python"), PYTHONDONTWRITEBYTECODE="1") + subprocess.run([str(uv), "python", "install", "3.12", "--no-bin"], check=True, env=environment, timeout=300) + python = Path(subprocess.check_output([str(uv), "python", "find", "--managed-python", "--system", "3.12"], env=environment, text=True, timeout=30).strip()).resolve(strict=True) + assert python.is_relative_to(versions.resolve()), "portable Python escaped the same-path version root" + staging = ManagementVersionStager(replace(configuration, python_executable=python)) + receipt = staging.prepare(ManagementPrepareIntent(source_sha, preparation)).receipt + assert staging.lookup(Path(receipt.environment_path).parent / "MANAGEMENT_READY.json") == receipt + bootstrap = BootstrapConfig( + "recovery-target", + Path("/management-child-state"), + Path("/management-oauth"), + Path("/management-operations"), + chain.origin + "/mcp", + receipt.management_runtime_id, + (), + crash_budget=0, + management_version_root=versions, + management_config=Path("/bootstrap/management.yaml"), + verification_timeout=1800, + recovery_epoch_config=Path("/bootstrap/recovery-epoch.json"), + ) + runtime = runtime_for_receipt(bootstrap, receipt) + raw = { + "instance_id": bootstrap.instance_id, + "state_root": str(bootstrap.state_root), + "oauth_root": str(bootstrap.oauth_root), + "journal_root": str(bootstrap.journal_root), + "listener_url": bootstrap.listener_url, + "initial_runtime": receipt.management_runtime_id, + "runtimes": [{**asdict(runtime), "environment": dict(runtime.environment)}], + "crash_budget": 0, + "management_version_root": str(versions), + "management_config": str(bootstrap.management_config), + "verification_timeout": 1800, + "recovery_epoch_config": "/bootstrap/recovery-epoch.json", + } + config = root / "bootstrap" / "config.json" + _write_private(config, json.dumps(raw, sort_keys=True, separators=(",", ":"), default=str).encode()) + BootstrapConfig.read(config) + consumer = root / "bootstrap" / "recovery-epoch.json" + _write_private(consumer, b"") + assert chain.history_binding is not None + mounts = [ + (root / "state", "/management-child-state", True), + (root / "journal", "/management-operations", True), + (chain.oauth, "/management-oauth", True), + (config, "/bootstrap/config.json", False), + (management, "/bootstrap/management.yaml", False), + (consumer, "/bootstrap/recovery-epoch.json", False), + (versions, str(versions), False), + (PEER_ROOT, "/management-recovery", False), + (Path(chain.history_binding.root) / "history.lock", "/management-recovery-history/history.lock", False), + ] + originals = tuple(sorted((item.original for item in captures), key=lambda item: item.generation_id)) + original_fences = [] + transfers = [] + for index, original in enumerate(originals): + assert original.owner is not None and original.namespace_pin is not None + destination = f"/management-recovery-originals/{index}/owner.lock" + mounts.append((Path(original.owner.path), destination, False)) + original_fences.append(OriginalFence(f"owner-{index}", destination, original.owner.device, original.owner.inode)) + transfers.append(TransferredResource(f"namespace-{index}", "network-namespace", original.namespace_pin.device, original.namespace_pin.inode)) + history = (Path(chain.history_binding.root) / "history.lock").stat() + original_fences.insert(0, OriginalFence("history", "/management-recovery-history/history.lock", history.st_dev, history.st_ino)) + for index, watcher in enumerate(sorted(watchers, key=lambda item: item.unit)): + transfers.append(TransferredResource(f"controller-{index}", "transferred-ex", watcher.lock.device, watcher.lock.inode)) + epoch = (EPOCH_ROOT / "epoch.lock").stat() + transfers.append(TransferredResource("epoch", "transferred-ex", epoch.st_dev, epoch.st_ino)) + name = "recovery-target-" + uuid.uuid4().hex + chain.names.append(name) + with socket.socket() as reservation: + reservation.bind(("127.0.0.1", 0)) + published_port = reservation.getsockname()[1] + if published_port == chain.port: + raise AssertionError("new target endpoint must differ from every old endpoint") + command = ["create", "--name", name, "--read-only", "--workdir", "/workspace", "--network", "bridge", "--publish", f"127.0.0.1:{published_port}:8080", "--env", "PYTHONDONTWRITEBYTECODE=1"] + for source, destination, writable in mounts: + command.extend(("--mount", f"type=bind,src={source},dst={destination}" + ("" if writable else ",readonly"))) + command.extend(("--entrypoint", "python", chain.image, "-m", "auto_coder.management.bootstrap_supervisor", "--config", "/bootstrap/config.json", "--recovery-epoch")) + identifier = chain.docker(*command) + generation = Generation("recovery-target", root, identifier, mounts) + socket_info, owner = PEER_ROOT.stat(), (root / "state" / ".supervisor-owner.lock").stat() + request_id = "request-" + uuid.uuid4().hex + selected = ConsumerConfiguration(1, request_id, authorization_id, identifier, 0, socket_info.st_dev, socket_info.st_ino, tuple(original_fences), TargetOwner("target-owner", owner.st_dev, owner.st_ino), tuple(transfers)) + with consumer.open("wb") as stream: + stream.write(canonical(asdict(selected))) + stream.flush() + os.fsync(stream.fileno()) + return PreparedTarget(generation, receipt, python, consumer, request_id, preparation, "enrollment-" + uuid.uuid4().hex, published_port) + + +def _install_authorization(chain: Chain, target: PreparedTarget, authorization_id: str, source_repository: Path, source_token_file: Path, source_sha: str, uv: Path, uv_version: str) -> None: + assert chain.history_binding is not None + socket_info, root, lock = PEER_ROOT.stat(), EPOCH_ROOT.stat(), (EPOCH_ROOT / "epoch.lock").stat() + authorization = EpochAuthorization( + AUTHORIZATION_SCHEMA, + chain.history_binding.deployment, + authorization_id, + target.request_id, + target.preparation_id, + target.generation.container, + target.enrollment_id, + chain.history_binding, + selected_file(target.consumer), + selected_file(Path(target.receipt.environment_path).parent / "MANAGEMENT_READY.json"), + str(source_repository), + str(source_token_file), + str(target.python), + str(uv), + uv_version, + source_sha, + socket_info.st_dev, + socket_info.st_ino, + ) + _write_private(AUTHORIZATION_FILE, canonical(asdict(authorization))) + inventory = EpochInventory( + 2, + str(EPOCH_ROOT), + root.st_dev, + root.st_ino, + lock.st_dev, + lock.st_ino, + authorization.deployment_id, + authorization_id, + (target.enrollment_id,), + target.request_id, + target.preparation_id, + target.generation.container, + source_identity().removeprefix("sha256:"), + _digest(target.generation.path("bootstrap") / "config.json"), + _digest(AUTHORIZATION_FILE), + ) + _write_private(EPOCH_ROOT / "inventory.json", canonical(asdict(inventory))) + + +def inspect_durable_result(root: Path, *, issuer_log: str, consumer_log: str) -> dict[str, object]: + """Check the native successful run's four original records and actual logs.""" + inventory = (root / "inventory.json").read_bytes() + inventory_sha = hashlib.sha256(inventory).hexdigest() + binding = None + previous = "" + digests = {} + for sequence, phase in enumerate(("reserved", "intent", "consumed", "receipt")): + encoded = (root / (phase + ".json")).read_bytes() + record = json.loads(encoded) + head_bytes = (root / (phase + "-head.json")).read_bytes() + head = json.loads(head_bytes) + assert canonical(record) == encoded and canonical(head) == head_bytes + assert record["schema_version"] == 1 and record["phase"] == phase and record["sequence"] == sequence + assert record["previous"] == previous and record["inventory_sha256"] == inventory_sha + assert head == {"schema_version": 1, "phase": phase, "record_sha256": hashlib.sha256(encoded).hexdigest(), "previous": previous, "inventory_sha256": inventory_sha} + if binding is None: + binding = record["binding"] + assert record["binding"] == binding + digests[phase] = hashlib.sha256(encoded).hexdigest() + previous = hashlib.sha256(head_bytes).hexdigest() + assert binding is not None + evidence = json.loads(binding["evidence_json"]) + assert hashlib.sha256(binding["evidence_json"].encode()).hexdigest() == binding["evidence_sha256"] + assert not evidence["lineage"]["unavailable"] and evidence["lineage"]["genesis"] is not None + assert evidence["inspection"]["unresolved_targets"] >= 1 and evidence["inspection"]["launch_authority"] is False + assert "One consumed recovery epoch grant send completed" in issuer_log + assert "Recovery custody acknowledged" in consumer_log + return { + "record_sha256": digests, + "evidence_sha256": binding["evidence_sha256"], + "inventory_sha256": inventory_sha, + "historical_keys": evidence["inspection"]["historical_keys"], + "unresolved_targets": evidence["inspection"]["unresolved_targets"], + "source_sha": evidence["target"]["receipt"]["source_sha"], + } + + +def _observation_cases(value: dict[str, object], name: str, schema: str, expected: frozenset[str]) -> dict[str, dict[str, object]]: + document = value.get(name) + assert isinstance(document, dict) and document.get("schema") == schema + rows = document.get("cases") + assert isinstance(rows, list) and len(rows) == len(expected) and all(isinstance(item, dict) for item in rows) + assert {item.get("case") for item in rows} == expected + return {item["case"]: item for item in rows} + + +def validate_result(value: dict[str, object]) -> None: + """The CI oracle requires every native case and rejects partial acceptance.""" + assert value.get("schema") == RESULT_SCHEMA and value.get("status") == "passed" + scenario = value.get("scenario", "grant") + assert set(value.get("cases", ())) == required_cases(scenario) + assert value.get("no_child_started") is True and value.get("preserved_originals") is True and value.get("preserved_workers") is True + assert value.get("original_outcomes", {}).get("D") == "never-created" + assert value.get("original_outcomes", {}).get("C") in {"launched", "unknown"} + assert isinstance(value.get("durable"), dict) + if scenario == "grant": + assert set(value["durable"]["record_sha256"]) == {"reserved", "intent", "consumed", "receipt"} + peer = _observation_cases(value, "peer_observations", "actual-recovery-native-peer-faults-v1", REQUIRED_PEER_CASES) + for item in peer.values(): + assert item.get("issuer_returncode") == 2 and item.get("epoch_unreserved") is True and item.get("issuer_endpoint_removed_by_original") is True + assert item.get("consumer_grant_accepted") is False and item.get("no_child_or_admission") is True + assert item.get("issuer_syscall", {}).get("bytes_observed", 0) > 0 + if item["case"] in {"queued-rights-without-receipt-refused", "received-rights-without-receipt-refused"}: + assert item.get("receiver_alive_until_issuer_refusal") is True and item.get("issuer_refused_before_receiver_termination") is True + factory = _observation_cases(value, "factory_override_observations", "actual-recovery-factory-overrides-v1", FACTORY_OBSERVATION_CASES) + for item in factory.values(): + assert item.get("returncode") == 2 and item.get("refusal") + assert re.fullmatch(r"[0-9a-f]{64}", item.get("alternate_inputs_sha256", "")) and re.fullmatch(r"[0-9a-f]{64}", item.get("epoch_sha256", "")) + tasks = _observation_cases(value, "task_observations", "actual-recovery-task-observations-v1", TASK_OBSERVATION_CASES) + assert tasks["escaped-nonleader-current-containment-refused"]["status"] == "unavailable" + exited = tasks["measured-exited-nonleader-current-containment-reverified"] + assert exited["status"] == "observed-current-recovery-containment" and exited["after_exit"] == exited["after_verification"] + assert exited["after_exit"]["pidfd_readable"] is True and exited["after_exit"]["nonleader_absent"] is True + assert exited["after_exit"]["leader"]["state"] == "Z" and exited["after_exit"]["leader"]["network_namespace"] is None + elif scenario in BOUNDARY_SCENARIOS: + publication, when = parse_boundary(scenario) + assert value["durable"]["publication"] == publication and value["durable"]["when"] == when + assert value["durable"]["consumer_grant_acknowledged"] is False + assert value["durable"]["file_fsync_observed"] is (when == "after") and value["durable"]["directory_fsync_observed"] is (when == "after") + elif scenario == "retarget-after-reservation": + assert value["durable"]["phase"] == "reserved" and value["durable"]["consumer_grant_acknowledged"] is False + assert value["durable"]["inventory_sha256"] != value["durable"]["retargeted_inventory_sha256"] + elif scenario in (*LATE_MUTATION_SCENARIOS, *POST_RECEIPT_MUTATION_SCENARIOS): + phase = "receipt" if scenario in POST_RECEIPT_MUTATION_SCENARIOS else "intent" + assert value["durable"]["phase"] == phase and value["durable"]["consumer_grant_acknowledged"] is False and value["durable"]["grant_send_completed"] is False + assert value["durable"]["mutation"]["scenario"] == scenario and value["durable"]["mutation"]["boundary"] == "after-" + phase + "-head" + assert value.get("preservation_baseline") == "after-deliberate-fixture-mutation" + assert value["durable"]["mutation"]["preservation_baseline"] == value["preservation_baseline"] + assert value["preserved_originals_sha256"] == value["durable"]["originals_after_injection_sha256"] + if scenario in POST_RECEIPT_MUTATION_SCENARIOS: + durable = value["durable"] + assert_post_receipt_refusal(durable["dispatch"]) + assert durable["dispatch"]["issuer_death_before_receiver_resumed"] is True and durable["replay_returncode"] == 2 + assert durable["retained_publications"] == list(PUBLICATIONS) + assert durable["epoch_sha256_before_resume"] == durable["epoch_sha256_after_refusal"] == durable["epoch_sha256_after_replay"] + boundary = durable["publication_boundary"] + assert (boundary["name"], boundary["when"], boundary["file_fsync_observed"], boundary["directory_fsync_observed"]) == ("receipt-head.json", "after", True, True) + assert all(durable["mutation"][name] == durable["dispatch"][name] for name in ("issuer_pid", "issuer_start_token")) + changed = durable["mutation"]["file_fault"] + assert tuple(changed["before"][:2]) == tuple(changed["after"][:2]) and changed["before"][-1] != changed["after"][-1] + if scenario == "late-target-loss": + target_fault = value["durable"]["mutation"]["target_fault"] + assert target_fault["pidfd_death_observed"] is True and target_fault["exact_docker_id_absent"] is True and target_fault["docker_rm_returncode"] == 0 + assert target_fault["other_docker_ids_preserved"] is True + elif scenario == "late-original-attempt": + attempt = value["durable"]["mutation"]["original_attempt"] + changed = value["durable"]["mutation"]["file_fault"] + assert (attempt["instance_id"], attempt["outcome"], attempt["terminal_kind"]) == ("D", "never-created", "never-created") + assert attempt["independent_lineage_verified"] is True and attempt["retained_intent_lineage_matched"] is True + assert attempt["original_file"]["path"] == changed["path"] and attempt["original_file"]["sha256"] == changed["before"][-1] + assert (attempt["original_file"]["device"], attempt["original_file"]["inode"]) == tuple(changed["before"][:2]) + else: + assert value["durable"]["phase"] == ("intent" if scenario == "kill-after-intent" else "consumed" if scenario == "kill-after-consume" else "receipt") + assert value["durable"]["consumer_grant_acknowledged"] is False + assert value["durable"]["grant_send_completed"] is (scenario == "lost-reply") + if scenario == "lost-reply": + assert value["durable"]["receiver_paused_before_send"] is True and value["durable"]["issuer_died_before_receiver_resumed"] is True + original = value["durable"]["native_observation"] + queued = value["durable"]["queued_grant"] + assert queued["schema"] == "actual-recovery-queued-grant-v1" and queued["grant_schema"] == GRANT_SCHEMA + assert all(queued.get(name) is True for name in ("peek_only", "socket_flags_unchanged", "bytes_still_queued", "receiver_stopped", "issuer_alive")) + assert queued["bytes_observed"] > 0 and re.fullmatch(r"[0-9a-f]{64}", queued["message_sha256"]) + packet = queued["grant"] + encoded = canonical(packet) + b"\n" + assert len(encoded) == queued["bytes_observed"] and hashlib.sha256(encoded).hexdigest() == queued["message_sha256"] + assert (packet["schema"], packet["target_id"], packet["host_pid"], packet["host_start_token"]) == (queued["grant_schema"], queued["target_id"], original["issuer_pid"], original["issuer_start_token"]) + assert queued["socket_inode"] > 0 and queued["socket_flags_before"] == queued["socket_flags_after"] + assert all(queued[name] == original[name] for name in ("target_pid", "target_start_token", "issuer_pid", "issuer_start_token")) + assert queued["peer_credentials"] == {"pid": original["issuer_pid"], "uid": 0, "gid": 0} + assert original["issuer_death_verified_before_receiver_resumed"] is True + + +def _selected_boundary(scenario: str) -> tuple[str, str]: + if scenario in BOUNDARY_SCENARIOS: + return parse_boundary(scenario) + if scenario in LATE_MUTATION_SCENARIOS or scenario == "kill-after-intent": + return "intent-head.json", "after" + if scenario in POST_RECEIPT_MUTATION_SCENARIOS: + return "receipt-head.json", "after" + if scenario == "kill-after-consume": + return "consumed-head.json", "after" + if scenario == "lost-reply": + return "receipt-head.json", "after" + if scenario in {"grant", "retarget-after-reservation"}: + return "reserved-head.json", "after" + raise ValueError("unsupported native recovery publication barrier") + + +def _lost_reply(context: NativeNegativeContext, barrier: KernelPublicationBarrier, issuer: subprocess.Popen, issuer_log: Path, consumer_log: Path, target: PreparedTarget) -> dict[str, object]: + """Observe successful delivery to the socket before the pinned issuer dies.""" + original_epoch = _snapshot((EPOCH_ROOT,)) + descriptor = os.pidfd_open(issuer.pid, 0) + try: + with pause_target_before_grant(context, issuer.pid) as observation: + assert "Recovery custody acknowledged" not in consumer_log.read_text() + barrier.release() + _wait(lambda: "One consumed recovery epoch grant send completed" in issuer_log.read_text(), "actual grant send while its receiver remains kernel-stopped", timeout=1800, process=issuer) + assert "Recovery custody acknowledged" not in consumer_log.read_text(), "the stopped receiver observed the grant" + assert _snapshot((EPOCH_ROOT,)) == original_epoch + queued = observe_queued_grant(context, issuer.pid, target_pid=observation["target_pid"], target_start_token=observation["target_start_token"], issuer_start_token=observation["issuer_start_token"]) + signal.pidfd_send_signal(descriptor, signal.SIGKILL) + assert issuer.wait(timeout=30) == -signal.SIGKILL + assert "Recovery custody acknowledged" not in consumer_log.read_text() + finally: + os.close(descriptor) + _wait(lambda: Chain.docker("inspect", "--format", "{{.State.Running}}", target.generation.container) == "false", "queued grant cannot survive the actual issuer lifetime", timeout=120) + assert Chain.docker("inspect", "--format", "{{.State.ExitCode}}", target.generation.container) == "1" + _no_target_child(target.generation, running=False) + assert "Recovery custody acknowledged" not in consumer_log.read_text() + assert _snapshot((EPOCH_ROOT,)) == original_epoch + replay = subprocess.run([sys.executable, "-B", "-m", "auto_coder.management.recovery_epoch"], capture_output=True, text=True, timeout=30) + assert replay.returncode == 2 and _snapshot((EPOCH_ROOT,)) == original_epoch, "a consumed epoch was reissued after its reply was lost" + return {"phase": "receipt", "consumer_grant_acknowledged": False, "grant_send_completed": True, "receiver_paused_before_send": True, "issuer_died_before_receiver_resumed": True, "native_observation": observation, "queued_grant": queued, "epoch_sha256": original_epoch} + + +def _publication_interruption(scenario: str, observed: PublicationObservation, issuer_log: Path, consumer_log: Path, target: PreparedTarget) -> dict[str, object]: + """Verify exactly the actual interrupted original publication prefix.""" + name, when = parse_boundary(scenario) + assert (observed.name, observed.when) == (name, when) + index = PUBLICATIONS.index(name) + (1 if when == "after" else 0) + assert set(os.listdir(EPOCH_ROOT)) == {"inventory.json", "epoch.lock", *PUBLICATIONS[:index]}, "issuer crossed the selected native publication boundary" + if when == "after": + published = EPOCH_ROOT / name + info = published.stat() + assert (observed.file_device, observed.file_inode, observed.file_sha256) == (info.st_dev, info.st_ino, _digest(published)) + assert observed.file_fsync_observed and observed.directory_fsync_observed + else: + assert observed.file_device is None and observed.file_inode is None and observed.file_sha256 is None + assert not observed.file_fsync_observed and not observed.directory_fsync_observed + _wait(lambda: Chain.docker("inspect", "--format", "{{.State.Running}}", target.generation.container) == "false", "native publication interruption ends target custody", timeout=90) + _no_target_child(target.generation, running=False) + assert "Recovery custody acknowledged" not in consumer_log.read_text() + assert "One consumed recovery epoch grant send completed" not in issuer_log.read_text() + before = _snapshot((EPOCH_ROOT,)) + replay = subprocess.run([sys.executable, "-B", "-m", "auto_coder.management.recovery_epoch"], capture_output=True, text=True, timeout=30) + assert replay.returncode == 2 and _snapshot((EPOCH_ROOT,)) == before, "interrupted publication or original endpoint was repaired/reissued" + return { + "publication": name, + "when": when, + "file_fsync_observed": observed.file_fsync_observed, + "directory_fsync_observed": observed.directory_fsync_observed, + "consumer_grant_acknowledged": False, + "grant_send_completed": False, + "published_names": list(PUBLICATIONS[:index]), + "native_observation": asdict(observed), + } + + +def _passed(result: dict[str, object], cases: list[str], preserved: str, worker_claims: str, worker_baseline: dict[str, object], worker_posts: dict[str, int]) -> None: + cases.extend(("originals-oauth-and-workers-preserved", "no-consumer-child-or-admission")) + result.update(status="passed", no_child_started=True, preserved_originals=True, preserved_workers=True, preserved_originals_sha256=preserved, preserved_worker_claims_sha256=worker_claims, worker_lifetimes=worker_baseline, worker_effect_posts=worker_posts) + validate_result(result) + logger.info("All {} native recovery ownership cases passed; no consumer child was launched", len(cases)) + + +def _issuer_refusal(context: NativeNegativeContext, target: PreparedTarget, label: str) -> None: + """Run the actual complete issuer, allowing no first publication on error.""" + assert_unreserved_epoch(context.epoch_root) + log_path = context.fixture_root / ("refusal-" + label + ".log") + assert Chain.docker("inspect", "--format", "{{.State.Running}}", target.generation.container) == "false" + process = None + started = False + with log_path.open("xb") as output: + try: + process = subprocess.Popen([sys.executable, "-B", "-m", "auto_coder.management.recovery_epoch"], stdout=output, stderr=subprocess.STDOUT, env=dict(os.environ, PYTHONDONTWRITEBYTECODE="1")) + _wait(lambda: process.poll() is not None or PEER_SOCKET.exists() and "Fixed recovery issuer endpoint is listening" in log_path.read_text(), "actual refused issuer startup " + label, timeout=180) + if process.poll() is None: + Chain.docker("start", target.generation.container) + started = True + _wait(lambda: process.poll() is not None or set(os.listdir(context.epoch_root)) != {"inventory.json", "epoch.lock"}, "actual prepublication refusal " + label, timeout=1800) + assert_unreserved_epoch(context.epoch_root) + assert process.wait(timeout=30) == 2, "the complete issuer did not refuse the injected original condition" + assert "One consumed recovery epoch grant send completed" not in log_path.read_text() + if started: + _wait(lambda: Chain.docker("inspect", "--format", "{{.State.Running}}", target.generation.container) == "false", "refused target exits without native state", timeout=120) + assert Chain.docker("inspect", "--format", "{{.State.ExitCode}}", target.generation.container) == "1" + consumer_log = target.generation.path("state") / "logs" / "recovery-epoch-bootstrap.log" + assert "Recovery custody acknowledged" not in consumer_log.read_text() + _no_target_child(target.generation, running=False) + finally: + if process is not None and process.poll() is None: + process.kill() + process.wait(timeout=30) + if started and Chain.docker("inspect", "--format", "{{.State.Running}}", target.generation.container) == "true": + Chain.docker("kill", target.generation.container) + assert_unreserved_epoch(context.epoch_root) + + +def _original_negatives(context: NativeNegativeContext, target: PreparedTarget, chain: Chain, preserved_snapshot: Callable[[], object]) -> list[str]: + directory = context.fixture_root / "native-original-negative-backups" + directory.mkdir(mode=0o700) + database = chain.generations[0].path("journal") / "management-operations.sqlite3" + saved = directory / "absent-original.sqlite3" + baseline = preserved_snapshot() + original = (database.stat().st_dev, database.stat().st_ino, _digest(database)) + database.rename(saved) + try: + injected = preserved_snapshot() + _issuer_refusal(context, target, "original-journal-absence-refused") + assert preserved_snapshot() == injected + finally: + assert_unreserved_epoch(context.epoch_root) + assert not database.exists() and (saved.stat().st_dev, saved.stat().st_ino, _digest(saved)) == original + saved.rename(database) + assert preserved_snapshot() == baseline + for label, path in (("changed-installed-ready-refused", Path(target.receipt.environment_path).parent / "MANAGEMENT_READY.json"), ("changed-target-management-config-refused", target.generation.path("bootstrap") / "management.yaml")): + baseline = preserved_snapshot() + with _replacement(context.epoch_root, path, directory / label, path.read_bytes() + b"\n"): + injected = preserved_snapshot() + _issuer_refusal(context, target, label) + assert preserved_snapshot() == injected + assert preserved_snapshot() == baseline + return sorted(ORIGINAL_NEGATIVE_CASES) + + +def _reserved_contender() -> None: + assert set(os.listdir(EPOCH_ROOT)) == {"inventory.json", "epoch.lock", "reserved.json", "reserved-head.json"} + before = _snapshot((EPOCH_ROOT,)) + contender = subprocess.run([sys.executable, "-B", "-m", "auto_coder.management.recovery_epoch"], capture_output=True, text=True, timeout=30) + assert contender.returncode == 2 and _snapshot((EPOCH_ROOT,)) == before, "a contender overlapped the actual original durable reservation" + + +def _retarget_reserved(barrier: KernelPublicationBarrier, issuer: subprocess.Popen, issuer_log: Path, consumer_log: Path, target: PreparedTarget) -> dict[str, object]: + inventory = EPOCH_ROOT / "inventory.json" + original_digest = _digest(inventory) + raw = json.loads(inventory.read_bytes()) + raw["preparation_id"] = "foreign-" + uuid.uuid4().hex + raw["target_id"] = ("f" if raw["target_id"] != "f" * 64 else "e") * 64 + with inventory.open("wb") as stream: + stream.write(canonical(raw)) + stream.flush() + os.fsync(stream.fileno()) + _reserved_contender() + injected = _snapshot((EPOCH_ROOT,)) + barrier.release() + issuer.wait(timeout=120) + assert issuer.returncode == 2 and _snapshot((EPOCH_ROOT,)) == injected + _wait(lambda: Chain.docker("inspect", "--format", "{{.State.Running}}", target.generation.container) == "false", "retargeted reservation ends current custody") + _no_target_child(target.generation, running=False) + assert "Recovery custody acknowledged" not in consumer_log.read_text() and "One consumed recovery epoch grant send completed" not in issuer_log.read_text() + replay = subprocess.run([sys.executable, "-B", "-m", "auto_coder.management.recovery_epoch"], capture_output=True, text=True, timeout=30) + assert replay.returncode == 2 and _snapshot((EPOCH_ROOT,)) == injected + return {"phase": "reserved", "consumer_grant_acknowledged": False, "inventory_sha256": original_digest, "retargeted_inventory_sha256": _digest(inventory), "retained_publications": ["reserved.json", "reserved-head.json"]} + + +def _late_mutation( + context: NativeNegativeContext, + scenario: str, + barrier: KernelPublicationBarrier, + issuer: subprocess.Popen, + issuer_log: Path, + consumer_log: Path, + target: PreparedTarget, + retained_roots: tuple[Path, ...], + check_workers: Callable[[], None], +) -> tuple[dict[str, object], str]: + """Require an actual late refusal while retaining the injected uncertainty.""" + original = _snapshot(retained_roots) + with mutate_after_publication(context, scenario, issuer.pid) as observation: + injected = _snapshot(retained_roots) + epoch = _snapshot((EPOCH_ROOT,)) + check_workers() + barrier.release() + assert issuer.wait(timeout=1800) == 2, "the actual issuer did not refuse the changed post-intent authority" + if scenario == "late-target-loss": + _require_target_absent(target.generation.container) + else: + _wait(lambda: Chain.docker("inspect", "--format", "{{.State.Running}}", target.generation.container) == "false", "late refusal ends the exact current consumer lifetime", timeout=120) + assert Chain.docker("inspect", "--format", "{{.State.ExitCode}}", target.generation.container) == "1" + _no_target_child(target.generation, running=False) + assert "Recovery custody acknowledged" not in consumer_log.read_text() and "One consumed recovery epoch grant send completed" not in issuer_log.read_text() + assert _snapshot((EPOCH_ROOT,)) == epoch, "late mutation reached consumption or rewrote original epoch evidence" + assert _snapshot(retained_roots) == injected, "production refusal changed the deliberately injected original state" + check_workers() + replay = subprocess.run([sys.executable, "-B", "-m", "auto_coder.management.recovery_epoch"], capture_output=True, text=True, timeout=30) + assert replay.returncode == 2 and _snapshot((EPOCH_ROOT,)) == epoch, "the uncertain original intent was repaired or reissued" + if scenario == "late-target-loss": + _require_target_absent(target.generation.container) + assert _snapshot(retained_roots) == injected + check_workers() + result = { + "phase": "intent", + "consumer_grant_acknowledged": False, + "grant_send_completed": False, + "mutation": observation, + "originals_before_injection_sha256": original, + "originals_after_injection_sha256": injected, + "epoch_sha256": epoch, + } + return result, injected + + +def _post_receipt_mutation( + context: NativeNegativeContext, + scenario: str, + barrier: KernelPublicationBarrier, + observed: PublicationObservation, + issuer: subprocess.Popen, + issuer_log: Path, + consumer_log: Path, + target: PreparedTarget, + retained_roots: tuple[Path, ...], + check_workers: Callable[[], None], + *, + mutation_control: bool, +) -> tuple[dict[str, object], str]: + """Require the same issuer to reject stale READY after its durable receipt.""" + assert (observed.name, observed.when, observed.file_fsync_observed, observed.directory_fsync_observed) == ("receipt-head.json", "after", True, True) + original = _snapshot(retained_roots) + detected = False + with mutate_after_publication(context, scenario, issuer.pid) as mutation: + injected = _snapshot(retained_roots) + epoch = _snapshot((EPOCH_ROOT,)) + assert set(os.listdir(EPOCH_ROOT)) == {"inventory.json", "epoch.lock", *PUBLICATIONS} + check_workers() + with observe_post_receipt_dispatch(context, issuer, barrier.release) as dispatch: + try: + assert_post_receipt_refusal(dispatch) + except PostReceiptGrantDispatched: + if not mutation_control: + raise + detected = True + if mutation_control and not detected: + raise AssertionError("the focused second-verifier mutation did not fail the actual no-grant oracle") + _wait(lambda: Chain.docker("inspect", "--format", "{{.State.Running}}", target.generation.container) == "false", "post-receipt refusal ends the original stopped receiver", timeout=120) + assert Chain.docker("inspect", "--format", "{{.State.ExitCode}}", target.generation.container) == "1" + _no_target_child(target.generation, running=False) + assert "Recovery custody acknowledged" not in consumer_log.read_text() + if not detected: + assert "One consumed recovery epoch grant send completed" not in issuer_log.read_text() + after_refusal = _snapshot((EPOCH_ROOT,)) + assert after_refusal == epoch and _snapshot(retained_roots) == injected + check_workers() + replay = subprocess.run([sys.executable, "-B", "-m", "auto_coder.management.recovery_epoch"], capture_output=True, text=True, timeout=30) + after_replay = _snapshot((EPOCH_ROOT,)) + assert replay.returncode == 2 and after_replay == epoch and _snapshot(retained_roots) == injected + check_workers() + result = { + "phase": "receipt", + "consumer_grant_acknowledged": False, + "grant_send_completed": detected, + "mutation": mutation, + "dispatch": dispatch, + "publication_boundary": asdict(observed), + "retained_publications": list(PUBLICATIONS), + "replay_returncode": replay.returncode, + "originals_before_injection_sha256": original, + "originals_after_injection_sha256": injected, + "epoch_sha256_before_resume": epoch, + "epoch_sha256_after_refusal": after_refusal, + "epoch_sha256_after_replay": after_replay, + } + return result, injected + + +def run_fixture(*, authorization_id: str, source_repository: Path, source_token_file: Path, source_sha: str, uv: Path, uv_version: str, output: Path, scenario: str = "grant", mutation_control_manifest: Path | None = None) -> dict[str, object]: + required_cases(scenario) + source_mutation = None + if mutation_control_manifest is not None: + if scenario not in POST_RECEIPT_MUTATION_SCENARIOS: + raise ValueError("the focused mutation control requires its exact post-receipt scenario") + source_mutation = verify_source_evidence(mutation_control_manifest, executing=True) + if not re.fullmatch(r"[0-9a-f]{40}", source_sha): + raise ValueError("an exact qualified source is required") + validate_uv_version(uv_version) + if any(not path.is_absolute() or path != path.resolve(strict=True) for path in (source_repository, source_token_file, uv)) or not output.is_absolute(): + raise ValueError("canonical absolute native fixture inputs are required") + for name in ("AUTOCODER_FRESH_TEST_BASE_IMAGE", "AUTOCODER_RECOVERY_CADDY_IMAGE"): + if re.fullmatch(r"sha256:[0-9a-f]{64}", os.environ.get(name, "")) is None: + raise ValueError("an exact already imported fixture image is required: " + name) + authorization = require_disposable_host(authorization_id) + root = _fixture_directory() + # No resource is provisioned until the original marker selected this VM. + os.umask(0o077) + for directory in (FIXTURE_ROOT, EPOCH_ROOT, PEER_ROOT, AUTHORIZATION_ROOT): + if directory.exists() or directory.is_symlink(): + raise FileExistsError("recovery fixture refuses a previous/partial root: " + str(directory)) + for parent in (Path("/var/lib/auto-coder"), Path("/etc/auto-coder")): + _private_directory(parent) + _private_directory(FIXTURE_ROOT) + root.mkdir(mode=0o700) + for directory in (EPOCH_ROOT, PEER_ROOT, AUTHORIZATION_ROOT): + _private_directory(directory) + _write_private(EPOCH_ROOT / "epoch.lock", b"") + setup_logger(stream=sys.stdout, log_file=str(root / "fixture.log")) + cases: list[str] = [] + result: dict[str, object] = {"schema": RESULT_SCHEMA, "status": "failed", "scenario": scenario, "cases": cases, "authorization_id": authorization_id, "source_identity": source_identity(), "no_child_started": False, "preserved_originals": False, "preserved_workers": False} + if source_mutation is not None: + result.update(schema=CONTROL_SCHEMA, control=CONTROL, source_mutation=source_mutation[0], source_evidence_sha256=source_mutation[1]) + chain = None + workers = None + stock = None + routes = None + watcher_fixtures: list[WatcherFixture] = [] + captures: list[CapturedGeneration] = [] + issuer = None + log_stream = None + target = None + barrier = None + try: + stock = InhibitedStockRoutes.install() + worker_root = root / "unrelated" + worker_root.mkdir(mode=0o700) + workers = WorkerFixture(worker_root, os.environ["AUTOCODER_FRESH_TEST_BASE_IMAGE"]) + worker_baseline, worker_posts = workers.workers(), workers.effect_posts() + worker_claims = _worker_claims(worker_root / "workers") + deployment_root = root / "deployment" + deployment_root.mkdir(mode=0o700) + chain = Chain(deployment_root, original_genesis=True) + original = chain.prepare("A") + assert chain.genesis is not None and chain.history_binding is not None + cases.append("affirmative-original-genesis") + chain.docker("start", original.container) + _wait(lambda: (original.path("state") / "original-genesis-first-launch.json").is_file(), "actual original first-launch boundary") + chain.authorize() + chain.ready(original, fresh=False) + chain.prove_child(original) + chain.seed(original) + captures.append(capture_generation(chain, original)) + fixture, watcher = start_watcher(original, captures[-1].original, first=True) + watcher_fixtures.append(fixture) + watchers = [watcher] + second = chain.prepare("B", original) + chain.start_cutover(original, second) + _checkpoint_stopped(original) + chain.seed(second) + captures.append(capture_generation(chain, second)) + fixture, watcher = start_watcher(second, captures[-1].original) + watcher_fixtures.append(fixture) + watchers.append(watcher) + cases.append("native-successor-ready") + _native_refused_branch(chain, second) + cases.append("native-failed-branch-retained") + third = _unresolved_successor(chain, second) + captures.append(capture_generation(chain, third)) + fixture, watcher = start_watcher(third, captures[-1].original) + watcher_fixtures.append(fixture) + watchers.append(watcher) + chain.stop(third) + _checkpoint_stopped(third) + _finalize_original_stores(chain, tuple(captures)) + for captured in captures: + assert captured.original.namespace_pin is not None + assert os.readlink(f"/proc/self/fd/{captured.namespace_fd}") == captured.original.namespace_pin.namespace + result["original_outcomes"] = _check_lineage(chain, tuple(captures)) + cases.extend(("native-unresolved-successor-retained", "missing-original-refused")) + target = _target(chain, tuple(captures), tuple(watchers), authorization_id=authorization_id, source_repository=source_repository, source_token_file=source_token_file, source_sha=source_sha, uv=uv, uv_version=uv_version) + routes = DisposableSystemdRoutes.install(fixture_id="ac-test-pr2829-" + uuid.uuid4().hex, manager_id=third.container) + routes.assert_inhibited() + stock.assert_inhibited() + selected_target = target_from_container(target.generation.container, bootstrap_config=target.generation.path("bootstrap") / "config.json", management_config=target.generation.path("bootstrap") / "management.yaml", published_port=target.published_port) + enrollment = enroll_recovery_scope(tuple(captures), selected_target, tuple(watchers), deployment_id=chain.history_binding.deployment, enrollment_id=target.enrollment_id, authorization=authorization) + _install_authorization(chain, target, authorization_id, source_repository, source_token_file, source_sha, uv, uv_version) + retained_roots = (Path(chain.history_binding.root), chain.oauth, *(path for generation in chain.generations for path in (generation.path("state"), generation.path("journal"), generation.path("bootstrap")))) + Chain.checkpoint(chain.oauth / "oauth.sqlite3") + preserved = _snapshot(retained_roots) + worker_before = (worker_baseline, worker_posts, worker_claims) + workers.preserved(*worker_before) + native_context = NativeNegativeContext(enrollment.enrollment, root, EPOCH_ROOT) + if scenario == "grant": + + def preservation() -> object: + workers.preserved(*worker_before) + return _snapshot(retained_roots, semantic=True), worker_claims, workers.effect_posts() + + cases.extend(LINEAGE_REPORT_CASES) + cases.extend(run_native_negatives(native_context, lambda label: _issuer_refusal(native_context, target, label), preservation)) + result["factory_override_observations"] = json.loads((root / "native-factory-override-observations.json").read_bytes()) + cases.extend(_original_negatives(native_context, target, chain, preservation)) + cases.extend(run_native_peer_cases(native_context, preservation)) + result["peer_observations"] = json.loads((root / "native-peer-observations.json").read_bytes()) + # Only the intentional reversible fixture injections changed path + # metadata. This fresh strict baseline precedes actual issuance. + preserved = _snapshot(retained_roots) + issuer_log = root / "issuer.log" + log_stream = issuer_log.open("xb") + issuer = subprocess.Popen([sys.executable, "-B", "-m", "auto_coder.management.recovery_epoch"], stdout=log_stream, stderr=subprocess.STDOUT, env=dict(os.environ, PYTHONDONTWRITEBYTECODE="1")) + _wait(lambda: PEER_SOCKET.exists() and "Fixed recovery issuer endpoint is listening" in issuer_log.read_text(), "actual issuer listening", process=issuer) + contender = subprocess.run([sys.executable, "-B", "-m", "auto_coder.management.recovery_epoch"], capture_output=True, text=True, timeout=30) + assert contender.returncode == 2 and not (EPOCH_ROOT / "reserved.json").exists(), "a second issuer acquired the same current epoch" + cases.append("independent-issuer-contender-refused") + # The actual ptrace stop lets the target finish its interpreter probe + # before T samples current tasks, then observes the chosen publication. + publication, when = _selected_boundary(scenario) + barrier = KernelPublicationBarrier(issuer, root=EPOCH_ROOT, name=publication, when=when) + barrier.start() + chain.docker("start", target.generation.container) + probe = json.loads(chain.docker("exec", target.generation.container, target.receipt.interpreter_path, "-I", "-B", "-m", "auto_coder.build_provenance", timeout=60).splitlines()[-1]) + assert probe["source_revision"]["available"] is True and probe["source_revision"]["value"] == source_sha + assert probe["distribution_version"]["available"] is True and probe["distribution_version"]["value"] == target.receipt.distribution_version + _wait(lambda: chain.docker("inspect", "--format", "{{json .ExecIDs}}", target.generation.container) in ("null", "[]"), "exact target probe has left no active Docker exec") + _no_target_child(target.generation, running=True) + cases.append("genuine-installed-ready-in-target") + if scenario == "grant": + cases.extend(verify_native_containment_reports(native_context, issuer.pid)) + result["task_observations"] = json.loads((root / "native-task-observations.json").read_bytes()) + assert _snapshot(retained_roots) == preserved + workers.preserved(*worker_before) + consumer_log = target.generation.path("state") / "logs" / "recovery-epoch-bootstrap.log" + result["source_proof"] = {name: getattr(target.receipt, name) for name in ("source_sha", "tested_source_ref", "tested_source_object", "test_run_id", "test_run_attempt", "lock_sha256", "distribution_version", "receipt_identity")} + barrier.resume_initial() + observed_boundary = barrier.wait(timeout=1800) + if scenario == "grant": + _reserved_contender() + cases.append("reservation-barrier-contender-refused") + result["reservation_barrier"] = asdict(observed_boundary) + barrier.release() + else: + if scenario == "retarget-after-reservation": + result["durable"] = _retarget_reserved(barrier, issuer, issuer_log, consumer_log, target) + elif scenario == "lost-reply": + result["durable"] = _lost_reply(native_context, barrier, issuer, issuer_log, consumer_log, target) + elif scenario in LATE_MUTATION_SCENARIOS: + result["durable"], preserved = _late_mutation(native_context, scenario, barrier, issuer, issuer_log, consumer_log, target, retained_roots, lambda: workers.preserved(*worker_before)) + result["preservation_baseline"] = "after-deliberate-fixture-mutation" + elif scenario in POST_RECEIPT_MUTATION_SCENARIOS: + result["durable"], preserved = _post_receipt_mutation(native_context, scenario, barrier, observed_boundary, issuer, issuer_log, consumer_log, target, retained_roots, lambda: workers.preserved(*worker_before), mutation_control=source_mutation is not None) + result["preservation_baseline"] = "after-deliberate-fixture-mutation" + else: + barrier.kill() + actual = scenario if scenario in BOUNDARY_SCENARIOS else "after-" + publication.removesuffix(".json") + result["durable"] = _publication_interruption(actual, observed_boundary, issuer_log, consumer_log, target) + if scenario in {"kill-after-intent", "kill-after-consume"}: + result["durable"]["phase"] = "intent" if scenario == "kill-after-intent" else "consumed" + cases.extend(("native-" + scenario, "interrupted-epoch-replay-refused")) + assert _snapshot(retained_roots) == preserved + workers.preserved(*worker_before) + if mutation_control_manifest is not None: + cases.extend(("originals-oauth-and-workers-preserved", "no-consumer-child-or-admission")) + result.update( + status="mutation-detected", + no_child_started=True, + preserved_originals=True, + preserved_workers=True, + preserved_originals_sha256=preserved, + preserved_worker_claims_sha256=worker_claims, + worker_lifetimes=worker_baseline, + worker_effect_posts=worker_posts, + oracle={"name": ORACLE, "detected": True, "queued_grant": result["durable"]["dispatch"]["queued_grant"]}, + ) + assert set(cases) == required_cases(scenario) + validate_control_result(result, mutation_control_manifest) + logger.info("The focused copied-source mutant failed the actual {} oracle; production acceptance is separate", ORACLE) + return result + _passed(result, cases, preserved, worker_claims, worker_baseline, worker_posts) + return result + _wait(lambda: (EPOCH_ROOT / "receipt-head.json").is_file() and consumer_log.is_file() and "Recovery custody acknowledged" in consumer_log.read_text(), "native grant and consumer custody", timeout=1800, process=issuer) + result["durable"] = inspect_durable_result(EPOCH_ROOT, issuer_log=issuer_log.read_text(), consumer_log=consumer_log.read_text()) + cases.append("complete-native-epoch-granted") + for captured in captures: + assert captured.original.owner is not None + with Path(captured.original.owner.path).open("rb") as stream: + try: + fcntl.flock(stream.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + pass + else: + raise AssertionError("old original owner was not excluded by actual SH custody") + for watcher in watchers: + with Path(watcher.lock.path).open("rb") as stream: + try: + fcntl.flock(stream.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + pass + else: + raise AssertionError("actual watcher EX custody was released") + cases.append("old-owner-and-controller-contenders-refused") + _no_target_child(target.generation, running=True) + assert _snapshot(retained_roots) == preserved + workers.preserved(*worker_before) + durable_before = _snapshot((EPOCH_ROOT,)) + issuer.kill() + issuer.wait(timeout=30) + _wait(lambda: chain.docker("inspect", "--format", "{{.State.Running}}", target.generation.container) == "false", "actual issuer death invalidates consumer custody") + assert chain.docker("inspect", "--format", "{{.State.ExitCode}}", target.generation.container) == "1" + _no_target_child(target.generation, running=False) + cases.append("issuer-death-ended-current-custody") + replay = subprocess.run([sys.executable, "-B", "-m", "auto_coder.management.recovery_epoch"], capture_output=True, text=True, timeout=30) + assert replay.returncode == 2 and _snapshot((EPOCH_ROOT,)) == durable_before, "a consumed epoch was reissued or repaired" + cases.append("consumed-epoch-replay-refused") + assert _snapshot(retained_roots) == preserved + workers.preserved(*worker_before) + _passed(result, cases, preserved, worker_claims, worker_baseline, worker_posts) + return result + except BaseException as error: + result["status"] = "failed" + result["failure_type"] = type(error).__name__ + if isinstance(error, NativeContainmentFailure): + result["containment_failure"] = error.diagnostics + _record_failure_sidecars(result, chain) + _record_failure_catalog(result, target.enrollment_id if target is not None else None) + command_failure = _command_failure(error) + if command_failure is not None: + result["command_failure"] = command_failure + for diagnostic in command_failure["diagnostics"]: + logger.error("Native fixture command diagnostic: {}", diagnostic) + logger.exception("Native recovery ownership fixture failed") + raise + finally: + diagnostics = [] + for path in (root / "issuer.log", root / "fixture.log"): + if path.is_file(): + diagnostics.extend(diagnostic_log_lines(path.read_text(errors="replace"))) + if target is not None: + path = target.generation.path("state") / "logs" / "recovery-epoch-bootstrap.log" + if path.is_file(): + diagnostics.extend(diagnostic_log_lines(path.read_text(errors="replace"))) + result["diagnostics"] = diagnostics[-120:] + output.parent.mkdir(parents=True, exist_ok=True) + _write_private(output, canonical(result)) + if barrier is not None: + barrier.close() + if issuer is not None and issuer.poll() is None: + issuer.send_signal(signal.SIGCONT) + issuer.terminate() + try: + issuer.wait(timeout=15) + except subprocess.TimeoutExpired: + issuer.kill() + issuer.wait(timeout=15) + if log_stream is not None: + log_stream.close() + if chain is not None: + chain.close() + for captured in reversed(captures): + captured.close() + for watcher_fixture in reversed(watcher_fixtures): + watcher_fixture.cleanup() + if routes is not None: + routes.cleanup() + if stock is not None: + stock.cleanup() + if workers is not None: + workers.close() + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--authorization-id", required=True) + parser.add_argument("--source-repository", type=Path, required=True) + parser.add_argument("--source-token-file", type=Path, required=True) + parser.add_argument("--source-sha", required=True) + parser.add_argument("--uv", type=Path, required=True) + parser.add_argument("--uv-version", required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--scenario", choices=SCENARIOS, default="grant") + parser.add_argument("--mutation-control-manifest", type=Path) + args = parser.parse_args(argv) + run_fixture(**vars(args)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/auto_coder/management/bootstrap_supervisor.py b/src/auto_coder/management/bootstrap_supervisor.py index 7970b21d3..f5783b86c 100644 --- a/src/auto_coder/management/bootstrap_supervisor.py +++ b/src/auto_coder/management/bootstrap_supervisor.py @@ -205,6 +205,8 @@ class BootstrapConfig: new_instance_initialization: bool = False new_docker_network: str | None = None new_docker_host_port: int | None = None + original_genesis: Path | None = None + recovery_epoch_config: Path | None = None @classmethod def read(cls, path: Path) -> "BootstrapConfig": @@ -232,6 +234,8 @@ def read(cls, path: Path) -> "BootstrapConfig": "new_instance_initialization", "new_docker_network", "new_docker_host_port", + "original_genesis", + "recovery_epoch_config", } if set(raw) - allowed: raise ValueError("unknown bootstrap configuration field") @@ -273,6 +277,8 @@ def read(cls, path: Path) -> "BootstrapConfig": raw.get("new_instance_initialization", False), raw.get("new_docker_network"), raw.get("new_docker_host_port"), + Path(raw["original_genesis"]) if raw.get("original_genesis") else None, + Path(raw["recovery_epoch_config"]) if raw.get("recovery_epoch_config") else None, ) config.validate() return config @@ -295,6 +301,12 @@ def validate(self) -> None: raise ValueError("invalid management self-update bootstrap profile") if type(self.new_instance_initialization) is not bool: raise ValueError("NEW initialization selector must be a boolean") + if self.original_genesis is not None: + if self.original_genesis != Path("/bootstrap/original-genesis.json") or self.new_instance_initialization or self.fresh_guard_path is not None or self.fresh_prior_lock_path is not None or self.archived_exclusions or self.fresh_runtime_binding is not None: + raise ValueError("original genesis requires the explicitly selected ordinary bootstrap profile") + if self.recovery_epoch_config is not None: + if self.recovery_epoch_config != Path("/bootstrap/recovery-epoch.json") or self.original_genesis is not None or self.new_instance_initialization or self.fresh_guard_path is not None or self.fresh_prior_lock_path is not None or self.fresh_runtime_binding is not None: + raise ValueError("recovery epoch requires its separate explicitly selected bootstrap profile") if (self.new_docker_network is None) != (self.new_docker_host_port is None): raise ValueError("NEW Docker loopback network and port must be selected together") if self.new_docker_network is not None: @@ -369,6 +381,8 @@ class ManagementBootstrapSupervisor: def __init__(self, config: BootstrapConfig, incarnation: str | None = None, bootstrap_config_path: Path | None = None, *, fresh_initialization: bool = False, new_initialization_request: Path | None = None, source_authority: TestedSourcePublisher | None = None) -> None: config.validate() + if config.recovery_epoch_config is not None: + raise ValueError("recovery custody cannot enter the ordinary child supervisor") if bool(new_initialization_request) != config.new_instance_initialization or (fresh_initialization and new_initialization_request is not None): raise ValueError("NEW bootstrap must select only its dedicated first-launch path") from .worker_network_client import configured_worker_bridge_environment @@ -401,6 +415,31 @@ def _owner_file(self) -> Iterator[IO[bytes]]: unenrolled Supervisor profiles keep their existing opening behavior. """ path = self.config.state_root / ".supervisor-owner.lock" + if self.config.original_genesis is not None: + from .original_attempts import history_lock, identity + from .original_genesis import HISTORY_MOUNT, LAUNCH_NAME, NATIVE_NAME, validate_history_genesis + + with history_lock(HISTORY_MOUNT, exclusive=False): + original = validate_history_genesis(HISTORY_MOUNT, mounted_genesis=self.config.original_genesis) + original_state = original.root("state") + if identity(self.config.state_root, directory=True) != (original_state.device, original_state.inode): + raise PermissionError("original genesis owner root differs") + # An interrupted first owner cannot be retried on an empty-looking + # root. Reconstruction requires both original native boundaries. + recorded = all((self.config.state_root / name).is_file() for name in (NATIVE_NAME, LAUNCH_NAME)) + flags = os.O_RDWR | os.O_NOFOLLOW | (0 if recorded else os.O_CREAT | os.O_EXCL) + descriptor = os.open(path, flags, 0o600) + with os.fdopen(descriptor, "a+b") as owner: + info = os.fstat(owner.fileno()) + if (info.st_dev, info.st_ino) != identity(path): + raise PermissionError("original genesis owner lock differs") + if not recorded: + os.fsync(owner.fileno()) + from .original_attempts import _sync + + _sync(self.config.state_root) + yield owner + return if self.new_initialization_request is not None: # O_EXCL is a durable fail-closed lifetime gate. A crashed or # partially published NEW owner cannot acquire a second owner. @@ -830,6 +869,9 @@ def _pristine_initial_state(self) -> bool: def run(self) -> int: """Remain alive as PID 1 and reap only the configured child.""" + if self.config.original_genesis is not None and (not self.config.state_root.is_dir() or self.config.state_root.is_symlink()): + logger.error("Original genesis state root is unavailable; refusing reconstruction") + return 1 if self.config.new_instance_initialization and (self.new_initialization_request is None or not self.config.state_root.is_dir() or self.config.state_root.is_symlink()): logger.error("NEW initialization requires an explicit original request and protected state root") return 1 @@ -897,6 +939,15 @@ def _run_owned(self) -> int: from .self_update_handoff import SelfUpdateHandoffConsumer owner_start_token = self._await_owner_start_token() + original_first_launch = None + if self.config.original_genesis is not None: + from .original_genesis import prepare_original_genesis + + try: + original_first_launch = prepare_original_genesis(self, owner_start_token) + except (OSError, ValueError, TypeError, KeyError, sqlite3.Error): + logger.exception("Original ordinary genesis is unavailable; preserving originals without launch") + return 1 if self.new_initialization_request is not None: if not self._start_new_initialization(): return 1 @@ -980,7 +1031,7 @@ def _run_owned(self) -> int: self.store.require_existing = True consumer = SelfUpdateHandoffConsumer(self) if self.config.management_config is not None else None record = self.store.load() - initial_state_pristine = self._pristine_initial_state() if record.child_generation == 0 else False + initial_state_pristine = original_first_launch is not None or (self._pristine_initial_state() if record.child_generation == 0 else False) if consumer is not None and record.child_generation == 0 and not consumer.store.path.exists() and not consumer.store.path.is_symlink(): try: consumer.store.active() @@ -1006,7 +1057,8 @@ def _run_owned(self) -> int: if not initial_state_pristine or self.store.has_unresolved_effects(): self.store.reserve(record, replace(record, phase=ChildPhase.UNKNOWN, exit_cause="unresolved_effect")) else: - self.start_selected(f"bootstrap-{uuid.uuid4()}", str(uuid.uuid4()), self.config.initial_runtime) + operation, attempt = original_first_launch or (f"bootstrap-{uuid.uuid4()}", str(uuid.uuid4())) + self.start_selected(operation, attempt, self.config.initial_runtime) if self.config.restart_after_clean_shutdown and record.child_generation > 0 and not self._container_shutdown_requested: from .clean_restart import restart_verified_stop @@ -1476,9 +1528,16 @@ def main() -> None: parser.add_argument("--config", required=True, type=Path) parser.add_argument("--fresh-initialization", action="store_true") parser.add_argument("--new-initialization", action="store_true") + parser.add_argument("--recovery-epoch", action="store_true") parser.add_argument("--request-file", type=Path) args = parser.parse_args() config = BootstrapConfig.read(args.config) + if config.recovery_epoch_config is not None or args.recovery_epoch: + if not args.recovery_epoch or config.recovery_epoch_config is None or args.new_initialization or args.fresh_initialization or args.request_file is not None: + parser.exit(1, "Recovery epoch requires its distinct fixed bootstrap profile\n") + from .recovery_epoch_bootstrap import run_recovery_epoch_bootstrap + + raise SystemExit(run_recovery_epoch_bootstrap(config)) if config.new_instance_initialization: if not args.new_initialization or args.fresh_initialization or args.request_file is None: parser.exit(1, "NEW initialization requires a separately admitted first-launch controller\n") @@ -1492,6 +1551,13 @@ def main() -> None: parser.exit(1, f"NEW first-launch original unavailable: {error}\n") elif args.new_initialization or args.request_file is not None: parser.exit(1, "NEW initialization flags require a selected NEW profile\n") + if config.original_genesis is not None: + from .original_genesis import validate_bootstrap_genesis + + try: + validate_bootstrap_genesis(config, args.config) + except (OSError, ValueError, TypeError, KeyError) as error: + parser.exit(1, f"Ordinary genesis original unavailable: {error}\n") setup_logger(stream=sys.stdout, log_file=str(config.state_root / "logs" / "management-bootstrap.log")) raise SystemExit(ManagementBootstrapSupervisor(config, bootstrap_config_path=args.config, fresh_initialization=args.fresh_initialization, new_initialization_request=args.request_file).run()) diff --git a/src/auto_coder/management/containment_cron_inhibition.py b/src/auto_coder/management/containment_cron_inhibition.py index 931139472..414efdc64 100644 --- a/src/auto_coder/management/containment_cron_inhibition.py +++ b/src/auto_coder/management/containment_cron_inhibition.py @@ -62,6 +62,7 @@ _MAX_GROUPS = 8192 _MAX_TASKS = 131072 _MAX_SERVICES = 4096 +_MAX_OBSERVATION_ATTEMPTS = 3 # systemd v255 cg_needs_escape also prefixes controller.* cgroup names. _ESCAPED_SERVICE_PREFIXES = ("cgroup", "cpu", "cpuacct", "cpuset", "io", "blkio", "memory", "devices", "pids", "bpf-firewall", "bpf-devices", "bpf-foreign", "bpf-socket-bind", "bpf-restrict-network-interfaces") _PROPERTIES = ("Id", "LoadState", "UnitFileState", "ActiveState", "SubState", "MainPID", "ControlPID", "NeedDaemonReload", "FragmentPath", "SourcePath", "DropInPaths", "ControlGroup", "Slice", "Delegate") @@ -71,6 +72,14 @@ class CronInhibitionUnavailable(ValueError): """The bounded stock cron execution scope is not currently closed.""" +class _ProcInventoryChanged(CronInhibitionUnavailable): + """An independently changed inventory invalidated the entire observation.""" + + def __init__(self, message: str) -> None: + super().__init__(message) + self.source_binding: _CronSourceBinding | None = None + + class CronScopeRetainsTasks(CronInhibitionUnavailable): """A measured task predicate failed, with a separate safe scope field.""" @@ -223,6 +232,19 @@ def parse(cls, value: object) -> CronInhibitedSource: return cls(PROFILE, value["boot_id"], "cron.service", value["scheduler_sha256"], properties, sources, value["mask_path"], tuple(mask), CronCgroupScope.parse(value["cgroups"], inhibited_units=units), services) +@dataclass(frozen=True) +class _CronSourceBinding: + boot_id: str + controller_properties: tuple[tuple[str, str], ...] + sources: tuple[CommandSourceFact, ...] + mask_path: str + mask_identity: tuple[int, ...] + + @classmethod + def from_observation(cls, value: CronInhibitedSource) -> _CronSourceBinding: + return cls(value.boot_id, value.controller_properties, value.sources, value.mask_path, value.mask_identity) + + def _validate_service_units(units: tuple[str, ...]) -> None: # Literal service names map to literal cgroup components. Templates, # instances, escaping and reserved controller prefixes need other profiles. @@ -325,44 +347,70 @@ def _empty_group(path: Path, relative: str, *, scope: str | None = None) -> Empt return EmptyCgroupFact(relative, (), (), 0, int(values["frozen"])) +def _proc_inventory(path: Path) -> tuple[str, ...]: + return tuple(sorted(entry.name for entry in path.iterdir() if entry.name.isdecimal())) + + +def _proc_identity_disappeared(proc: Path, pid: str, tid: str | None) -> bool: + if pid not in _proc_inventory(proc): + return True + if tid is None: + return False + try: + return tid not in _proc_inventory(proc / pid / "task") + except FileNotFoundError: + # A live process with an unavailable task directory is not an exit. + return pid not in _proc_inventory(proc) + + def _proc_scopes_empty(roots: tuple[str, ...], *, proc: Path = Path("/proc"), inhibited_units: tuple[str, ...] = ("cron.service",)) -> None: - pids = tuple(sorted(path.name for path in proc.iterdir() if path.name.isdecimal())) + pids = _proc_inventory(proc) count = 0 for pid in pids: task_root = proc / pid / "task" - tids = tuple(sorted(path.name for path in task_root.iterdir() if path.name.isdecimal())) - for tid in tids: - count += 1 - if count > _MAX_TASKS: - raise CronInhibitionUnavailable("stock cron task inventory exceeds its bound") - task = task_root / tid - before = _read(task / "stat", 16384) - tail = before.rpartition(")")[2].split() - if len(tail) < 20 or not tail[19].isdecimal(): - raise CronInhibitionUnavailable("stock cron task lifetime is unavailable") - groups = _read(task / "cgroup", 16384).splitlines() - if len(groups) != 1 or not groups[0].startswith("0::/"): - raise CronInhibitionUnavailable("stock cron task cgroup membership is unsupported") - group = groups[0][3:] - if any(group == root or group.startswith(root + "/") for root in roots) or any(part in inhibited_units for part in Path(group).parts): - raise CronScopeRetainsTasks("inhibited stock cron scope retains tasks: pid=" + pid + " tid=" + tid + " state=" + tail[0] + " start=" + tail[19], scope=_retained_task_scope(group, roots, inhibited_units)) - after = _read(task / "stat", 16384).rpartition(")")[2].split() - if len(after) < 20 or after[19] != tail[19] or _read(task / "cgroup", 16384).splitlines() != groups: - raise CronInhibitionUnavailable("stock cron task changed during scope inspection") - if tids != tuple(sorted(path.name for path in task_root.iterdir() if path.name.isdecimal())): - raise CronInhibitionUnavailable("stock cron thread inventory changed during inspection") - # A daemon started outside its normal unit still remains an ordinary - # scheduler, even if that service's own cgroup is empty. - with (proc / pid / "cmdline").open("rb") as stream: - command = stream.read(131073) - if len(command) > 131072: - raise CronInhibitionUnavailable("stock cron process command exceeds its bound") - if command: - executable = os.readlink(proc / pid / "exe") - if executable.removesuffix(" (deleted)") in ("/usr/sbin/cron", *_OTHER_EXECUTABLES): - raise CronScopeRetainsTasks("inhibited stock cron scope retains tasks: scheduler pid=" + pid) - if pids != tuple(sorted(path.name for path in proc.iterdir() if path.name.isdecimal())): - raise CronInhibitionUnavailable("stock cron process inventory changed during inspection") + active_tid = None + try: + tids = _proc_inventory(task_root) + if not tids or pid not in tids: + raise CronInhibitionUnavailable("stock cron live process task inventory is incomplete") + for tid in tids: + active_tid = tid + count += 1 + if count > _MAX_TASKS: + raise CronInhibitionUnavailable("stock cron task inventory exceeds its bound") + task = task_root / tid + before = _read(task / "stat", 16384) + tail = before.rpartition(")")[2].split() + if len(tail) < 20 or not tail[19].isdecimal(): + raise CronInhibitionUnavailable("stock cron task lifetime is unavailable") + groups = _read(task / "cgroup", 16384).splitlines() + if len(groups) != 1 or not groups[0].startswith("0::/"): + raise CronInhibitionUnavailable("stock cron task cgroup membership is unsupported") + group = groups[0][3:] + if any(group == root or group.startswith(root + "/") for root in roots) or any(part in inhibited_units for part in Path(group).parts): + raise CronScopeRetainsTasks("inhibited stock cron scope retains tasks: pid=" + pid + " tid=" + tid + " state=" + tail[0] + " start=" + tail[19], scope=_retained_task_scope(group, roots, inhibited_units)) + after = _read(task / "stat", 16384).rpartition(")")[2].split() + if len(after) < 20 or after[19] != tail[19] or _read(task / "cgroup", 16384).splitlines() != groups: + raise CronInhibitionUnavailable("stock cron task changed during scope inspection") + active_tid = None + if tids != _proc_inventory(task_root): + raise _ProcInventoryChanged("stock cron thread inventory changed during inspection") + # A daemon started outside its normal unit remains an ordinary + # scheduler, even if that service's own cgroup is empty. + with (proc / pid / "cmdline").open("rb") as stream: + command = stream.read(131073) + if len(command) > 131072: + raise CronInhibitionUnavailable("stock cron process command exceeds its bound") + if command: + executable = os.readlink(proc / pid / "exe") + if executable.removesuffix(" (deleted)") in ("/usr/sbin/cron", *_OTHER_EXECUTABLES): + raise CronScopeRetainsTasks("inhibited stock cron scope retains tasks: scheduler pid=" + pid) + except FileNotFoundError as exc: + if _proc_identity_disappeared(proc, pid, active_tid): + raise _ProcInventoryChanged("stock cron process/thread inventory changed during inspection") from exc + raise + if pids != _proc_inventory(proc): + raise _ProcInventoryChanged("stock cron process inventory changed during inspection") def _cgroup_snapshot(*, root: Path = _CGROUP_ROOT, mountinfo: Path = Path("/proc/self/mountinfo"), proc: Path = Path("/proc"), inhibited_units: tuple[str, ...] = ("cron.service",)) -> CronCgroupScope: @@ -455,7 +503,7 @@ def _properties() -> tuple[tuple[str, str], ...]: return tuple(sorted(values.items())) -def _snapshot(scheduler_sha256: str, inhibited_services: tuple[tuple[str, str], ...]) -> CronInhibitedSource: +def _source_snapshot() -> _CronSourceBinding: boot = _read(Path("/proc/sys/kernel/random/boot_id"), 128).strip() if not re.fullmatch(r"[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}", boot): raise CronInhibitionUnavailable("stock cron boot identity is unavailable") @@ -466,7 +514,17 @@ def _snapshot(scheduler_sha256: str, inhibited_services: tuple[tuple[str, str], if not stat.S_ISLNK(info.st_mode) or info.st_uid != 0 or os.readlink(mask) != "/dev/null": raise CronInhibitionUnavailable("stock cron protected mask is unavailable") identity = (info.st_dev, info.st_ino, info.st_ctime_ns, info.st_mode, info.st_uid, info.st_gid) - return CronInhibitedSource(PROFILE, boot, "cron.service", scheduler_sha256, properties, _profile_sources(), str(mask), identity, _cgroup_snapshot(inhibited_units=tuple(unit for unit, _ in inhibited_services)), inhibited_services) + return _CronSourceBinding(boot, properties, _profile_sources(), str(mask), identity) + + +def _snapshot(scheduler_sha256: str, inhibited_services: tuple[tuple[str, str], ...]) -> CronInhibitedSource: + source = _source_snapshot() + try: + cgroups = _cgroup_snapshot(inhibited_units=tuple(unit for unit, _ in inhibited_services)) + except _ProcInventoryChanged as exc: + exc.source_binding = source + raise + return CronInhibitedSource(PROFILE, source.boot_id, "cron.service", scheduler_sha256, source.controller_properties, source.sources, source.mask_path, source.mask_identity, cgroups, inhibited_services) def observe_inhibited_stock_cron_scope(systemd_facts: tuple[object, ...]) -> CronInhibitedSource: @@ -485,10 +543,33 @@ def observe_inhibited_stock_cron_scope(systemd_facts: tuple[object, ...]) -> Cro raise CronInhibitionUnavailable("stock cron controller observation identity is unavailable") try: services = _inhibited_service_bindings(systemd_facts) - first = _snapshot(digest, services) - if _snapshot(digest, services) != first: - raise CronInhibitionUnavailable("stock cron source/controller/task scope changed during observation") - return first + source_binding: _CronSourceBinding | None = None + + def retain_source_binding(current: _CronSourceBinding) -> None: + nonlocal source_binding + if source_binding is not None and current != source_binding: + raise CronInhibitionUnavailable("stock cron source/controller authority changed during observation") + source_binding = current + + for attempt in range(_MAX_OBSERVATION_ATTEMPTS): + try: + first = _snapshot(digest, services) + retain_source_binding(_CronSourceBinding.from_observation(first)) + second = _snapshot(digest, services) + retain_source_binding(_CronSourceBinding.from_observation(second)) + except _ProcInventoryChanged as exc: + if exc.source_binding is None: + raise CronInhibitionUnavailable("stock cron incomplete observation lacks its source binding") from exc + retain_source_binding(exc.source_binding) + if attempt == _MAX_OBSERVATION_ATTEMPTS - 1: + raise CronInhibitionUnavailable("stock cron process/thread inventory did not stabilize") from exc + # No earlier successful or partial task snapshot supplies any + # part of the next positive observation pair. + continue + if second != first: + raise CronInhibitionUnavailable("stock cron source/controller/task scope changed during observation") + return first + raise CronInhibitionUnavailable("stock cron complete observation pair is unavailable") except (OSError, ValueError, subprocess.SubprocessError) as exc: if isinstance(exc, CronInhibitionUnavailable): raise diff --git a/src/auto_coder/management/containment_observation.py b/src/auto_coder/management/containment_observation.py index b0e49c283..43cd82826 100644 --- a/src/auto_coder/management/containment_observation.py +++ b/src/auto_coder/management/containment_observation.py @@ -174,6 +174,22 @@ def _docker_read(arguments: tuple[str, ...]) -> object: def _docker(scope: ProposedScope) -> tuple[DockerObservation, ...]: + return _docker_selected((scope.management_id,), (scope.proxy_id,), all_bind_sources=False) + + +def observe_recovery_docker(management_ids: tuple[str, ...], proxy_ids: tuple[str, ...]) -> tuple[DockerObservation, ...]: + """Enumerate the whole engine against the explicitly selected recovery union. + + This only discovers candidates. The recovery observer separately validates + the protected selection, every object profile and all installed routes. + """ + return _docker_selected(management_ids, proxy_ids, all_bind_sources=True) + + +def _docker_selected(management_ids: tuple[str, ...], proxy_ids: tuple[str, ...], *, all_bind_sources: bool) -> tuple[DockerObservation, ...]: + selected = management_ids + proxy_ids + if not management_ids or not proxy_ids or len(selected) > 64 or len(set(selected)) != len(selected) or any(not isinstance(item, str) or not re.fullmatch(r"[0-9a-f]{64}", item) for item in selected): + raise ValueError("selected Docker identities are absent, duplicate or ambiguous") # A remote context cannot supply this host's kernel/Docker correlation. if os.environ.get("DOCKER_HOST") or os.environ.get("DOCKER_CONTEXT"): raise ValueError("Docker endpoint override is unsupported for local observation") @@ -186,7 +202,7 @@ def _docker(scope: ProposedScope) -> tuple[DockerObservation, ...]: # Enumerate the engine, not an operator/label-filtered selection. Unknown # objects remain visible; enumeration does not establish their enrollment. containers = _docker_ids() - if not {scope.management_id, scope.proxy_id} <= set(containers): + if not set(selected) <= set(containers): raise ValueError("Selected Docker objects are absent from the complete engine inventory") observations = [] inspected: dict[str, dict[str, object]] = {} @@ -219,12 +235,15 @@ def _docker(scope: ProposedScope) -> tuple[DockerObservation, ...]: # its exact images, an installed management command or its network namespace. # This is candidate discovery, not proof that all ordinary controllers are # known; the separate installed-route inventory is still required. - management_image = inspected[scope.management_id]["Image"] - proxy_image = inspected[scope.proxy_id]["Image"] - management_mounts = inspected[scope.management_id]["Mounts"] - if not isinstance(management_mounts, list): - raise ValueError("Selected management mount inventory is unavailable") - management_sources = tuple(source for mount in management_mounts if isinstance(mount, dict) and mount.get("Type") == "bind" and mount.get("Destination") in _MANAGEMENT_MOUNT_DESTINATIONS and isinstance(source := mount.get("Source"), str) and os.path.isabs(source)) + management_images = {inspected[identifier]["Image"] for identifier in management_ids} + proxy_images = {inspected[identifier]["Image"] for identifier in proxy_ids} + management_mounts: list[object] = [] + for identifier in management_ids: + mounts = inspected[identifier]["Mounts"] + if not isinstance(mounts, list): + raise ValueError("Selected management mount inventory is unavailable") + management_mounts.extend(mounts) + management_sources = tuple(source for mount in management_mounts if isinstance(mount, dict) and mount.get("Type") == "bind" and (all_bind_sources or mount.get("Destination") in _MANAGEMENT_MOUNT_DESTINATIONS) and isinstance(source := mount.get("Source"), str) and os.path.isabs(source)) def overlapping_management_source(candidate_mounts: object) -> bool: if not isinstance(candidate_mounts, list): @@ -259,19 +278,19 @@ def overlapping_management_source(candidate_mounts: object) -> bool: raise ValueError("Docker network mode is ambiguous") arguments = tuple((entrypoint or []) + (command or [])) roles = [] - if item.container_id == scope.management_id: + if item.container_id in management_ids: roles.append("selected-management") - elif item.image_id == management_image: + elif item.image_id in management_images: roles.append("same-management-image") - if item.container_id == scope.proxy_id: + if item.container_id in proxy_ids: roles.append("selected-proxy") - elif item.image_id == proxy_image: + elif item.image_id in proxy_images: roles.append("same-proxy-image") if "auto_coder.management.bootstrap_supervisor" in arguments or ("auto_coder.cli" in arguments and "management" in arguments and "serve" in arguments): roles.append("management-command") - if network_mode == "container:" + scope.management_id: + if network_mode in {"container:" + identifier for identifier in management_ids}: roles.append("selected-management-network") - if item.container_id not in (scope.management_id, scope.proxy_id) and overlapping_management_source(obj["Mounts"]): + if item.container_id not in selected and overlapping_management_source(obj["Mounts"]): roles.append("selected-management-mount-source") if roles or "ExecIDs" not in obj: classified.append(replace(item, management_route_roles=tuple(roles))) diff --git a/src/auto_coder/management/containment_route_manifest.py b/src/auto_coder/management/containment_route_manifest.py index 1753f8289..1f9686f4a 100644 --- a/src/auto_coder/management/containment_route_manifest.py +++ b/src/auto_coder/management/containment_route_manifest.py @@ -17,7 +17,7 @@ import re import stat import subprocess -from dataclasses import asdict, dataclass +from dataclasses import asdict, dataclass, replace from pathlib import Path from .archived_exclusions import unique_pairs @@ -26,6 +26,7 @@ ENROLLMENT_ROOT, EnrollmentObservation, EnrollmentUnavailable, + OriginalFile, _approved_directory, _file_identity, _protected, @@ -113,6 +114,34 @@ class RouteManifestUnavailable(ValueError): """An approved, complete and currently inhibited route set is unavailable.""" +@dataclass(frozen=True) +class RecoveryRouteScope: + """The protected recovery selection, without a historical or grant claim.""" + + deployment_id: str + enrollment_id: str + authorization_id: str + profile: str + container_ids: tuple[str, ...] + original_files: tuple[OriginalFile, ...] + + +@dataclass(frozen=True) +class RecoveryRouteSelection: + enrollment: RecoveryRouteScope + enrollment_sha256: str + + +def _route_container_ids(approval: EnrollmentObservation | RecoveryRouteSelection) -> tuple[str, ...]: + if isinstance(approval, RecoveryRouteSelection): + scope = approval.enrollment + selected = scope.container_ids + if scope.profile != "ordinary-recovery-v1" or not 3 <= len(selected) <= 64 or selected != tuple(sorted(set(selected))) or any(not _CONTAINER.fullmatch(item) for item in selected): + raise RouteManifestUnavailable("recovery route selection differs") + return selected + return approval.enrollment.management_id, approval.enrollment.proxy_id + + @dataclass(frozen=True) class SystemdRoute: unit: str @@ -282,9 +311,10 @@ def parse(cls, value: object) -> ApprovedRouteManifest: bindings: tuple[DeploymentRootBinding, ...] = () user_digest = "" user_routes: tuple[UserSystemdRoute, ...] = () - elif value.get("schema") == "approved-management-ordinary-routes-v2" and set(value) == set(cls.__dataclass_fields__): + elif value.get("schema") in ("approved-management-ordinary-routes-v2", "approved-management-recovery-routes-v1") and set(value) == set(cls.__dataclass_fields__): raw_bindings = value["deployment_roots"] - if not isinstance(raw_bindings, list) or not 1 <= len(raw_bindings) <= 128: + limit = 1024 if value["schema"] == "approved-management-recovery-routes-v1" else 128 + if not isinstance(raw_bindings, list) or not 1 <= len(raw_bindings) <= limit: raise RouteManifestUnavailable("approved deployment root inventory is absent or oversized") bindings = tuple(DeploymentRootBinding.parse(item) for item in raw_bindings) binding_keys = tuple((item.container_id, item.destination) for item in bindings) @@ -307,7 +337,8 @@ def parse(cls, value: object) -> ApprovedRouteManifest: for name in ("deployment_id", "enrollment_id", "authorization_id"): if not isinstance(value[name], str) or not re.fullmatch(r"[A-Za-z0-9_.:-]{1,128}", value[name]): raise RouteManifestUnavailable("approved ordinary-route deployment differs") - if value["profile"] not in ("ordinary-v1", "quarantined-original-v1"): + profiles = ("ordinary-recovery-v1",) if value["schema"] == "approved-management-recovery-routes-v1" else ("ordinary-v1", "quarantined-original-v1") + if value["profile"] not in profiles: raise RouteManifestUnavailable("approved ordinary-route profile differs") if not isinstance(value["enrollment_sha256"], str) or not _HASH.fullmatch(value["enrollment_sha256"]): raise RouteManifestUnavailable("approved ordinary-route enrollment binding differs") @@ -626,14 +657,20 @@ def _user_controllers(systemd: tuple[SystemdRouteFact, ...], cues: tuple[str, .. return inventory, tuple(sorted(units, key=lambda item: (item.uid, item.fact.unit))) -def _deployment_root_bindings(approval: EnrollmentObservation, *, trusted_uid: int = 0) -> tuple[DeploymentRootBinding, ...]: +def _deployment_root_bindings(approval: EnrollmentObservation | RecoveryRouteSelection, *, trusted_uid: int = 0) -> tuple[DeploymentRootBinding, ...]: """Derive scope roots from the exact enrolled pair, never a caller root list.""" + return observe_selected_bind_roots(_route_container_ids(approval), trusted_uid=trusted_uid) + + +def observe_selected_bind_roots(selected: tuple[str, ...], *, trusted_uid: int = 0) -> tuple[DeploymentRootBinding, ...]: + """Read actual selected-object bind identities; this does not enroll them.""" if os.environ.get("DOCKER_HOST") or os.environ.get("DOCKER_CONTEXT"): raise RouteManifestUnavailable("remote Docker root discovery is unsupported") - selected = (approval.enrollment.management_id, approval.enrollment.proxy_id) + if not 1 <= len(selected) <= 64 or len(set(selected)) != len(selected) or any(not isinstance(item, str) or not _CONTAINER.fullmatch(item) for item in selected): + raise RouteManifestUnavailable("selected bind-root Docker identities differ") raw = _command("docker", "inspect", "--type", "container", *selected, max_bytes=8 * _MAX_MANIFEST_BYTES) decoded = json.loads(raw, object_pairs_hook=unique_pairs) - if not isinstance(decoded, list) or len(decoded) != 2 or {item.get("Id") for item in decoded if isinstance(item, dict)} != set(selected): + if not isinstance(decoded, list) or len(decoded) != len(selected) or {item.get("Id") for item in decoded if isinstance(item, dict)} != set(selected): raise RouteManifestUnavailable("deployment root Docker scope differs") result = [] for container in decoded: @@ -821,14 +858,14 @@ def _local_script(path: Path, *, expected_uid: int) -> tuple[str, int, int]: return hashlib.sha256(data).hexdigest(), after.st_dev, after.st_ino -def _deployment_cues(approval: EnrollmentObservation, bindings: tuple[DeploymentRootBinding, ...]) -> tuple[str, ...]: - return tuple(sorted({item.source for item in bindings} | {item.path for item in approval.enrollment.original_files} | {approval.enrollment.management_id, approval.enrollment.proxy_id} | set(CRON_ROUTE_CUES))) +def _deployment_cues(approval: EnrollmentObservation | RecoveryRouteSelection, bindings: tuple[DeploymentRootBinding, ...]) -> tuple[str, ...]: + return tuple(sorted({item.source for item in bindings} | {item.path for item in approval.enrollment.original_files} | set(_route_container_ids(approval)) | set(CRON_ROUTE_CUES))) -def _deployment_modules(approval: EnrollmentObservation) -> tuple[tuple[str, str], ...]: +def _deployment_modules(approval: EnrollmentObservation | RecoveryRouteSelection) -> tuple[tuple[str, str], ...]: # This known original controller is supported only through its separately # approved source binding and isolated Python command, not arbitrary -m. - return tuple(("auto_coder.management.proxy_watch_cli", item.path) for item in approval.enrollment.original_files if getattr(item, "purpose", None) == "watcher-source") + return tuple(sorted({("auto_coder.management.proxy_watch_cli", item.path) for item in approval.enrollment.original_files if getattr(item, "purpose", None) == "watcher-source"})) def _validate_user_scope(manifest: ApprovedRouteManifest, inventory: UserControllerInventory, facts: tuple[UserSystemdRouteFact, ...]) -> None: @@ -856,8 +893,30 @@ def _validate_user_scope(manifest: ApprovedRouteManifest, inventory: UserControl raise RouteManifestUnavailable("installed user management source lacks an independently inhibited unit") +def _cron_approval_binding(entries: tuple[CronEntry, ...], *, recovery: bool) -> tuple[CronEntry, ...]: + """Bind recovery approval to every inhibited scope, not unrelated groups. + + A target's ordinary Docker scope can be created after route approval. Only + unrelated cgroup directory identities are excluded from this historical + comparison. Complete fresh snapshots and returned observations retain them. + Ordinary profiles keep their existing exact approval comparison. + """ + if not recovery: + return entries + bindings = [] + for entry in entries: + source = entry.inhibited_source + if source is None: + bindings.append(entry) + continue + scope = source.cgroups + directories = tuple(item for item in scope.directories if item.path in ("/", "/system.slice") or any(item.path == root or item.path.startswith(root + "/") or root.startswith(item.path.rstrip("/") + "/") for root in scope.required_empty_roots)) + bindings.append(replace(entry, inhibited_source=replace(source, cgroups=replace(scope, directories=directories)))) + return tuple(bindings) + + def _observe_approved_route_manifest( - approval: EnrollmentObservation, + approval: EnrollmentObservation | RecoveryRouteSelection, docker: tuple[object, ...], *, root: Path, @@ -883,7 +942,10 @@ def _observe_approved_route_manifest( ids = tuple(sorted(getattr(item, "container_id", "") for item in docker)) if ids != manifest.docker_container_ids: raise RouteManifestUnavailable("whole-engine Docker inventory differs from approved routes") - is_closed_version = manifest.schema == "approved-management-ordinary-routes-v2" + is_closed_version = manifest.schema in ("approved-management-ordinary-routes-v2", "approved-management-recovery-routes-v1") + recovery_routes = manifest.schema == "approved-management-recovery-routes-v1" + if isinstance(approval, RecoveryRouteSelection) != recovery_routes: + raise RouteManifestUnavailable("route manifest and selection profile differ") bindings = _deployment_root_bindings(approval, trusted_uid=expected_uid) if is_closed_version else () if bindings != manifest.deployment_roots: raise RouteManifestUnavailable("original deployment root/config binding differs from approved routes") @@ -904,7 +966,7 @@ def _observe_approved_route_manifest( if manifest.cron_restarter_paths and not _CRON_SCHEDULERS.intersection(restarters): raise RouteManifestUnavailable("ordinary cron restarter lacks scheduler inhibition") cron = _cron_catalog(roots=cron_roots, trusted_uid=expected_uid, extra_cues=cues, inspect_local_inputs=is_closed_version, module_sources=modules, source_formats=cron_source_formats, systemd_facts=observed_systemd) - if cron != manifest.cron_entries: + if _cron_approval_binding(cron, recovery=recovery_routes) != _cron_approval_binding(manifest.cron_entries, recovery=recovery_routes): raise RouteManifestUnavailable("unknown, omitted or changed cron/at route") if any(entry.management_candidate and entry.path not in manifest.cron_restarter_paths for entry in cron): raise RouteManifestUnavailable("detected management cron/at route is excluded") @@ -945,13 +1007,23 @@ def observe_approved_route_manifest(approval: EnrollmentObservation, docker: tup return _observe_approved_route_manifest(approval, docker, root=ENROLLMENT_ROOT, expected_uid=0) -def propose_unapproved_route_draft(approval: EnrollmentObservation, docker: tuple[object, ...], *, closed_profile: bool = False) -> dict[str, object]: +def observe_recovery_route_manifest(approval: RecoveryRouteSelection, docker: tuple[object, ...]) -> RouteManifestObservation: + """Read only the fixed recovery approval and the entire real route scope.""" + from .recovery_containment_model import RECOVERY_ENROLLMENT_ROOT + + return _observe_approved_route_manifest(approval, docker, root=RECOVERY_ENROLLMENT_ROOT, expected_uid=0) + + +def propose_unapproved_route_draft(approval: EnrollmentObservation | RecoveryRouteSelection, docker: tuple[object, ...], *, closed_profile: bool = False) -> dict[str, object]: """Provide inspectable values for a human review; never attest completeness. All exclusions and absent local/remote sources are unreviewed. This draft cannot be installed as usable approval until a trusted root explicitly audits and replaces the false completeness declaration. """ + recovery = isinstance(approval, RecoveryRouteSelection) + if recovery and not closed_profile: + raise RouteManifestUnavailable("recovery requires complete current user/root observations") bindings = _deployment_root_bindings(approval) if closed_profile else () cues = _deployment_cues(approval, bindings) if closed_profile else () modules = _deployment_modules(approval) if closed_profile else () @@ -960,7 +1032,7 @@ def propose_unapproved_route_draft(approval: EnrollmentObservation, docker: tupl cron = _cron_catalog(roots=_CRON_ROOTS, extra_cues=cues, inspect_local_inputs=closed_profile, module_sources=modules, systemd_facts=systemd) users, user_units = _user_controllers(systemd, cues, modules) if closed_profile else (None, ()) return { - "schema": "approved-management-ordinary-routes-v2" if closed_profile else "approved-management-ordinary-routes-v1", + "schema": "approved-management-recovery-routes-v1" if recovery else "approved-management-ordinary-routes-v2" if closed_profile else "approved-management-ordinary-routes-v1", "deployment_id": approval.enrollment.deployment_id, "enrollment_id": approval.enrollment.enrollment_id, "authorization_id": approval.enrollment.authorization_id, diff --git a/src/auto_coder/management/exclusion_host.py b/src/auto_coder/management/exclusion_host.py index 679a5409e..22588ad56 100644 --- a/src/auto_coder/management/exclusion_host.py +++ b/src/auto_coder/management/exclusion_host.py @@ -73,6 +73,21 @@ def inherited_support(mounts: list[object]) -> tuple[tuple[ArchivedExclusion, .. if raw.get("fresh_prior_lock_path") is None: if archived: raise ValueError("archives lack fresh predecessor provenance") + if raw.get("original_genesis") is not None: + from .original_genesis import GENESIS_NAME, HISTORY_MOUNT, read_genesis, validate_history_genesis + + if raw["original_genesis"] != "/bootstrap/" + GENESIS_NAME: + raise ValueError("ordinary original genesis path differs") + original = read_genesis(bootstrap.parent / GENESIS_NAME) + if validate_history_genesis(Path(original.binding.root)) != original or original.instance_id != raw.get("instance_id") or original.file("bootstrap").sha256 != hashlib.sha256(content).hexdigest(): + raise ValueError("ordinary original genesis authority differs") + for role in ("state", "journal", "oauth", "history"): + expected = original.root(role) + selected = _mount(mounts, expected.container_path, rw=role != "history") + current = selected.lstat() + if str(selected) != expected.host_path or (current.st_dev, current.st_ino) != (expected.device, expected.inode): + raise ValueError("ordinary original genesis mounted root differs") + return (), {str(HISTORY_MOUNT)} return (), set() if raw.get("state_root") != "/management-child-state" or raw.get("fresh_prior_lock_path") != "/management-prior-owner.lock": raise ValueError("predecessor fresh paths differ") @@ -119,6 +134,16 @@ def required_exclusions(prior: PriorContainer) -> tuple[ArchivedExclusion, ...]: def validate_successor(prior: PriorContainer, intent: FreshInitializationIntent, bootstrap_path: Path | None) -> tuple[ArchivedExclusion, ...]: required = required_exclusions(prior) prior_config = json.loads(protected_bytes(Path(prior.bootstrap_source), private=False), object_pairs_hook=unique_pairs) + if prior_config.get("original_genesis") is not None: + from .original_genesis import GENESIS_NAME, read_genesis, validate_history_genesis + + if prior_config["original_genesis"] != "/bootstrap/" + GENESIS_NAME: + raise ValueError("original genesis predecessor selection differs") + genesis = read_genesis(Path(prior.bootstrap_source).parent / GENESIS_NAME) + if validate_history_genesis(Path(genesis.binding.root)) != genesis or (genesis.instance_id, genesis.management_id, genesis.file("bootstrap").sha256) != (prior.instance_id, prior.container_id, prior.bootstrap_sha256): + raise ValueError("original genesis predecessor authority differs") + if (intent.original_history, intent.original_history_id) != (genesis.binding.root, genesis.binding.history): + raise ValueError("covered original genesis history cannot be omitted or replaced") if prior_config.get("fresh_prior_lock_path") is not None: # An explicitly enrolled original predecessor cannot silently lose its # source-bound history when the next real host cutover is prepared. diff --git a/src/auto_coder/management/fresh_host_observation.py b/src/auto_coder/management/fresh_host_observation.py index 264b36788..58376e902 100644 --- a/src/auto_coder/management/fresh_host_observation.py +++ b/src/auto_coder/management/fresh_host_observation.py @@ -18,6 +18,7 @@ import subprocess import time import uuid +from contextlib import closing from dataclasses import asdict, dataclass, replace from pathlib import Path from typing import Callable @@ -313,7 +314,7 @@ def _capture_supervisor_identity(prior: PriorContainer, values: list[object], ob ledger = Path(state_sources[0]) / "management-child.sqlite3" if ledger.is_symlink() or not ledger.is_file(): raise ValueError("prior supervisor ledger unavailable") - with sqlite3.connect(ledger.as_uri() + "?mode=ro", uri=True, timeout=2) as connection: + with closing(sqlite3.connect(ledger.as_uri() + "?mode=ro", uri=True, timeout=2)) as connection: rows = connection.execute("SELECT instance_id,incarnation,pid,start_token FROM supervisor_presence").fetchall() if len(rows) != 1 or rows[0][0] != instance_id or rows[0][2:] != (1, token) or not isinstance(rows[0][1], str) or str(uuid.UUID(rows[0][1])) != rows[0][1]: raise ValueError("prior live supervisor identity differs") @@ -663,6 +664,10 @@ def assert_host_listener_absent(port: int) -> None: def assert_replacement_published_listener(intent: FreshInitializationIntent) -> None: + assert_container_published_listener(intent.new_container_id, intent.image_digest, intent.host_listener_port) + + +def assert_container_published_listener(container_id: str, image_id: str, published_port: int) -> None: """Prove the host listener is Docker's proxy for this exact replacement. This intentionally supports only the Docker userland-proxy profile. Kernel @@ -672,8 +677,10 @@ def assert_replacement_published_listener(intent: FreshInitializationIntent) -> try: if os.readlink("/proc/self/ns/net") != os.readlink("/proc/1/ns/net") or os.readlink("/proc/self/ns/pid") != os.readlink("/proc/1/ns/pid"): raise ValueError("observer is not in the host namespaces") - values = _inspect(intent.new_container_id) - if values[1] != intent.image_digest or values[2] is not True or values[17] != {"8080/tcp": [{"HostIp": "127.0.0.1", "HostPort": str(intent.host_listener_port)}]}: + if not re.fullmatch(r"[0-9a-f]{64}", container_id) or not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id) or type(published_port) is not int or not 0 < published_port <= 65535: + raise ValueError("published endpoint selection differs") + values = _inspect(container_id) + if values[1] != image_id or values[2] is not True or values[17] != {"8080/tcp": [{"HostIp": "127.0.0.1", "HostPort": str(published_port)}]}: raise ValueError("replacement published endpoint differs") networks = values[18] if not isinstance(networks, dict): @@ -693,7 +700,7 @@ def listener_inodes() -> set[str]: if len(fields) < 10: raise ValueError("listener table incomplete") address, port = fields[1].rsplit(":", 1) - if int(port, 16) == intent.host_listener_port and fields[3] == "0A": + if int(port, 16) == published_port and fields[3] == "0A": if table != "tcp" or address != "0100007F" or not fields[9].isdecimal(): raise ValueError("unexpected host listener") found.add(fields[9]) @@ -705,8 +712,8 @@ def listener_inodes() -> set[str]: daemon = int(Path("/var/run/docker.pid").read_text(encoding="ascii").strip()) if daemon <= 0 or os.stat(f"/proc/{daemon}/exe") != os.stat("/usr/bin/dockerd"): raise ValueError("Docker daemon identity differs") - expected_prefix = ["/usr/bin/docker-proxy", "-proto", "tcp", "-host-ip", "127.0.0.1", "-host-port", str(intent.host_listener_port), "-container-ip"] - socket_owners = subprocess.run(["ss", "-ltnpH", f"sport = :{intent.host_listener_port}"], check=True, capture_output=True, text=True, timeout=10).stdout + expected_prefix = ["/usr/bin/docker-proxy", "-proto", "tcp", "-host-ip", "127.0.0.1", "-host-port", str(published_port), "-container-ip"] + socket_owners = subprocess.run(["ss", "-ltnpH", f"sport = :{published_port}"], check=True, capture_output=True, text=True, timeout=10).stdout candidates = [int(value) for value in re.findall(r'"docker-proxy",pid=(\d+),fd=\d+', socket_owners)] if len(candidates) != 1: raise ValueError("Docker proxy is missing or ambiguous") @@ -729,7 +736,7 @@ def listener_inodes() -> set[str]: raise ValueError("replacement proxy lifetime changed") if listener_inodes() != inodes: raise ValueError("replacement listener inode changed") - final_values = _inspect(intent.new_container_id) + final_values = _inspect(container_id) # Docker reports mounts as an unordered collection. Compare every # complete mount object, including duplicate entries, independent @@ -866,7 +873,7 @@ def assert_replacement_owner(intent: FreshInitializationIntent, incarnation: str ledger = Path(intent.host_state_root) / "management-child.sqlite3" if ledger.is_symlink() or not ledger.is_file(): raise ValueError("replacement ledger unavailable") - with sqlite3.connect(ledger.as_uri() + "?mode=ro", uri=True, timeout=2) as connection: + with closing(sqlite3.connect(ledger.as_uri() + "?mode=ro", uri=True, timeout=2)) as connection: rows = connection.execute("SELECT instance_id,incarnation,pid,start_token FROM supervisor_presence").fetchall() fields = (Path("/proc") / str(peer_pid) / "stat").read_text(encoding="utf-8").rsplit(")", 1)[1].split() if len(fields) < 20 or rows != [(intent.new_instance, incarnation, 1, fields[19])]: diff --git a/src/auto_coder/management/original_attempt_evidence.py b/src/auto_coder/management/original_attempt_evidence.py index 2ae45d4b7..9ff7d63f2 100644 --- a/src/auto_coder/management/original_attempt_evidence.py +++ b/src/auto_coder/management/original_attempt_evidence.py @@ -20,6 +20,7 @@ from .fresh_initialization import FreshInitializationIntent, parse_fresh_intent, read_fresh_status from .original_attempts import PROFILE, HistoryBinding, HistoryEvent, OriginalEvidenceUnavailable, canonical, history_lock, identity, read_frontier, read_original, source_identity, validate_original_lifetime +from .original_genesis import HISTORY_PROFILE, GenesisAnchor @dataclass(frozen=True) @@ -51,6 +52,46 @@ class OriginalAttemptEvidence: # There is deliberately no completeness boolean or authorization field. +@dataclass(frozen=True) +class OriginalGeneration: + """Original scoped identities; an unresolved outcome remains unresolved.""" + + request_id: str + instance_id: str + management_id: str + image_id: str + bootstrap_config_sha256: str + management_config_sha256: str + state_root: str + state_device: int + state_inode: int + journal_root: str + journal_device: int + journal_inode: int + oauth_root: str + oauth_device: int + oauth_inode: int + owner_path: str + owner_device: int + owner_inode: int + outcome: str + expected_stores: tuple[tuple[str, str], ...] + proxy_id: str = "" + proxy_image_id: str = "" + + +@dataclass(frozen=True) +class OriginalLineageEvidence: + binding: HistoryBinding + profile: str = HISTORY_PROFILE + genesis: GenesisAnchor | None = None + frontier: str = "" + attempts: tuple[AttemptEvidence, ...] = () + generations: tuple[OriginalGeneration, ...] = () + originals: tuple[OriginalFile, ...] = () + unavailable: tuple[str, ...] = () + + def _snapshot(path: Path) -> OriginalFile: data, (device, inode) = read_original(path) return OriginalFile(str(path), device, inode, hashlib.sha256(data).hexdigest()) @@ -114,15 +155,39 @@ def _check_owner_fence(path: Path, fd: int, expected: tuple[int, int]) -> None: raise OriginalEvidenceUnavailable("consumer does not hold original-store owner fence") -def derive(binding: HistoryBinding, fence: int, owner_fences: tuple[int, ...] = ()) -> OriginalAttemptEvidence: +def _derive(binding: HistoryBinding, fence: int, owner_fences: tuple[int, ...] = (), *, preserve_outcomes: bool = False) -> tuple[OriginalAttemptEvidence, tuple[OriginalGeneration, ...]]: """Enumerate original authority, independently of any submitted result.""" files: dict[str, OriginalFile] = {} attempts: list[AttemptEvidence] = [] + generations: list[OriginalGeneration] = [] unavailable: list[str] = [] held_owners: list[tuple[Path, int, tuple[int, int]]] = [] scanned_namespaces: dict[Path, tuple[tuple[int, int], tuple[str, ...]]] = {} frontier = "" + def generation(intent: FreshInitializationIntent, outcome: str, stores: tuple[tuple[str, str], ...], owner: tuple[int, int] = (0, 0)) -> OriginalGeneration: + return OriginalGeneration( + intent.request_id, + intent.new_instance, + intent.new_container_id, + intent.image_digest, + intent.bootstrap_config_sha256, + intent.management_config_sha256, + intent.host_state_root, + intent.state_device, + intent.state_inode, + intent.host_journal_root, + intent.journal_device, + intent.journal_inode, + intent.host_oauth_root, + intent.oauth_device, + intent.oauth_inode, + str(Path(intent.host_state_root) / ".supervisor-owner.lock"), + *owner, + outcome, + stores, + ) + def retain(path: Path) -> bytes: # Hash the exact bytes consumed by this derivation. A second read # could parse changed bytes under the first read's digest. @@ -230,6 +295,7 @@ def retain(path: Path) -> bytes: if kinds[-1] == "never-created": if anchor["pristine"] is not True or json.loads(branch[-1].value_json) != {"lifetime": anchor["lifetime"], "stores": expected} or scanned_namespaces[state][1] or scanned_namespaces[journal][1]: raise OriginalEvidenceUnavailable("never-created original boundary conflicts with effects") + generations.append(generation(intent, "never-created", tuple(sorted(expected.items())))) continue if "claimed" not in kinds: unavailable.append(attempt + ": original admission unresolved") @@ -287,11 +353,18 @@ def retain(path: Path) -> bytes: raise OriginalEvidenceUnavailable("unsupported nested evidence profile") retain(path) status = read_fresh_status(state, host=True) - if status["status"] != "succeeded": + if preserve_outcomes: + # The actual native receipt follows this original claim. + # A missing claim is a provenance gap, not an unknown outcome. + retain(state / "fresh-launch-claimed.json") + if status["status"] not in ("succeeded", "unknown", "launched"): + raise OriginalEvidenceUnavailable("original first-launch boundary unavailable") + generations.append(generation(intent, str(status["status"]), tuple(sorted(expected.items())), owner_identity)) + elif status["status"] != "succeeded": unavailable.append(attempt + ": original outcome remains " + str(status["status"])) except (OSError, ValueError, TypeError, KeyError, sqlite3.Error) as error: unavailable.append(attempt + ": " + (str(error) or type(error).__name__)) - if not events: + if not events and not (preserve_outcomes and authority.genesis_path): unavailable.append("prospective enrollment has no original attempts; legacy coverage unavailable") # Re-read every byte and identity; no consumer verdict supplies stability. for original_path, snapshot in files.items(): @@ -309,7 +382,139 @@ def retain(path: Path) -> bytes: raise OriginalEvidenceUnavailable("original source changed during derivation") except (OSError, ValueError, TypeError, KeyError, sqlite3.Error) as error: unavailable.append(str(error) or type(error).__name__) - return OriginalAttemptEvidence(binding=binding, frontier=frontier, attempts=tuple(attempts), originals=tuple(sorted(files.values(), key=lambda item: item.path)), unavailable=tuple(unavailable)) + return OriginalAttemptEvidence(binding=binding, frontier=frontier, attempts=tuple(attempts), originals=tuple(sorted(files.values(), key=lambda item: item.path)), unavailable=tuple(unavailable)), tuple(generations) + + +def derive(binding: HistoryBinding, fence: int, owner_fences: tuple[int, ...] = ()) -> OriginalAttemptEvidence: + """Keep the prospective-attempt reader's existing outcome requirements.""" + return _derive(binding, fence, owner_fences)[0] + + +def derive_lineage(binding: HistoryBinding, fence: int, owner_fences: tuple[int, ...] = ()) -> OriginalLineageEvidence: + """Join original genesis and every retained attempted successor read-only.""" + from .original_genesis import LAUNCH_NAME, NATIVE_NAME, native_paths, validate_history_genesis, validate_native_genesis + + evidence, successors = _derive(binding, fence, owner_fences, preserve_outcomes=True) + originals = {item.path: item for item in evidence.originals} + unavailable = list(evidence.unavailable) + generations: list[OriginalGeneration] = [] + genesis = None + try: + _check_fence(binding, fence) + authority, events = read_frontier(Path(binding.root), binding) + if not authority.genesis_path or authority.profile != HISTORY_PROFILE: + raise OriginalEvidenceUnavailable("full retained lineage requires original prospective genesis; earlier history remains unavailable") + path = Path(authority.genesis_path) + genesis = validate_history_genesis(Path(binding.root)) + if genesis.binding != binding or binding.source != source_identity(): + raise OriginalEvidenceUnavailable("original genesis source/history differs") + state, journal, oauth = (genesis.root(role) for role in ("state", "journal", "oauth")) + owner_path = Path(state.host_path) / ".supervisor-owner.lock" + owner_identity = identity(owner_path) + matching = [fd for fd in owner_fences if (os.fstat(fd).st_dev, os.fstat(fd).st_ino) == owner_identity] + if len(matching) != 1: + raise OriginalEvidenceUnavailable("current original genesis owner fence unavailable") + _check_owner_fence(owner_path, matching[0], owner_identity) + snapshots = [path, owner_path, Path(state.host_path) / NATIVE_NAME, Path(state.host_path) / LAUNCH_NAME] + snapshots.extend(Path(item.path) for item in genesis.files) + namespaces = {} + for directory in (Path(state.host_path), Path(journal.host_path)): + namespaces[directory] = (identity(directory, directory=True), tuple(sorted(item.name for item in directory.iterdir()))) + for item in sorted(directory.iterdir()): + if directory == Path(state.host_path) and item.name == "logs": + info = item.lstat() + if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.geteuid(): + raise OriginalEvidenceUnavailable("original genesis diagnostics directory differs") + continue + if item.is_symlink() or item.is_dir(): + raise OriginalEvidenceUnavailable("unsupported nested original genesis evidence") + snapshots.append(item) + for original_path in snapshots: + observed = _snapshot(original_path) + if observed.path in originals and originals[observed.path] != observed: + raise OriginalEvidenceUnavailable("original genesis changed during derivation") + if any(item.path != observed.path and (item.device, item.inode) == (observed.device, observed.inode) for item in originals.values()): + raise OriginalEvidenceUnavailable("original lineage file identity aliases") + originals[observed.path] = observed + validate_native_genesis(genesis, path, host=True) + stores = native_paths(genesis, host=True) + for store in stores.values(): + _database(store) + # This is a factual original launch boundary. It never upgrades child, + # handoff, recovery or management-operation unknowns into success. + with closing(sqlite3.connect(stores["child"].as_uri() + "?mode=ro&immutable=1", uri=True)) as connection: + rows = connection.execute("SELECT instance_id,record_json FROM child_state").fetchall() + if len(rows) > 1 or rows and rows[0][0] != genesis.instance_id: + raise OriginalEvidenceUnavailable("original genesis child identity differs") + outcome = "unknown" if not rows else json.loads(rows[0][1])["phase"] + if outcome not in {"stopped", "starting", "running", "draining", "unknown", "needs_attention"}: + raise OriginalEvidenceUnavailable("original genesis child outcome is malformed") + generations.append( + OriginalGeneration( + genesis.authorization_id, + genesis.instance_id, + genesis.management_id, + genesis.management_image_id, + genesis.file("bootstrap").sha256, + genesis.file("management").sha256, + state.host_path, + state.device, + state.inode, + journal.host_path, + journal.device, + journal.inode, + oauth.host_path, + oauth.device, + oauth.inode, + str(owner_path), + *owner_identity, + outcome, + tuple(sorted((name, str(store)) for name, store in stores.items())), + genesis.proxy_id, + genesis.proxy_image_id, + ) + ) + complete = (generations[0], *successors) + expected_owners = {(item.owner_device, item.owner_inode) for item in complete if item.owner_inode > 0} + observed_owners = [(os.fstat(fd).st_dev, os.fstat(fd).st_ino) for fd in owner_fences] + if len(observed_owners) != len(expected_owners) or set(observed_owners) != expected_owners: + raise OriginalEvidenceUnavailable("original lineage owner descriptor set differs") + by_instance = {item.instance_id: item for item in complete} + for event in events: + if event.kind != "anchor": + continue + intent = parse_fresh_intent(json.loads(event.value_json)["intent"]) + generation = by_instance.get(intent.new_instance) + if generation is None or generation.outcome == "never-created": + continue + predecessor = by_instance.get(intent.old_instance) + if predecessor is None or (intent.old_container_id, intent.owner_lock_device, intent.owner_lock_inode) != (predecessor.management_id, predecessor.owner_device, predecessor.owner_inode): + raise OriginalEvidenceUnavailable("original native successor owner differs from its retained predecessor") + if (intent.host_oauth_root, intent.oauth_device, intent.oauth_inode) != (oauth.host_path, oauth.device, oauth.inode) or identity(Path(intent.host_oauth_root), directory=True) != (oauth.device, oauth.inode): + raise OriginalEvidenceUnavailable("original native successor retained OAuth root differs") + for original_name, expected in originals.items(): + if _snapshot(Path(original_name)) != expected: + raise OriginalEvidenceUnavailable("original lineage changed during derivation") + for directory, (expected_id, names) in namespaces.items(): + if identity(directory, directory=True) != expected_id or tuple(sorted(item.name for item in directory.iterdir())) != names: + raise OriginalEvidenceUnavailable("original genesis namespace changed during derivation") + if read_frontier(Path(binding.root), binding) != (authority, events) or validate_history_genesis(Path(binding.root)) != genesis: + raise OriginalEvidenceUnavailable("original lineage advanced during derivation") + _check_fence(binding, fence) + _check_owner_fence(owner_path, matching[0], owner_identity) + except (OSError, ValueError, TypeError, KeyError, sqlite3.Error) as error: + unavailable.append(str(error) or type(error).__name__) + return OriginalLineageEvidence(binding, HISTORY_PROFILE, genesis, evidence.frontier, evidence.attempts, tuple(generations) + successors, tuple(sorted(originals.values(), key=lambda item: item.path)), tuple(unavailable)) + + +def verify_lineage(submitted: OriginalLineageEvidence, expected: HistoryBinding, fence: int, owner_fences: tuple[int, ...] = ()) -> OriginalLineageEvidence: + """Independently reconstruct all originals at the consumer's current fence.""" + current = derive_lineage(expected, fence, owner_fences) + if current != submitted or submitted.binding != expected: + raise OriginalEvidenceUnavailable("submitted lineage is forged, stale, truncated or cross-deployment") + if current.unavailable: + raise OriginalEvidenceUnavailable("; ".join(current.unavailable)) + return current def verify(submitted: OriginalAttemptEvidence, expected: HistoryBinding, fence: int, owner_fences: tuple[int, ...] = ()) -> OriginalAttemptEvidence: diff --git a/src/auto_coder/management/original_attempts.py b/src/auto_coder/management/original_attempts.py index d0cdf3e06..3c8c08e37 100644 --- a/src/auto_coder/management/original_attempts.py +++ b/src/auto_coder/management/original_attempts.py @@ -44,6 +44,7 @@ class HistoryAuthority: profile: str root: tuple[int, int] lock: tuple[int, int] + genesis_path: str = "" @dataclass(frozen=True) @@ -187,7 +188,15 @@ def read_frontier(root: Path, expected: HistoryBinding | None = None) -> tuple[H """Validate the whole bounded append-only event namespace, never truncate.""" raw, _ = read_original(root / "authority.json") authority = json.loads(raw) - if not isinstance(authority, dict) or set(authority) != {"binding", "profile", "root", "lock"} or authority["profile"] != PROFILE or canonical(authority) != raw: + from .original_genesis import HISTORY_PROFILE, validate_history_genesis + + genesis = None + if isinstance(authority, dict) and authority.get("profile") == HISTORY_PROFILE: + genesis = validate_history_genesis(root) + fields = {"binding", "profile", "root", "lock", "genesis"} + else: + fields = {"binding", "profile", "root", "lock"} + if not isinstance(authority, dict) or set(authority) != fields or authority["profile"] not in (PROFILE, HISTORY_PROFILE) or canonical(authority) != raw: raise OriginalEvidenceUnavailable("unsupported history profile") binding = HistoryBinding(**authority["binding"]) if binding.root != str(root) or (expected is not None and binding != expected) or authority["root"] != list(identity(root, directory=True)) or authority["lock"] != list(identity(root / "history.lock")): @@ -289,10 +298,10 @@ def read_frontier(root: Path, expected: HistoryBinding | None = None) -> tuple[H # All prior anchors must form one admissible forward history, including # failed branches. Checking only a new claim against its immediate parent # would accept an unrelated rewritten ancestor elsewhere in the frontier. - baseline: tuple[str, str] | None = None - known_predecessors: set[tuple[str, str]] = set() - seen_targets: set[str] = set() - seen_roots: set[tuple[int, int]] = set() + baseline: tuple[str, str] | None = (genesis.instance_id, genesis.management_id) if genesis is not None else None + known_predecessors: set[tuple[str, str]] = {baseline} if baseline is not None else set() + seen_targets: set[str] = {item.host_path for item in genesis.roots} if genesis is not None else set() + seen_roots: set[tuple[int, int]] = {(item.device, item.inode) for item in genesis.roots} if genesis is not None else set() earlier_predecessor_names: set[str] = set() for event in events: if event.kind == "claimed": @@ -330,7 +339,7 @@ def read_frontier(root: Path, expected: HistoryBinding | None = None) -> tuple[H if previous != frontier["digest"]: raise OriginalEvidenceUnavailable("truncated original frontier") - return HistoryAuthority(binding, authority["profile"], tuple(authority["root"]), tuple(authority["lock"])), events + return HistoryAuthority(binding, authority["profile"], tuple(authority["root"]), tuple(authority["lock"]), authority["genesis"]["path"] if genesis is not None else ""), events def _append(root: Path, kind: str, attempt: str, value: object) -> None: @@ -377,13 +386,18 @@ def capture_claim(intent: FreshInitializationIntent) -> Iterator[None]: if authority.binding.history != intent.original_history_id or authority.binding.source != source_identity(): raise OriginalEvidenceUnavailable("original history source or enrollment differs") anchors = [event for event in events if event.kind == "anchor"] + genesis = None + if authority.genesis_path: + from .original_genesis import read_genesis + + genesis = read_genesis(Path(authority.genesis_path)) keys = {intent.new_instance, intent.new_container_id, intent.host_state_root, intent.host_journal_root} prior_targets = [json.loads(event.value_json)["intent"] for event in anchors] if any(event.attempt == intent.request_id for event in anchors) or any(keys & {prior["new_instance"], prior["new_container_id"], prior["host_state_root"], prior["host_journal_root"]} for prior in prior_targets): raise OriginalEvidenceUnavailable("original attempt or target already anchored; no replay") - if anchors: - baseline = json.loads(anchors[0].value_json)["intent"] - predecessors = [(baseline["old_instance"], baseline["old_container_id"])] + if anchors or genesis is not None: + baseline_pair = (genesis.instance_id, genesis.management_id) if genesis is not None else (json.loads(anchors[0].value_json)["intent"]["old_instance"], json.loads(anchors[0].value_json)["intent"]["old_container_id"]) + predecessors = [baseline_pair] for prior_event in anchors: prior_kinds = [event.kind for event in events if event.attempt == prior_event.attempt] # A refused, interrupted or merely anchored attempt never established @@ -391,7 +405,7 @@ def capture_claim(intent: FreshInitializationIntent) -> Iterator[None]: if prior_kinds in (["anchor", "claimed"], ["anchor", "claimed", "grant-possible"]): prior = json.loads(prior_event.value_json)["intent"] predecessors.append((prior["new_instance"], prior["new_container_id"])) - if (intent.old_instance, intent.old_container_id) not in predecessors or intent.new_instance == baseline["old_instance"] or intent.new_container_id == baseline["old_container_id"]: + if (intent.old_instance, intent.old_container_id) not in predecessors or intent.new_instance == baseline_pair[0] or intent.new_container_id == baseline_pair[1]: raise OriginalEvidenceUnavailable("uncovered, refused or cyclic original predecessor") if any(prior["old_instance"] == intent.new_instance for prior in prior_targets): raise OriginalEvidenceUnavailable("attempt would cycle covered ancestry") @@ -402,6 +416,8 @@ def capture_claim(intent: FreshInitializationIntent) -> Iterator[None]: if state_id != (intent.state_device, intent.state_inode) or journal_id != (intent.journal_device, intent.journal_inode): raise OriginalEvidenceUnavailable("intended root identities differ before anchor") prior_root_ids = {tuple(json.loads(event.value_json)[field]) for event in anchors for field in ("state", "journal")} + if genesis is not None: + prior_root_ids.update((item.device, item.inode) for item in genesis.roots) if state_id == journal_id or state_id in prior_root_ids or journal_id in prior_root_ids: raise OriginalEvidenceUnavailable("original target aliases a protected mounted root") pristine = not any(Path(intent.host_state_root).iterdir()) and not any(Path(intent.host_journal_root).iterdir()) diff --git a/src/auto_coder/management/original_genesis.py b/src/auto_coder/management/original_genesis.py new file mode 100644 index 000000000..00573c371 --- /dev/null +++ b/src/auto_coder/management/original_genesis.py @@ -0,0 +1,592 @@ +"""Prospective ordinary-deployment originals, never recovery or launch authority. + +A separately authorized capture observes two never-started local Docker objects. +The existing ordinary Supervisor records native store and first-launch boundaries +under its own original exclusive lock. Neither entry point converts old roots. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import sqlite3 +import stat +import sys +import uuid +from contextlib import closing +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import TYPE_CHECKING + +from loguru import logger + +from .original_attempts import HistoryBinding, OriginalEvidenceUnavailable, _create, _lifetime, _sync, canonical, history_lock, identity, read_original, source_identity, validate_original_lifetime + +if TYPE_CHECKING: + from .bootstrap_supervisor import BootstrapConfig, ManagementBootstrapSupervisor + +PROFILE = "ordinary-original-genesis-v1" +HISTORY_PROFILE = "ordinary-original-history-v1" +GENESIS_NAME = "original-genesis.json" +NATIVE_NAME = "original-genesis-stores.json" +LAUNCH_NAME = "original-genesis-first-launch.json" +HISTORY_MOUNT = Path("/management-original-history") +ROOTS_PROFILE = "ordinary-original-roots-v1" +ROOTS_NAME = "original-genesis-roots.json" +ROOT_CLAIM_NAME = "original-genesis-root-claim.json" + + +@dataclass(frozen=True) +class GenesisRootAuthorization: + """Select prospective paths before the first Docker object exists.""" + + profile: str + purpose: str + deployment_id: str + authorization_id: str + bootstrap_path: str + management_path: str + proxy_config_path: str + state_root: str + journal_root: str + oauth_root: str + history_root: str + + +@dataclass(frozen=True) +class GenesisSelection: + """The protected separate new-deployment authorization, not a verdict.""" + + profile: str + purpose: str + deployment_id: str + authorization_id: str + management_id: str + proxy_id: str + history_root: str + bootstrap_path: str + management_path: str + proxy_config_path: str + root_authority_path: str + + +@dataclass(frozen=True) +class GenesisFile: + role: str + path: str + device: int + inode: int + sha256: str + + +@dataclass(frozen=True) +class GenesisRoot: + role: str + host_path: str + container_path: str + device: int + inode: int + + +@dataclass(frozen=True) +class GenesisAnchor: + profile: str + binding: HistoryBinding + authorization_id: str + instance_id: str + engine_id: str + management_id: str + management_image_id: str + management_inspection_sha256: str + proxy_id: str + proxy_image_id: str + proxy_inspection_sha256: str + files: tuple[GenesisFile, ...] + roots: tuple[GenesisRoot, ...] + lifetime_json: str + + def file(self, role: str) -> GenesisFile: + matches = [item for item in self.files if item.role == role] + if len(matches) != 1: + raise OriginalEvidenceUnavailable("original genesis file role is incomplete") + return matches[0] + + def root(self, role: str) -> GenesisRoot: + matches = [item for item in self.roots if item.role == role] + if len(matches) != 1: + raise OriginalEvidenceUnavailable("original genesis root role is incomplete") + return matches[0] + + +def _file(role: str, path: Path) -> GenesisFile: + encoded, (device, inode) = read_original(path) + return GenesisFile(role, str(path), device, inode, hashlib.sha256(encoded).hexdigest()) + + +def _require_host_root() -> None: + if os.geteuid() != 0: + raise OriginalEvidenceUnavailable("ordinary original genesis producer requires host root") + + +def prepare_original_genesis_roots(authorization_path: Path) -> Path: + """Create the authorized original roots once, without any Docker effects. + + Preexisting paths are never enrolled. A partial claim remains permanently + unavailable, including when the directories it created still look empty. + """ + _require_host_root() + encoded, _ = read_original(authorization_path) + value = json.loads(encoded) + if not isinstance(value, dict) or set(value) != set(GenesisRootAuthorization.__dataclass_fields__) or canonical(value) != encoded or any(not isinstance(item, str) for item in value.values()): + raise OriginalEvidenceUnavailable("original root authorization fields differ") + request = GenesisRootAuthorization(**value) + if request.profile != ROOTS_PROFILE or request.purpose != "separately-authorized-new-deployment" or not re.fullmatch(r"[A-Za-z0-9_.:-]{1,128}", request.deployment_id) or str(uuid.UUID(request.authorization_id)) != request.authorization_id: + raise OriginalEvidenceUnavailable("explicit new-root authorization required") + bootstrap = Path(request.bootstrap_path).parent + identity(bootstrap, directory=True) + configurations = tuple(_file(role, Path(selected)) for role, selected in (("bootstrap", request.bootstrap_path), ("management", request.management_path), ("proxy", request.proxy_config_path))) + paths = tuple((role, Path(getattr(request, role + "_root")), container) for role, container in (("state", "/management-child-state"), ("journal", "/management-operations"), ("oauth", "/management-oauth"), ("history", str(HISTORY_MOUNT)))) + if any(a == b or a in b.parents or b in a.parents for index, a in enumerate([bootstrap, *(path for _, path, _ in paths)]) for b in [bootstrap, *(path for _, path, _ in paths)][index + 1 :]): + raise OriginalEvidenceUnavailable("authorized original root paths overlap") + for _, path, _ in paths: + if not path.is_absolute() or path != path.resolve() or path.exists() or path.is_symlink(): + raise OriginalEvidenceUnavailable("original root already exists; no empty-root conversion") + identity(path.parent, directory=True) + original_authorization = _file("root-authorization", authorization_path) + claim = {"profile": ROOTS_PROFILE, "authorization": asdict(original_authorization), "source": source_identity(), "request": value, "lifetime": _lifetime(), "configurations": [asdict(item) for item in configurations]} + _create(bootstrap, ROOT_CLAIM_NAME, claim) + roots = [] + for role, path, container in paths: + path.mkdir(mode=0o700) + _sync(path) + _sync(path.parent) + roots.append(GenesisRoot(role, str(path), container, *identity(path, directory=True))) + if tuple(_file(item.role, Path(item.path)) for item in configurations) != configurations or _file("root-authorization", authorization_path) != original_authorization: + raise OriginalEvidenceUnavailable("original root authorization changed during creation") + _create(bootstrap, ROOTS_NAME, {"profile": ROOTS_PROFILE, "claim": asdict(_file("root-claim", bootstrap / ROOT_CLAIM_NAME)), "roots": [asdict(item) for item in roots]}) + _read_root_authority(bootstrap / ROOTS_NAME) + logger.bind(original_boundary="genesis-roots", evidence_only=True).info("Original separately authorized deployment roots created: deployment={}", request.deployment_id) + return bootstrap / ROOTS_NAME + + +def _read_root_authority(path: Path) -> tuple[GenesisRootAuthorization, tuple[GenesisRoot, ...], tuple[GenesisFile, ...]]: + encoded, _ = read_original(path) + value = json.loads(encoded) + if not isinstance(value, dict) or set(value) != {"profile", "claim", "roots"} or value["profile"] != ROOTS_PROFILE or canonical(value) != encoded: + raise OriginalEvidenceUnavailable("original root creation receipt incomplete") + claim_file = GenesisFile(**value["claim"]) + if claim_file.role != "root-claim" or Path(claim_file.path) != path.parent / ROOT_CLAIM_NAME or _file("root-claim", Path(claim_file.path)) != claim_file: + raise OriginalEvidenceUnavailable("original root creation claim differs") + claim_encoded, _ = read_original(Path(claim_file.path)) + claim = json.loads(claim_encoded) + if not isinstance(claim, dict) or set(claim) != {"profile", "authorization", "source", "request", "lifetime", "configurations"} or claim["profile"] != ROOTS_PROFILE or canonical(claim) != claim_encoded or claim["source"] != source_identity(): + raise OriginalEvidenceUnavailable("original root producer source or fields differ") + validate_original_lifetime(claim["lifetime"]) + authorization = GenesisFile(**claim["authorization"]) + if authorization.role != "root-authorization" or _file("root-authorization", Path(authorization.path)) != authorization: + raise OriginalEvidenceUnavailable("original root authorization changed") + request = GenesisRootAuthorization(**claim["request"]) + if read_original(Path(authorization.path))[0] != canonical(asdict(request)) or path != Path(request.bootstrap_path).parent / ROOTS_NAME: + raise OriginalEvidenceUnavailable("original root authorization scope differs") + roots = tuple(GenesisRoot(**item) for item in value["roots"]) + if len(roots) != 4 or {item.role for item in roots} != {"state", "journal", "oauth", "history"} or len({(item.device, item.inode) for item in roots}) != 4: + raise OriginalEvidenceUnavailable("original root creation set is incomplete or aliased") + destinations = {"state": "/management-child-state", "journal": "/management-operations", "oauth": "/management-oauth", "history": str(HISTORY_MOUNT)} + for root in roots: + if root.host_path != getattr(request, root.role + "_root") or root.container_path != destinations[root.role] or identity(Path(root.host_path), directory=True) != (root.device, root.inode): + raise OriginalEvidenceUnavailable("original created root changed") + configurations = tuple(GenesisFile(**item) for item in claim["configurations"]) + if len(configurations) != 3 or {item.role for item in configurations} != {"bootstrap", "management", "proxy"}: + raise OriginalEvidenceUnavailable("original root configuration set is incomplete") + for item in configurations: + selected = request.proxy_config_path if item.role == "proxy" else getattr(request, item.role + "_path") + if item.path != selected or _file(item.role, Path(item.path)) != item: + raise OriginalEvidenceUnavailable("original root configuration changed") + return request, roots, (authorization, claim_file, _file("roots", path)) + + +def read_selection(path: Path) -> GenesisSelection: + encoded, _ = read_original(path) + value = json.loads(encoded) + if not isinstance(value, dict) or set(value) != set(GenesisSelection.__dataclass_fields__) or canonical(value) != encoded or any(not isinstance(item, str) for item in value.values()): + raise OriginalEvidenceUnavailable("genesis authorization fields or encoding differ") + selection = GenesisSelection(**value) + if selection.profile != PROFILE or selection.purpose != "separately-authorized-new-deployment" or not re.fullmatch(r"[A-Za-z0-9_.:-]{1,128}", selection.deployment_id) or str(uuid.UUID(selection.authorization_id)) != selection.authorization_id: + raise OriginalEvidenceUnavailable("explicit prospective new-deployment authorization required") + if any(not re.fullmatch(r"[0-9a-f]{64}", value) for value in (selection.management_id, selection.proxy_id)) or selection.management_id == selection.proxy_id: + raise OriginalEvidenceUnavailable("distinct exact original Docker IDs required") + for selected in (selection.history_root, selection.bootstrap_path, selection.management_path, selection.proxy_config_path, selection.root_authority_path): + candidate = Path(selected) + if not candidate.is_absolute() or candidate != candidate.resolve(strict=True): + raise OriginalEvidenceUnavailable("original genesis selection is not canonical") + return selection + + +def read_genesis(path: Path) -> GenesisAnchor: + encoded, _ = read_original(path) + raw = json.loads(encoded) + if not isinstance(raw, dict) or set(raw) != set(GenesisAnchor.__dataclass_fields__) or canonical(raw) != encoded or raw.get("profile") != PROFILE: + raise OriginalEvidenceUnavailable("original genesis schema or encoding differs") + if not isinstance(raw["binding"], dict) or set(raw["binding"]) != set(HistoryBinding.__dataclass_fields__) or any(not isinstance(value, str) for value in raw["binding"].values()): + raise OriginalEvidenceUnavailable("original genesis history binding differs") + binding = HistoryBinding(**raw["binding"]) + if str(uuid.UUID(binding.history)) != binding.history or not re.fullmatch(r"[A-Za-z0-9_.:-]{1,128}", binding.deployment) or not re.fullmatch(r"sha256:[0-9a-f]{64}", binding.source): + raise OriginalEvidenceUnavailable("original genesis source or history identity differs") + for key in ("management_id", "proxy_id", "management_inspection_sha256", "proxy_inspection_sha256"): + if not isinstance(raw[key], str) or not re.fullmatch(r"[0-9a-f]{64}", raw[key]): + raise OriginalEvidenceUnavailable("original genesis Docker identity differs") + for key in ("management_image_id", "proxy_image_id"): + if not isinstance(raw[key], str) or not re.fullmatch(r"sha256:[0-9a-f]{64}", raw[key]): + raise OriginalEvidenceUnavailable("original genesis image identity differs") + if raw["management_id"] == raw["proxy_id"] or any(not isinstance(raw[key], str) or not raw[key] or len(raw[key]) > 128 for key in ("instance_id", "engine_id")) or str(uuid.UUID(raw["authorization_id"])) != raw["authorization_id"]: + raise OriginalEvidenceUnavailable("original genesis deployment identity differs") + files = [] + if not isinstance(raw["files"], list) or len(raw["files"]) != 7: + raise OriginalEvidenceUnavailable("original genesis original files are incomplete") + for record in raw["files"]: + if not isinstance(record, dict) or set(record) != set(GenesisFile.__dataclass_fields__): + raise OriginalEvidenceUnavailable("original genesis original file fields differ") + if any(type(record[key]) is not int or record[key] < (1 if key == "inode" else 0) for key in ("device", "inode")) or not isinstance(record["path"], str) or not Path(record["path"]).is_absolute() or not isinstance(record["sha256"], str) or not re.fullmatch(r"[0-9a-f]{64}", record["sha256"]): + raise OriginalEvidenceUnavailable("original genesis original file identity differs") + files.append(GenesisFile(**record)) + roots = [] + if not isinstance(raw["roots"], list) or len(raw["roots"]) != 5: + raise OriginalEvidenceUnavailable("original genesis root set is incomplete") + for record in raw["roots"]: + if not isinstance(record, dict) or set(record) != set(GenesisRoot.__dataclass_fields__): + raise OriginalEvidenceUnavailable("original genesis root fields differ") + if any(type(record[key]) is not int or record[key] < (1 if key == "inode" else 0) for key in ("device", "inode")) or any(not isinstance(record[key], str) or not Path(record[key]).is_absolute() for key in ("host_path", "container_path")): + raise OriginalEvidenceUnavailable("original genesis root identity differs") + roots.append(GenesisRoot(**record)) + if {item.role for item in files} != {"authorization", "bootstrap", "management", "proxy", "root-authorization", "root-claim", "roots"} or {item.role for item in roots} != {"bootstrap", "state", "journal", "oauth", "history"}: + raise OriginalEvidenceUnavailable("original genesis roles are incomplete or duplicated") + if len({(item.device, item.inode) for item in files}) != len(files) or len({(item.device, item.inode) for item in roots}) != len(roots): + raise OriginalEvidenceUnavailable("original genesis identities alias") + if not isinstance(raw["lifetime_json"], str): + raise OriginalEvidenceUnavailable("original genesis producer lifetime missing") + validate_original_lifetime(json.loads(raw["lifetime_json"])) + return GenesisAnchor(**{**raw, "binding": binding, "files": tuple(files), "roots": tuple(roots)}) + + +def _created_pair(selection: GenesisSelection): + """Read the full local engine and the exact finite ordinary command profile.""" + from .containment_observation import ProposedScope, _docker + from .exclusion_host import _mount + from .fresh_host_observation import _supported_prior_command + from .installed_bootstrap_mount_profile import _inspect, _mounts, _unshadowed_bootstrap_execution + from .recovery_source_profile import read_image_source_identity + + scope = ProposedScope(selection.deployment_id, selection.authorization_id, selection.authorization_id, "ordinary-v1", selection.management_id, selection.proxy_id) + observed = _docker(scope) + selected = {item.container_id: item for item in observed if item.container_id in (selection.management_id, selection.proxy_id)} + if any(item.management_route_roles for item in observed if item.container_id not in selected): + raise OriginalEvidenceUnavailable("another original management/proxy Docker route is present") + for item in selected.values(): + if (item.status, item.running, item.pid, item.started_at, item.finished_at, item.restart_count, item.restart_policy, item.maximum_retry_count) != ("created", False, 0, "0001-01-01T00:00:00Z", "0001-01-01T00:00:00Z", 0, "no", 0): + raise OriginalEvidenceUnavailable("genesis requires exact created and never-started Docker objects") + management, proxy = _inspect(selection.management_id), _inspect(selection.proxy_id) + for object_id, inspected in ((selection.management_id, management), (selection.proxy_id, proxy)): + if hashlib.sha256(canonical([inspected])).hexdigest() != selected[object_id].inspection_sha256: + raise OriginalEvidenceUnavailable("Docker object changed during original capture") + host, config = inspected.get("HostConfig"), inspected.get("Config") + if not isinstance(host, dict) or not isinstance(config, dict) or host.get("Privileged") is not False or host.get("PidMode") not in ("", None) or host.get("CapAdd") not in (None, []) or host.get("Devices") not in (None, []) or config.get("User") != "": + raise OriginalEvidenceUnavailable("ordinary genesis container execution profile differs") + config = management["Config"] + assert isinstance(config, dict) + if not _supported_prior_command(config.get("Entrypoint"), config.get("Cmd")): + raise OriginalEvidenceUnavailable("genesis requires the ordinary bootstrap command") + mounts = _mounts(management) + _unshadowed_bootstrap_execution(config, mounts) + raw_mounts = management["Mounts"] + assert isinstance(raw_mounts, list) + if _mount(raw_mounts, "/bootstrap") / "config.json" != Path(selection.bootstrap_path) or _mount(raw_mounts, str(HISTORY_MOUNT)) != Path(selection.history_root): + raise OriginalEvidenceUnavailable("original genesis bootstrap or history mount differs") + if Path(selection.management_path) != Path(selection.bootstrap_path).parent / "management.yaml": + raise OriginalEvidenceUnavailable("original genesis management file is not the fixed bootstrap member") + proxy_config, proxy_host = proxy["Config"], proxy["HostConfig"] + assert isinstance(proxy_config, dict) and isinstance(proxy_host, dict) + if proxy_config.get("Entrypoint") != ["caddy"] or proxy_config.get("Cmd") != ["run", "--config", "/etc/caddy/Caddyfile", "--adapter", "caddyfile"] or proxy_host.get("NetworkMode") != "container:" + selection.management_id: + raise OriginalEvidenceUnavailable("original genesis proxy is not the exact ordinary paired command") + proxy_mounts = proxy["Mounts"] + assert isinstance(proxy_mounts, list) + if _mount(proxy_mounts, "/etc/caddy/Caddyfile") != Path(selection.proxy_config_path): + raise OriginalEvidenceUnavailable("original genesis proxy configuration mount differs") + actual_source = read_image_source_identity(selection.management_id) + if actual_source != source_identity(): + raise OriginalEvidenceUnavailable("created original bootstrap source differs from executing producer") + return observed, management, actual_source + + +def _validate_management_settings(config: BootstrapConfig, management_path: Path, bootstrap_path: Path) -> None: + """Bind the service's effective stores to the native original receipts.""" + from .oauth_config import load_management_settings, management_listener_binding_matches + + settings = load_management_settings(management_path) + if settings.bootstrap_config_path != bootstrap_path or settings.state_root != config.oauth_root or settings.operation_state_root != config.journal_root: + raise OriginalEvidenceUnavailable("ordinary genesis effective management roots differ") + if not management_listener_binding_matches(config, settings): + raise OriginalEvidenceUnavailable("ordinary genesis effective management listener differs") + + +def capture_original_genesis(authorization_path: Path) -> GenesisAnchor: + """Capture only; this operation never starts, configures or stops Docker.""" + from .bootstrap_supervisor import BootstrapConfig + from .exclusion_host import _mount + + _require_host_root() + selection = read_selection(authorization_path) + request, created_roots, root_files = _read_root_authority(Path(selection.root_authority_path)) + if any(getattr(request, name) != getattr(selection, name) for name in ("purpose", "deployment_id", "authorization_id", "history_root", "bootstrap_path", "management_path", "proxy_config_path")): + raise OriginalEvidenceUnavailable("original root authorization differs from selected deployment") + config = BootstrapConfig.read(Path(selection.bootstrap_path)) + if config.original_genesis != Path("/bootstrap") / GENESIS_NAME or config.fresh_guard_path is not None or config.new_instance_initialization or config.archived_exclusions: + raise OriginalEvidenceUnavailable("ordinary bootstrap did not explicitly select original genesis") + if (str(config.state_root), str(config.journal_root), str(config.oauth_root)) != ("/management-child-state", "/management-operations", "/management-oauth"): + raise OriginalEvidenceUnavailable("ordinary genesis retained root mapping differs") + runtime = next(item for item in config.runtimes if item.reference == config.initial_runtime) + if runtime.executable != Path("/opt/auto-coder/.venv/bin/auto-coder") or runtime.arguments != ("management", "serve", "--config", "/bootstrap/management.yaml") or runtime.execution_kind != "source-script" or config.management_config not in (None, Path("/bootstrap/management.yaml")): + raise OriginalEvidenceUnavailable("ordinary genesis management runtime differs") + _validate_management_settings(config, Path(selection.management_path), Path("/bootstrap/config.json")) + files = (*tuple(_file(role, path) for role, path in (("authorization", authorization_path), ("bootstrap", Path(selection.bootstrap_path)), ("management", Path(selection.management_path)), ("proxy", Path(selection.proxy_config_path)))), *root_files) + observed, management, source = _created_pair(selection) + mounts = management["Mounts"] + assert isinstance(mounts, list) + roots = tuple( + GenesisRoot(role, str(host), str(container), *identity(host, directory=True)) + for role, host, container in ( + ("bootstrap", Path(selection.bootstrap_path).parent, Path("/bootstrap")), + ("state", _mount(mounts, str(config.state_root), rw=True), config.state_root), + ("journal", _mount(mounts, str(config.journal_root), rw=True), config.journal_root), + ("oauth", _mount(mounts, str(config.oauth_root), rw=True), config.oauth_root), + ("history", Path(selection.history_root), HISTORY_MOUNT), + ) + ) + host_roots = [Path(item.host_path) for item in roots] + if tuple(item for item in roots if item.role != "bootstrap") != created_roots: + raise OriginalEvidenceUnavailable("Docker mounts differ from affirmative original root creation") + if any(a == b or a in b.parents or b in a.parents for index, a in enumerate(host_roots) for b in host_roots[index + 1 :]): + raise OriginalEvidenceUnavailable("original genesis roots overlap") + for root in roots: + if root.role != "bootstrap" and any(Path(root.host_path).iterdir()): + raise OriginalEvidenceUnavailable("new separately authorized genesis roots must be empty") + genesis_path = Path(selection.bootstrap_path).parent / GENESIS_NAME + if genesis_path.exists() or genesis_path.is_symlink(): + raise OriginalEvidenceUnavailable("original genesis already captured or interrupted; no reenrollment") + history_root = Path(selection.history_root) + # The first durable claim serializes capture and permanently poisons a partial + # publication. Existing roots never acquire a replacement original authority. + _create(history_root, "history.lock", {}) + with history_lock(history_root, exclusive=True): + binding = HistoryBinding(str(history_root), selection.deployment_id, str(uuid.uuid4()), source) + selected = {item.container_id: item for item in observed} + manager, proxy = selected[selection.management_id], selected[selection.proxy_id] + anchor = GenesisAnchor(PROFILE, binding, selection.authorization_id, config.instance_id, manager.engine_id, manager.container_id, manager.image_id, manager.inspection_sha256, proxy.container_id, proxy.image_id, proxy.inspection_sha256, files, roots, canonical(_lifetime()).decode()) + if _created_pair(selection) != (observed, management, source) or tuple(_file(item.role, Path(item.path)) for item in files) != files: + raise OriginalEvidenceUnavailable("original genesis inputs changed during capture") + for root in roots: + if identity(Path(root.host_path), directory=True) != (root.device, root.inode): + raise OriginalEvidenceUnavailable("original genesis root changed during capture") + allowed = {"history.lock"} if root.role == "history" else set() + if root.role != "bootstrap" and {item.name for item in Path(root.host_path).iterdir()} != allowed: + raise OriginalEvidenceUnavailable("original genesis root acquired earlier records") + _create(genesis_path.parent, genesis_path.name, asdict(anchor)) + original = _file("genesis", genesis_path) + _create(history_root, "authority.json", {"binding": asdict(binding), "profile": HISTORY_PROFILE, "root": identity(history_root, directory=True), "lock": identity(history_root / "history.lock"), "genesis": asdict(original)}) + _create(history_root, "frontier.json", {"count": 0, "digest": ""}) + validate_history_genesis(history_root) + logger.bind(original_history=str(history_root), original_boundary="genesis", evidence_only=True).info("Original ordinary deployment genesis captured: deployment={} history={}", binding.deployment, binding.history) + return anchor + + +def validate_history_genesis(root: Path, *, mounted_genesis: Path | None = None) -> GenesisAnchor: + """Read the original authority and its exact genesis, with no store creation.""" + encoded, _ = read_original(root / "authority.json") + value = json.loads(encoded) + if not isinstance(value, dict) or set(value) != {"binding", "profile", "root", "lock", "genesis"} or value.get("profile") != HISTORY_PROFILE or canonical(value) != encoded: + raise OriginalEvidenceUnavailable("ordinary original history authority is incomplete") + original = value["genesis"] + if not isinstance(original, dict) or set(original) != set(GenesisFile.__dataclass_fields__) or original.get("role") != "genesis": + raise OriginalEvidenceUnavailable("ordinary original genesis pointer is incomplete") + path = mounted_genesis or Path(original["path"]) + actual = _file("genesis", path) + if (actual.device, actual.inode, actual.sha256) != (original["device"], original["inode"], original["sha256"]) or mounted_genesis is None and actual.path != original["path"]: + raise OriginalEvidenceUnavailable("original genesis inode or bytes changed") + anchor = read_genesis(path) + if value["binding"] != asdict(anchor.binding) or value["root"] != list(identity(root, directory=True)) or value["lock"] != list(identity(root / "history.lock")): + raise OriginalEvidenceUnavailable("original genesis/history identities differ") + history = anchor.root("history") + if (history.device, history.inode) != identity(root, directory=True) or history.host_path != anchor.binding.root: + raise OriginalEvidenceUnavailable("original genesis history root differs") + if mounted_genesis is None and (str(root) != anchor.binding.root or str(path) != str(Path(anchor.file("bootstrap").path).parent / GENESIS_NAME)): + raise OriginalEvidenceUnavailable("original genesis authority path differs") + if mounted_genesis is None: + _, created_roots, _ = _read_root_authority(Path(anchor.file("roots").path)) + if tuple(item for item in anchor.roots if item.role != "bootstrap") != created_roots: + raise OriginalEvidenceUnavailable("original genesis roots differ from creation authority") + for original_file in anchor.files: + if _file(original_file.role, Path(original_file.path)) != original_file: + raise OriginalEvidenceUnavailable("original genesis configuration or authorization changed") + for original_root in anchor.roots: + if identity(Path(original_root.host_path), directory=True) != (original_root.device, original_root.inode): + raise OriginalEvidenceUnavailable("original genesis protected root changed") + return anchor + + +def native_paths(anchor: GenesisAnchor, *, host: bool) -> dict[str, Path]: + state, journal = anchor.root("state"), anchor.root("journal") + state_path = Path(state.host_path if host else state.container_path) + journal_path = Path(journal.host_path if host else journal.container_path) + return {"child": state_path / "management-child.sqlite3", "handoff": state_path / "management-self-update.sqlite3", "recovery": state_path / "management-recovery.sqlite3", "operations": journal_path / "management-operations.sqlite3"} + + +def validate_native_genesis(anchor: GenesisAnchor, genesis_path: Path, *, host: bool) -> tuple[dict[str, object], dict[str, object]]: + """Check original native identities; an unknown outcome is not missing history.""" + state = Path(anchor.root("state").host_path if host else anchor.root("state").container_path) + encoded, _ = read_original(state / NATIVE_NAME) + native = json.loads(encoded) + expected_keys = {"profile", "history", "source", "genesis_sha256", "instance_id", "lifetime", "owner_fence", "stores"} + if not isinstance(native, dict) or set(native) != expected_keys or canonical(native) != encoded: + raise OriginalEvidenceUnavailable("native genesis original receipt is incomplete") + validate_original_lifetime(native["lifetime"]) + if (native["profile"], native["history"], native["source"], native["genesis_sha256"], native["instance_id"]) != (PROFILE, anchor.binding.history, anchor.binding.source, _file("genesis", genesis_path).sha256, anchor.instance_id): + raise OriginalEvidenceUnavailable("native genesis source/history identity differs") + if native["owner_fence"] != list(identity(state / ".supervisor-owner.lock")) or native["stores"] != {name: list(identity(path)) for name, path in native_paths(anchor, host=host).items()}: + raise OriginalEvidenceUnavailable("native genesis original owner/store inode differs") + launch_encoded, _ = read_original(state / LAUNCH_NAME) + launch = json.loads(launch_encoded) + if not isinstance(launch, dict) or set(launch) != {"profile", "history", "genesis_sha256", "native_sha256", "operation_id", "attempt_id", "lifetime"} or canonical(launch) != launch_encoded: + raise OriginalEvidenceUnavailable("original genesis first-launch boundary is incomplete") + if launch["profile"] != PROFILE or launch["history"] != anchor.binding.history or launch["genesis_sha256"] != native["genesis_sha256"] or launch["native_sha256"] != hashlib.sha256(encoded).hexdigest() or launch["lifetime"] != native["lifetime"]: + raise OriginalEvidenceUnavailable("original genesis first-launch boundary differs") + if not isinstance(launch["operation_id"], str) or not launch["operation_id"].startswith("bootstrap-") or str(uuid.UUID(launch["operation_id"][10:])) != launch["operation_id"][10:] or str(uuid.UUID(launch["attempt_id"])) != launch["attempt_id"]: + raise OriginalEvidenceUnavailable("original genesis first-launch identity differs") + return native, launch + + +def _validate_bootstrap_inputs(config: BootstrapConfig, bootstrap_path: Path) -> GenesisAnchor: + """Validate only original inputs while the caller holds history SH.""" + if config.original_genesis is None: + raise OriginalEvidenceUnavailable("ordinary bootstrap genesis selection missing") + anchor = validate_history_genesis(HISTORY_MOUNT, mounted_genesis=config.original_genesis) + if anchor.binding.source != source_identity() or anchor.instance_id != config.instance_id: + raise OriginalEvidenceUnavailable("ordinary bootstrap source or instance differs from genesis") + for root in anchor.roots: + if identity(Path(root.container_path), directory=True) != (root.device, root.inode): + raise OriginalEvidenceUnavailable("ordinary bootstrap original mounted root differs") + for role, path in (("bootstrap", bootstrap_path), ("management", Path("/bootstrap/management.yaml"))): + original = anchor.file(role) + actual = _file(role, path) + if (actual.device, actual.inode, actual.sha256) != (original.device, original.inode, original.sha256): + raise OriginalEvidenceUnavailable("ordinary bootstrap original configuration differs") + if (config.state_root, config.journal_root, config.oauth_root) != tuple(Path(anchor.root(role).container_path) for role in ("state", "journal", "oauth")): + raise OriginalEvidenceUnavailable("ordinary bootstrap retained paths differ from genesis") + _validate_management_settings(config, Path(anchor.root("bootstrap").container_path) / "management.yaml", bootstrap_path) + from .bootstrap_supervisor import _verify_fresh_console_entrypoint + + runtime = next(item for item in config.runtimes if item.reference == config.initial_runtime) + if not runtime.verify(initial=True): + raise OriginalEvidenceUnavailable("ordinary bootstrap original console receipt differs") + _verify_fresh_console_entrypoint(runtime) + return anchor + + +def validate_bootstrap_genesis(config: BootstrapConfig, bootstrap_path: Path) -> GenesisAnchor: + """Refuse missing/replaced originals before even a diagnostic directory.""" + with history_lock(HISTORY_MOUNT, exclusive=False): + return _validate_bootstrap_inputs(config, bootstrap_path) + + +def prepare_original_genesis(supervisor: ManagementBootstrapSupervisor, owner_start_token: str) -> tuple[str, str] | None: + """Called only by the actual ordinary bootstrap under its own exclusive lock.""" + from .fresh_host_observation import _assert_pid_flock + from .operation_store import ManagementOperationStore + from .recovery_mailbox import RecoveryMailbox + from .self_update_handoff import HandoffStore + + config = supervisor.config + if config.original_genesis is None or supervisor.bootstrap_config_path is None: + raise OriginalEvidenceUnavailable("ordinary bootstrap genesis selection missing") + descriptor = os.open(config.state_root / ".supervisor-owner.lock", os.O_RDONLY | os.O_NOFOLLOW) + try: + _assert_pid_flock(os.getpid(), descriptor, "WRITE") + finally: + os.close(descriptor) + with history_lock(HISTORY_MOUNT, exclusive=False): + anchor = _validate_bootstrap_inputs(config, supervisor.bootstrap_config_path) + native_path, launch_path = config.state_root / NATIVE_NAME, config.state_root / LAUNCH_NAME + if native_path.exists() or native_path.is_symlink() or launch_path.exists() or launch_path.is_symlink(): + validate_native_genesis(anchor, config.original_genesis, host=False) + return None + frontier, _ = read_original(HISTORY_MOUNT / "frontier.json") + if frontier != canonical({"count": 0, "digest": ""}) or {path.name for path in HISTORY_MOUNT.iterdir()} != {"history.lock", "authority.json", "frontier.json"}: + raise OriginalEvidenceUnavailable("ordinary genesis first launch has an advanced or partial history") + allowed = {".supervisor-owner.lock", "logs"} + if any(path.name not in allowed or path.is_symlink() for path in config.state_root.iterdir()) or any(config.journal_root.iterdir()) or any(config.oauth_root.iterdir()): + raise OriginalEvidenceUnavailable("ordinary genesis cannot initialize earlier or interrupted roots") + logs = config.state_root / "logs" + if logs.exists() and (not logs.is_dir() or logs.lstat().st_uid != os.geteuid()): + raise OriginalEvidenceUnavailable("ordinary genesis diagnostics directory differs") + previous_umask = os.umask(0o077) + try: + supervisor.store.publish_supervisor(supervisor.incarnation, os.getpid(), owner_start_token) + supervisor.store.require_existing = True + with closing(ManagementOperationStore(config.journal_root)._connect()): + pass + with closing(HandoffStore(config.state_root, config.instance_id)._connect()): + pass + RecoveryMailbox(config.state_root, config.instance_id).initialize() + for path in native_paths(anchor, host=False).values(): + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + _sync(config.state_root) + _sync(config.journal_root) + lifetime = _lifetime() + native = { + "profile": PROFILE, + "history": anchor.binding.history, + "source": anchor.binding.source, + "genesis_sha256": _file("genesis", config.original_genesis).sha256, + "instance_id": anchor.instance_id, + "lifetime": lifetime, + "owner_fence": identity(config.state_root / ".supervisor-owner.lock"), + "stores": {name: identity(path) for name, path in native_paths(anchor, host=False).items()}, + } + _create(config.state_root, NATIVE_NAME, native) + logger.bind(original_history=anchor.binding.root, original_boundary="genesis-stores", evidence_only=True).info("Original ordinary bootstrap stores captured: history={}", anchor.binding.history) + operation, attempt = "bootstrap-" + str(uuid.uuid4()), str(uuid.uuid4()) + _create(config.state_root, LAUNCH_NAME, {"profile": PROFILE, "history": anchor.binding.history, "genesis_sha256": native["genesis_sha256"], "native_sha256": hashlib.sha256(canonical(native)).hexdigest(), "operation_id": operation, "attempt_id": attempt, "lifetime": lifetime}) + validate_native_genesis(anchor, config.original_genesis, host=False) + logger.bind(original_history=anchor.binding.root, original_boundary="genesis-first-launch", evidence_only=True).info("Original ordinary bootstrap first-launch boundary recorded: history={}", anchor.binding.history) + return operation, attempt + finally: + os.umask(previous_umask) + + +def main() -> int: + from auto_coder.logger_config import setup_logger + + parser = argparse.ArgumentParser(description="Capture separately authorized ordinary genesis originals; no Docker effects or recovery authority") + mode = parser.add_mutually_exclusive_group(required=True) + mode.add_argument("--authorization", type=Path) + mode.add_argument("--prepare-roots", type=Path) + parser.add_argument("--log-file", required=True, type=Path) + args = parser.parse_args() + setup_logger(stream=sys.stdout, log_file=str(args.log_file)) + try: + if args.prepare_roots is not None: + print(json.dumps({"original_root_authority": str(prepare_original_genesis_roots(args.prepare_roots))}, sort_keys=True)) + else: + print(json.dumps(asdict(capture_original_genesis(args.authorization)), sort_keys=True)) + return 0 + except (OSError, ValueError, TypeError, KeyError, sqlite3.Error) as error: + parser.exit(1, "original ordinary genesis unavailable: " + str(error) + "\n") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/auto_coder/management/recovery_containment.py b/src/auto_coder/management/recovery_containment.py new file mode 100644 index 000000000..bedb1742b --- /dev/null +++ b/src/auto_coder/management/recovery_containment.py @@ -0,0 +1,580 @@ +"""Independently repeatable current ordinary-recovery containment observations. + +This read-only producer neither enrolls a scope, inhibits a controller, acquires +an owner fence, repairs history, creates admission nor grants recovery authority. +The fixed protected selection and every current technical observation are joined +here; the epoch issuer separately establishes complete original history and READY. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import select +import sqlite3 +import stat +import subprocess +import sys +import tarfile +import time +import uuid +from contextlib import ExitStack +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import cast + +from auto_coder.logger_config import get_logger, setup_logger + +from .archived_exclusions import unique_pairs +from .containment_enrollment import OriginalFile, _approved_directory, _file_identity, _protected, _trusted_ancestors +from .containment_kernel_subobservation import _holder_start_token, _require_capture_host, capture_original_caddy_container +from .containment_observation import DockerObservation, observe_recovery_docker +from .containment_route_manifest import RecoveryRouteScope, RecoveryRouteSelection, RouteManifestObservation, observe_recovery_route_manifest +from .docker_inspection import canonical_legacy_inspection +from .fresh_host import _evidence +from .fresh_host_observation import ( + PriorContainer, + _assert_pid_flock, + _capture_prior_kernel, + _capture_supervisor_identity, + _cgroup_tasks, + _engine_id, + _inspect, + assert_container_published_listener, + assert_host_listener_absent, + assert_prior_network_unreferenced, + assert_prior_processes_absent, + capture_prior_container, +) +from .installed_bootstrap_mount_profile import _inspect as _full_inspect +from .installed_bootstrap_mount_profile import _mounts +from .original_attempts import source_identity +from .proxy_reconnect import ProxyBinding +from .recovery_containment_model import ( + RECOVERY_ENROLLMENT_ROOT, + NamedDescriptor, + RecoveryContainmentUnavailable, + RecoveryEnrollment, + RecoveryEnrollmentObservation, + RecoveryFile, + RecoveryOriginal, + RecoveryResources, + RecoveryTarget, +) +from .recovery_containment_profile import RecoveryManagerFact, _mapped_file, assert_disjoint_target, digest, read_bound_file, validate_manager_profile, validate_proxy_profile +from .recovery_containment_resources import RecoveryResourceObservation, _private_lock, hold_observed_resources +from .recovery_diagnostics import refusal_causes +from .recovery_watcher_source import RecoveryWatcherUnitFact, observe_recovery_watcher_unit + +logger = get_logger(__name__) +_REPORT_SCHEMA = "current-recovery-containment-v1" +_STATUS = "observed-current-recovery-containment" +_MAX_BYTES = 8 * 1024 * 1024 + + +@dataclass(frozen=True) +class RecoveryKernelFact: + generation_id: str + current_state: str + boot_id: str + manager_pid: int + manager_start_token: str + proxy_pid: int + proxy_start_token: str + original_namespace: str + owner_device: int + owner_inode: int + published_port: int + original_cgroups_empty: bool + original_namespace_unoccupied: bool + original_listener_absent: bool + + +@dataclass(frozen=True) +class RecoveryTargetFact: + container_id: str + pid: int + start_token: str + boot_id: str + network_namespace: str + pid_namespace: str + cgroup_path: str + cgroup_device: int + cgroup_inode: int + owner_device: int + owner_inode: int + published_port: int + + +@dataclass(frozen=True) +class RecoveryWatcherFact: + unit: str + manager_id: str + proxy_id: str + configuration: RecoveryFile + source: RecoveryFile + installed_unit: RecoveryWatcherUnitFact + lock_device: int + lock_inode: int + + +@dataclass(frozen=True) +class RecoveryContainmentFacts: + enrollment: RecoveryEnrollmentObservation + source_identity: str + docker: tuple[DockerObservation, ...] + managers: tuple[RecoveryManagerFact, ...] + proxy_inspections: tuple[tuple[str, str], ...] + kernel: tuple[RecoveryKernelFact, ...] + target: RecoveryTargetFact + resources: RecoveryResourceObservation + routes: RouteManifestObservation + watchers: tuple[RecoveryWatcherFact, ...] + effective_files: tuple[RecoveryFile, ...] + + +@dataclass(frozen=True) +class RecoveryContainmentReport: + schema: str + status: str + observation_id: str + observer_pid: int + observer_start_token: str + boot_id: str + started_monotonic_ns: int + finished_monotonic_ns: int + facts: RecoveryContainmentFacts + evidence_sha256: str + grant_issued: bool = False + history_asserted: bool = False + admission_created: bool = False + + @property + def managers(self) -> tuple[RecoveryManagerFact, ...]: + return self.facts.managers + + @property + def docker_ids(self) -> tuple[str, ...]: + return tuple(item.container_id for item in self.facts.docker) + + @property + def target_pid(self) -> int: + return self.facts.target.pid + + @property + def target_start_token(self) -> str: + return self.facts.target.start_token + + @property + def enrollment(self) -> RecoveryEnrollmentObservation: + return self.facts.enrollment + + +def _load_enrollment(enrollment_id: str, *, root: Path, expected_uid: int) -> RecoveryEnrollmentObservation: + """Controlled protected fixture seam; the production factory fixes both.""" + from .recovery_containment_model import _IDENTIFIER + + if not isinstance(enrollment_id, str) or not _IDENTIFIER.fullmatch(enrollment_id) or os.geteuid() != expected_uid: + raise RecoveryContainmentUnavailable("invalid protected recovery enrollment selector") + _approved_directory(root, uid=expected_uid) + path = root / (enrollment_id + ".json") + content, info = _protected(path, uid=expected_uid, size_limit=1024 * 1024) + if info.st_nlink != 1: + raise RecoveryContainmentUnavailable("recovery enrollment has an ambiguous file identity") + selected = RecoveryEnrollment.parse(json.loads(content, object_pairs_hook=unique_pairs)) + if selected.enrollment_id != enrollment_id: + raise RecoveryContainmentUnavailable("protected recovery enrollment selector differs") + return RecoveryEnrollmentObservation(selected, hashlib.sha256(content).hexdigest(), info.st_dev, info.st_ino, info.st_ctime_ns) + + +def load_enrollment(enrollment_id: str) -> RecoveryEnrollmentObservation: + """Load only the fixed trusted-host scope; no request/environment override.""" + _require_capture_host() + return _load_enrollment(enrollment_id, root=RECOVERY_ENROLLMENT_ROOT, expected_uid=0) + + +def route_selection(approved: RecoveryEnrollmentObservation) -> RecoveryRouteSelection: + """Pass the full independently protected recovery scope to shared readers.""" + selected = approved.enrollment + originals: list[OriginalFile] = [] + for item in selected.originals: + originals.extend((OriginalFile("bootstrap-config", **asdict(item.bootstrap_config)), OriginalFile("management-config", **asdict(item.management_config)), OriginalFile("proxy-config", **asdict(item.proxy_config)))) + originals.extend((OriginalFile("bootstrap-config", **asdict(selected.target.bootstrap_config)), OriginalFile("management-config", **asdict(selected.target.management_config)))) + for watcher in selected.watchers: + originals.extend((OriginalFile("watcher-source", **asdict(watcher.source)), OriginalFile("watcher-config", **asdict(watcher.configuration)), OriginalFile("support-source", **asdict(watcher.unit_source)))) + scope = RecoveryRouteScope(selected.deployment_id, selected.enrollment_id, selected.authorization_id, selected.profile, selected.container_ids, tuple(originals)) + return RecoveryRouteSelection(scope, approved.sha256) + + +def _engine(enrollment: RecoveryEnrollment) -> tuple[DockerObservation, ...]: + managers = tuple(item.management_id for item in enrollment.originals) + (enrollment.target.management_id,) + proxies = tuple(item.proxy_id for item in enrollment.originals) + docker = observe_recovery_docker(managers, proxies) + selected = set(enrollment.container_ids) + if any(item.engine_id != enrollment.engine_id or item.management_route_roles and item.container_id not in selected for item in docker): + raise RecoveryContainmentUnavailable("foreign, unknown or omitted recovery management candidate") + if any(item.container_id in selected and (item.restart_policy != "no" or item.maximum_retry_count != 0) for item in docker): + raise RecoveryContainmentUnavailable("an ordinary recovery Docker restart route remains enabled") + return docker + + +def _original_kernel(original: RecoveryOriginal, manager: RecoveryManagerFact, current: dict[str, DockerObservation]) -> RecoveryKernelFact: + if original.state == "never-started": + for identifier in (original.management_id, original.proxy_id): + item = current[identifier] + if item.running or item.status != "created" or item.pid != 0 or item.restart_count != 0 or item.started_at != "0001-01-01T00:00:00Z" or item.finished_at != "0001-01-01T00:00:00Z": + raise RecoveryContainmentUnavailable("selected never-started Docker object has a lifetime") + assert_host_listener_absent(manager.published_port) + return RecoveryKernelFact(original.generation_id, "currently-created-never-started", "", 0, "", 0, "", "", 0, 0, manager.published_port, False, False, True) + if original.prior_evidence is None or original.owner is None or original.namespace_pin is None: + raise RecoveryContainmentUnavailable("original lifetime evidence or resource identity is missing") + read_bound_file(original.prior_evidence) + prior, proxy, prior_sha = _evidence(Path(original.prior_evidence.path)) + if prior_sha != original.prior_evidence.sha256 or original.boot_id != Path("/proc/sys/kernel/random/boot_id").read_text(encoding="ascii").strip(): + raise RecoveryContainmentUnavailable("original evidence differs or came from another boot") + if ( + prior.engine_id != proxy.engine_id + or prior.engine_id != current[original.management_id].engine_id + or (prior.container_id, prior.image_id, prior.instance_id) != (original.management_id, original.management_image_id, original.instance_id) + or (proxy.container_id, proxy.image_id, proxy.network_mode) != (original.proxy_id, original.proxy_image_id, "container:" + original.management_id) + or prior.net_namespace != proxy.net_namespace + or prior.net_namespace != original.namespace_pin.namespace + or (prior.owner_lock_device, prior.owner_lock_inode) != (original.owner.device, original.owner.inode) + or (prior.bootstrap_source, prior.bootstrap_sha256, prior.published_port) != (original.bootstrap_config.path, original.bootstrap_config.sha256, manager.published_port) + or prior.oauth_source != manager.oauth_root + ): + raise RecoveryContainmentUnavailable("recovery captured original scope differs") + oauth = Path(manager.oauth_root).stat() + if (oauth.st_dev, oauth.st_ino) != (prior.oauth_device, prior.oauth_inode): + raise RecoveryContainmentUnavailable("recovery captured OAuth object was replaced") + for lifetime in (prior, proxy): + observed = current[lifetime.container_id] + mounts = tuple(sorted((item.source, item.destination, item.writable) for item in original.mounts if item.container_id == lifetime.container_id)) + mount_sha = hashlib.sha256(json.dumps(mounts, separators=(",", ":")).encode()).hexdigest() + if observed.running or observed.pid != 0 or observed.status != "exited" or observed.image_id != lifetime.image_id or observed.started_at != lifetime.started_at or observed.restart_count != lifetime.restart_count or mount_sha != lifetime.mounts_sha256: + raise RecoveryContainmentUnavailable("recovery original Docker lifetime/configuration changed") + assert_prior_processes_absent(lifetime) + assert_prior_network_unreferenced(prior.net_namespace) + assert_host_listener_absent(prior.published_port) + read_bound_file(original.prior_evidence) + return RecoveryKernelFact(original.generation_id, "original-lifetimes-absent", original.boot_id, prior.pid, prior.start_token, proxy.pid, proxy.start_token, prior.net_namespace, original.owner.device, original.owner.inode, prior.published_port, True, True, True) + + +def _target_kernel(enrollment: RecoveryEnrollment, current: dict[str, DockerObservation], kernels: tuple[RecoveryKernelFact, ...]) -> RecoveryTargetFact: + target = enrollment.target + observed = current[target.management_id] + if not observed.running or observed.status != "running" or observed.pid <= 1 or observed.image_id != target.management_image_id: + raise RecoveryContainmentUnavailable("exact recovery target is not running its custody entrypoint") + proc = Path("/proc") / str(observed.pid) + with ExitStack() as stack: + lifetime = os.pidfd_open(observed.pid, 0) + stack.callback(os.close, lifetime) + poller = select.poll() + poller.register(lifetime, select.POLLIN) + token = _holder_start_token(observed.pid) + network, pid_namespace = os.readlink(proc / "ns/net"), os.readlink(proc / "ns/pid") + if network in {item.original_namespace for item in kernels} or network == os.readlink("/proc/1/ns/net") or pid_namespace == os.readlink("/proc/1/ns/pid"): + raise RecoveryContainmentUnavailable("recovery target namespace aliases an original or the host") + command = (proc / "cmdline").read_bytes().rstrip(b"\0").split(b"\0") + if not command or command[0] not in (b"python", b"/opt/auto-coder/.venv/bin/python") or command[1:] != [b"-m", b"auto_coder.management.bootstrap_supervisor", b"--config", b"/bootstrap/config.json", b"--recovery-epoch"]: + raise RecoveryContainmentUnavailable("target process is not the exact prelaunch consumer") + expected_cgroup = "/system.slice/docker-" + target.management_id + ".scope" + if (proc / "cgroup").read_text(encoding="ascii").splitlines() != ["0::" + expected_cgroup]: + raise RecoveryContainmentUnavailable("recovery target cgroup differs") + cgroup = Path("/sys/fs/cgroup") / expected_cgroup.lstrip("/") + info = cgroup.stat() + if _cgroup_tasks(cgroup) != {observed.pid}: + raise RecoveryContainmentUnavailable("recovery prelaunch target has unexpected child tasks") + _private_lock(Path(target.owner.path), target.owner, uid=0) + owner = os.open(target.owner.path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC | os.O_NONBLOCK) + stack.callback(os.close, owner) + if (os.fstat(owner).st_dev, os.fstat(owner).st_ino) != (target.owner.device, target.owner.inode): + raise RecoveryContainmentUnavailable("recovery target owner changed during open") + _assert_pid_flock(observed.pid, owner, "WRITE") + assert_container_published_listener(target.management_id, target.management_image_id, target.published_port) + if _holder_start_token(observed.pid) != token or poller.poll(0) or os.readlink(proc / "ns/net") != network or _cgroup_tasks(cgroup) != {observed.pid} or (cgroup.stat().st_dev, cgroup.stat().st_ino) != (info.st_dev, info.st_ino): + raise RecoveryContainmentUnavailable("recovery target lifetime changed during observation") + return RecoveryTargetFact(target.management_id, observed.pid, token, Path("/proc/sys/kernel/random/boot_id").read_text(encoding="ascii").strip(), network, pid_namespace, expected_cgroup, info.st_dev, info.st_ino, target.owner.device, target.owner.inode, target.published_port) + + +def _watchers(enrollment: RecoveryEnrollment, routes: RouteManifestObservation) -> tuple[RecoveryWatcherFact, ...]: + from . import proxy_watch_cli + + actual_source = str(Path(cast(str, proxy_watch_cli.__file__)).resolve(strict=True)) + configs = {item.configuration.path for item in enrollment.watchers} + selected_units = {item.unit for item in enrollment.watchers} + routes_by_name = {item.unit: item for item in routes.systemd} + result = [] + pairs = {(item.management_id, item.management_image_id, item.proxy_id, item.proxy_image_id): item for item in enrollment.originals} + for selected in enrollment.watchers: + if selected.source.path != actual_source: + raise RecoveryContainmentUnavailable("watcher source is not the actual current installed module") + read_bound_file(selected.source, source=True) + content = read_bound_file(selected.configuration) + raw = json.loads(content, object_pairs_hook=unique_pairs) + if not isinstance(raw, dict) or set(raw) != {"state_root", "manager_id", "manager_image", "proxy_id", "proxy_image", "config_sha256"} or any(not isinstance(value, str) for value in raw.values()): + raise RecoveryContainmentUnavailable("watcher effective configuration differs") + binding = ProxyBinding(raw["manager_id"], raw["manager_image"], raw["proxy_id"], raw["proxy_image"], raw["config_sha256"]) + binding.validate() + pair = pairs.get((binding.manager_id, binding.manager_image, binding.proxy_id, binding.proxy_image)) + if pair is None or pair.proxy_config.sha256 != binding.config_sha256 or selected.lock.path != str(Path(raw["state_root"]) / "proxy-watch.lock"): + raise RecoveryContainmentUnavailable("watcher source/configuration does not use the exact enrolled controller lock") + root = Path(raw["state_root"]) + _approved_directory(root, uid=0) + fact = routes_by_name.get(selected.unit) + if fact is None: + raise RecoveryContainmentUnavailable("watcher installed unit is outside the observed routes") + installed_unit = observe_recovery_watcher_unit(selected, fact) + result.append(RecoveryWatcherFact(selected.unit, binding.manager_id, binding.proxy_id, selected.configuration, selected.source, installed_unit, selected.lock.device, selected.lock.inode)) + source_sets = [(item.unit, item.command_sources) for item in routes.systemd] + source_sets += [(str(item.uid) + ":" + item.fact.unit, item.fact.command_sources) for item in routes.user_systemd] + source_sets += [(item.path, item.command_sources) for item in routes.cron_entries] + for name, sources in source_sets: + paths = {item.path for item in sources} + if actual_source in paths and (len(paths & configs) != 1 or name.endswith(".service") and name not in selected_units): + raise RecoveryContainmentUnavailable("unknown watcher invocation/configuration is outside the actual controller fences") + return tuple(result) + + +def _effective_files(enrollment: RecoveryEnrollment, managers: tuple[RecoveryManagerFact, ...]) -> tuple[RecoveryFile, ...]: + result: dict[str, RecoveryFile] = {} + members: tuple[RecoveryOriginal | RecoveryTarget, ...] = (*enrollment.originals, enrollment.target) + for selected, manager in zip(members, managers): + raw = json.loads(read_bound_file(selected.bootstrap_config), object_pairs_hook=unique_pairs) + for name in ("original_genesis", "recovery_epoch_config"): + configured = raw.get(name) + if configured is None: + continue + path = Path(_mapped_file(configured, manager.mounts, writable=False)) + data, info = _protected(path, uid=0, size_limit=1024 * 1024) + if info.st_nlink != 1: + raise RecoveryContainmentUnavailable("effective original/consumer input is aliased") + result[str(path)] = RecoveryFile(str(path), hashlib.sha256(data).hexdigest(), info.st_dev, info.st_ino) + if name == "recovery_epoch_config": + _consumer_custody_profile(enrollment, data) + return tuple(result[key] for key in sorted(result)) + + +def _consumer_custody_profile(enrollment: RecoveryEnrollment, encoded: bytes) -> None: + """Bind the target's actual prelaunch inputs to this exact selected scope. + + History completeness, epoch EX and grant admission remain issuer duties. + Here those input roles must still name their real mapped objects rather + than an alternate socket, a different original owner or missing resources. + """ + from .recovery_epoch_consumer import parse_consumer_configuration + + selected = parse_consumer_configuration(encoded) + target = enrollment.target + mounts = {item.destination: item for item in target.mounts} + socket_directory = mounts.get("/management-recovery") + if ( + selected.authorization_id != enrollment.authorization_id + or selected.issuer_uid != 0 + or socket_directory is None + or socket_directory.source != "/var/lib/auto-coder/recovery-epoch-peer" + or socket_directory.writable + or not stat.S_ISDIR(socket_directory.mode) + or (selected.socket_directory_device, selected.socket_directory_inode) != (socket_directory.device, socket_directory.inode) + or (selected.target_owner.name, selected.target_owner.device, selected.target_owner.inode) != ("target-owner", target.owner.device, target.owner.inode) + ): + raise RecoveryContainmentUnavailable("recovery consumer's actual issuer socket or target owner differs") + owners = {item.name: item for item in selected.original_fences} + expected = {"history"} + allowed_owner_destinations = set() + transferred = {item.name: item for item in selected.transferred_resources} + expected_transfers = {"epoch"} + for index, original in enumerate(enrollment.originals): + if original.owner is None: + continue + name, destination = f"owner-{index}", f"/management-recovery-originals/{index}/owner.lock" + expected.add(name) + allowed_owner_destinations.add(destination) + chosen, mount = owners.get(name), mounts.get(destination) + if ( + chosen is None + or mount is None + or mount.writable + or not stat.S_ISREG(mount.mode) + or chosen.path != destination + or mount.source != original.owner.path + or (chosen.device, chosen.inode) != (original.owner.device, original.owner.inode) + or (mount.device, mount.inode) != (chosen.device, chosen.inode) + ): + raise RecoveryContainmentUnavailable("recovery consumer original SH input is missing or bound to another owner") + namespace = transferred.get(f"namespace-{index}") + pin = original.namespace_pin + if namespace is None or pin is None or namespace.kind != "network-namespace" or (namespace.device, namespace.inode) != (pin.device, pin.inode): + raise RecoveryContainmentUnavailable("recovery consumer original namespace input differs") + expected_transfers.add(namespace.name) + if {item.destination for item in target.mounts if item.destination.startswith("/management-recovery-originals/")} != allowed_owner_destinations: + raise RecoveryContainmentUnavailable("recovery target has undeclared original owner mounts") + history = owners.get("history") + history_mount = mounts.get("/management-recovery-history/history.lock") + if set(owners) != expected or history is None or history_mount is None or history.path != history_mount.destination or history_mount.writable or not stat.S_ISREG(history_mount.mode) or (history.device, history.inode) != (history_mount.device, history_mount.inode): + raise RecoveryContainmentUnavailable("recovery consumer history SH input or original set differs") + for index, watcher in enumerate(enrollment.watchers): + controller = transferred.get(f"controller-{index}") + if controller is None or controller.kind != "transferred-ex" or (controller.device, controller.inode) != (watcher.lock.device, watcher.lock.inode): + raise RecoveryContainmentUnavailable("recovery consumer controller custody input differs") + expected_transfers.add(controller.name) + epoch = transferred.get("epoch") + if set(transferred) != expected_transfers or epoch is None or epoch.kind != "transferred-ex": + raise RecoveryContainmentUnavailable("recovery consumer deployment custody input is incomplete") + + +def _facts(approved: RecoveryEnrollmentObservation, resources: RecoveryResourceObservation) -> RecoveryContainmentFacts: + enrollment = approved.enrollment + docker = _engine(enrollment) + current = {item.container_id: item for item in docker} + full = {identifier: _full_inspect(identifier) for identifier in enrollment.container_ids} + if any(digest([full[identifier]]) != current[identifier].inspection_sha256 for identifier in full): + raise RecoveryContainmentUnavailable("recovery Docker objects changed after whole-engine enumeration") + members: tuple[RecoveryOriginal | RecoveryTarget, ...] = (*enrollment.originals, enrollment.target) + managers = tuple(validate_manager_profile(item, full[item.management_id], source=enrollment.source_identity) for item in members) + assert_disjoint_target(enrollment, managers) + proxies = tuple((item.generation_id, validate_proxy_profile(item, full[item.proxy_id])) for item in enrollment.originals) + kernels = tuple(_original_kernel(item, manager, current) for item, manager in zip(enrollment.originals, managers)) + target = _target_kernel(enrollment, current, kernels) + routes = observe_recovery_route_manifest(route_selection(approved), docker) + watchers = _watchers(enrollment, routes) + effective = _effective_files(enrollment, managers) + if not routes.closed_profile or routes.deployment_roots != tuple(sorted((item for original in members for item in original.mounts), key=lambda item: (item.container_id, item.destination))): + raise RecoveryContainmentUnavailable("recovery route closure omitted or changed an actual selected root") + # The complete route reader already repeats its system/user/cron catalog. + # Repeat the other actual predicates around that reader while custody is + # retained, without redundantly multiplying the full systemd scan count. + if ( + tuple(validate_manager_profile(item, full[item.management_id], source=enrollment.source_identity) for item in members) != managers + or tuple((item.generation_id, validate_proxy_profile(item, full[item.proxy_id])) for item in enrollment.originals) != proxies + or tuple(_original_kernel(item, manager, current) for item, manager in zip(enrollment.originals, managers)) != kernels + or _target_kernel(enrollment, current, kernels) != target + or _watchers(enrollment, routes) != watchers + or _effective_files(enrollment, managers) != effective + ): + raise RecoveryContainmentUnavailable("recovery source, kernel or effective configuration changed during observation") + if _engine(enrollment) != docker or any(_full_inspect(identifier) != full[identifier] for identifier in full): + raise RecoveryContainmentUnavailable("recovery complete engine/source/configuration changed during observation") + return RecoveryContainmentFacts(approved, enrollment.source_identity, docker, managers, proxies, kernels, target, resources, routes, watchers, effective) + + +def observe(enrollment_id: str, resources: RecoveryResources) -> RecoveryContainmentReport: + """Produce complete current facts or raise; partial evidence is never success.""" + try: + started = time.monotonic_ns() + approved = load_enrollment(enrollment_id) + resources = RecoveryResources.parse(json.loads(json.dumps(asdict(resources)))) + if approved.enrollment.source_identity != source_identity(): + raise RecoveryContainmentUnavailable("enrolled recovery package source is not the current observer source") + with hold_observed_resources(approved.enrollment, resources) as held: + last = _facts(approved, held) + if load_enrollment(enrollment_id) != approved: + raise RecoveryContainmentUnavailable("recovery protected enrollment changed during observations") + payload = digest(asdict(last)) + report = RecoveryContainmentReport(_REPORT_SCHEMA, _STATUS, str(uuid.uuid4()), os.getpid(), _holder_start_token(os.getpid()), resources.holder_boot_id, started, time.monotonic_ns(), last, payload) + logger.bind(observation_only=True, recovery_grant=False, admission_created=False).info("Current recovery containment observed: enrollment={} evidence={}", enrollment_id, payload) + return report + except (OSError, ValueError, TypeError, KeyError, IndexError, UnicodeError, sqlite3.Error, subprocess.SubprocessError, tarfile.TarError) as error: + if isinstance(error, RecoveryContainmentUnavailable): + raise + raise RecoveryContainmentUnavailable("current recovery containment unavailable: " + str(error)) from error + + +def verify(enrollment_id: str, previous: RecoveryContainmentReport | object, resources: RecoveryResources) -> RecoveryContainmentReport: + """A separate consumer repeats current reads; a saved status is no authority.""" + prior = asdict(previous) if isinstance(previous, RecoveryContainmentReport) else previous + if not isinstance(prior, dict) or set(prior) != set(RecoveryContainmentReport.__dataclass_fields__) or prior.get("schema") != _REPORT_SCHEMA or prior.get("status") != _STATUS or any(prior.get(name) is not False for name in ("grant_issued", "history_asserted", "admission_created")): + raise RecoveryContainmentUnavailable("previous recovery report is forged, partial or authorizing") + if any(type(prior.get(name)) is not int or cast(int, prior[name]) <= 0 for name in ("observer_pid", "started_monotonic_ns", "finished_monotonic_ns")) or not isinstance(prior.get("observer_start_token"), str): + raise RecoveryContainmentUnavailable("previous recovery observer lifetime differs") + if not isinstance(prior.get("facts"), dict) or prior.get("evidence_sha256") != digest(prior["facts"]): + raise RecoveryContainmentUnavailable("previous recovery fact digest differs") + if not isinstance(prior.get("observation_id"), str) or str(uuid.UUID(prior["observation_id"])) != prior["observation_id"]: + raise RecoveryContainmentUnavailable("previous recovery observation identity differs") + # The sampling command may have exited. Its PID is attribution only; the + # actual resource producers and holders are independently pinned and checked + # again by observe(), and a saved positive status supplies no authority. + current = observe(enrollment_id, resources) + if prior["boot_id"] != current.boot_id or not cast(int, prior["started_monotonic_ns"]) <= cast(int, prior["finished_monotonic_ns"]) <= current.started_monotonic_ns or current.started_monotonic_ns - cast(int, prior["finished_monotonic_ns"]) > 600 * 1_000_000_000: + raise RecoveryContainmentUnavailable("previous recovery sample is stale, future or from another boot") + if prior["evidence_sha256"] != current.evidence_sha256 or json.loads(json.dumps(prior["facts"])) != json.loads(json.dumps(asdict(current.facts))): + raise RecoveryContainmentUnavailable("previous recovery scope/source/resources/current facts differ") + return current + + +def capture_recovery_original_pair(original: RecoveryOriginal, *, source: str) -> tuple[str, PriorContainer, PriorContainer]: + """Capture real live original lifetimes without claiming a branch succeeded. + + This reads the separately selected command/configuration and the native + supervisor owner/presence. The caller must pin its real network namespace + while alive and install approval separately; this function does neither. + """ + _require_capture_host() + engine = _engine_id() + full = _full_inspect(original.management_id) + validate_manager_profile(original, full, source=source) + proxy_full = _full_inspect(original.proxy_id) + validate_proxy_profile(original, proxy_full) + values = canonical_legacy_inspection(_inspect(original.management_id)) + if values[1:3] != [original.management_image_id, True] or values[4] != "running": + raise RecoveryContainmentUnavailable("recovery original is not the exact live manager") + mounts = _mounts(full) + manager = _capture_prior_kernel(original.management_id, original.management_image_id, engine, values, mounts, require_owner_lock=True) + manager = _capture_supervisor_identity(manager, values, mounts, Path(original.bootstrap_config.path)) + proxy_mounts = _mounts(proxy_full) + if tuple(destination for _, destination, _ in proxy_mounts) == ("/etc/caddy/Caddyfile",): + # Ordinary cutover and original stock Caddy retain their separate + # capture contracts. Recovery composes them without widening either. + proxy = capture_prior_container(original.proxy_id, original.proxy_image_id, expected_mounts=proxy_mounts) + else: + proxy = capture_original_caddy_container(original.proxy_id, original.proxy_image_id, expected_mounts=proxy_mounts) + if _full_inspect(original.management_id) != full or _full_inspect(original.proxy_id) != proxy_full or _engine_id() != engine or manager.net_namespace != proxy.net_namespace: + raise RecoveryContainmentUnavailable("recovery original changed during live capture") + validate_manager_profile(original, full, source=source) + return Path("/proc/sys/kernel/random/boot_id").read_text(encoding="ascii").strip(), manager, proxy + + +def _input(path: Path) -> object: + if not path.is_absolute() or path != path.resolve(strict=True): + raise RecoveryContainmentUnavailable("recovery observation input is not canonical") + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC | os.O_NONBLOCK) + with os.fdopen(descriptor, "rb") as stream: + first = os.fstat(stream.fileno()) + if not stat.S_ISREG(first.st_mode): + raise RecoveryContainmentUnavailable("recovery observation input is not regular") + data = stream.read(_MAX_BYTES + 1) + if len(data) > _MAX_BYTES or _file_identity(first) != _file_identity(os.fstat(stream.fileno())) or _file_identity(first) != _file_identity(path.lstat()): + raise RecoveryContainmentUnavailable("recovery observation input changed or exceeds its bound") + return json.loads(data, object_pairs_hook=unique_pairs) + + +def main() -> int: + parser = argparse.ArgumentParser(description="Read current ordinary recovery containment; no admission or grant") + parser.add_argument("action", choices=("observe", "verify")) + parser.add_argument("--enrollment", required=True) + parser.add_argument("--resources", required=True, type=Path) + parser.add_argument("--previous", type=Path) + args = parser.parse_args() + setup_logger(stream=sys.stdout, log_file="/var/log/auto-coder/recovery-containment.log") + try: + resources = RecoveryResources.parse(_input(args.resources)) + if (args.action == "verify") != (args.previous is not None): + raise RecoveryContainmentUnavailable("separate verification requires exactly one previous report") + report = verify(args.enrollment, _input(args.previous), resources) if args.previous is not None else observe(args.enrollment, resources) + logger.complete() + print(json.dumps(asdict(report), sort_keys=True, separators=(",", ":")), flush=True) + return 0 + except (OSError, ValueError, TypeError, KeyError, IndexError, UnicodeError, subprocess.SubprocessError) as error: + logger.bind(observation_only=True, recovery_grant=False, admission_created=False).warning("Current recovery containment unavailable: {}", str(error)) + causes = refusal_causes(error) + for index, cause in enumerate(causes, start=1): + logger.bind(observation_only=True, recovery_grant=False, admission_created=False).warning("Current recovery containment refusal cause {}: {}", index, json.dumps(cause, sort_keys=True, separators=(",", ":"))) + logger.complete() + print(json.dumps({"schema": _REPORT_SCHEMA, "status": "unavailable", "grant_issued": False, "history_asserted": False, "admission_created": False, "reason": str(error), "causes": causes}, sort_keys=True), flush=True) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/auto_coder/management/recovery_containment_model.py b/src/auto_coder/management/recovery_containment_model.py new file mode 100644 index 000000000..f3d7648f0 --- /dev/null +++ b/src/auto_coder/management/recovery_containment_model.py @@ -0,0 +1,314 @@ +"""Finite scope and resource types for observation-only ordinary recovery. + +Protected enrollment selects the independently inspectable objects. None of these +records attests original history, technical containment, admission or a grant. +""" + +from __future__ import annotations + +import re +import uuid +from dataclasses import dataclass +from pathlib import Path +from typing import cast + +from .containment_enrollment import OriginalNetworkPin +from .containment_route_manifest import DeploymentRootBinding + +RECOVERY_ENROLLMENT_ROOT = Path("/var/lib/auto-coder/approved-recovery-containment-enrollments") +RECOVERY_PROFILE = "ordinary-recovery-v1" +_MAX_GENERATIONS = 16 +_HASH = re.compile(r"[0-9a-f]{64}") +_IMAGE = re.compile(r"sha256:[0-9a-f]{64}") +_IDENTIFIER = re.compile(r"[A-Za-z0-9_.:-]{1,128}") + + +class RecoveryContainmentUnavailable(ValueError): + """A required current recovery observation is absent or unsupported.""" + + +def _object(value: object, fields: object, name: str) -> dict[str, object]: + if not isinstance(value, dict) or set(value) != set(cast(dict[str, object], fields)): + raise RecoveryContainmentUnavailable(name + " schema differs") + return value + + +def _text(value: object, pattern: re.Pattern[str], name: str) -> str: + if not isinstance(value, str) or not pattern.fullmatch(value): + raise RecoveryContainmentUnavailable(name + " differs") + return value + + +def _path(value: object) -> str: + if not isinstance(value, str) or not value or len(value) > 4096 or not Path(value).is_absolute() or str(Path(value)) != value or ".." in Path(value).parts or value == "/": + raise RecoveryContainmentUnavailable("recovery path is not canonical") + return value + + +def _positive(value: object) -> int: + if type(value) is not int or value <= 0: + raise RecoveryContainmentUnavailable("recovery kernel identity differs") + return value + + +def _boot(value: object) -> str: + try: + if not isinstance(value, str) or str(uuid.UUID(value)) != value: + raise ValueError("noncanonical boot identity") + except (ValueError, TypeError, AttributeError) as error: + raise RecoveryContainmentUnavailable("recovery boot identity differs") from error + return value + + +@dataclass(frozen=True) +class RecoveryFile: + path: str + sha256: str + device: int + inode: int + + @classmethod + def parse(cls, value: object) -> RecoveryFile: + raw = _object(value, cls.__dataclass_fields__, "recovery file") + return cls(_path(raw["path"]), _text(raw["sha256"], _HASH, "file digest"), _positive(raw["device"]), _positive(raw["inode"])) + + +@dataclass(frozen=True) +class RecoveryLock: + path: str + device: int + inode: int + + @classmethod + def parse(cls, value: object) -> RecoveryLock: + raw = _object(value, cls.__dataclass_fields__, "recovery lock") + return cls(_path(raw["path"]), _positive(raw["device"]), _positive(raw["inode"])) + + +def _mounts(value: object, containers: tuple[str, ...]) -> tuple[DeploymentRootBinding, ...]: + if not isinstance(value, list) or not 1 <= len(value) <= 64: + raise RecoveryContainmentUnavailable("recovery bind inventory is absent or oversized") + mounts = tuple(DeploymentRootBinding.parse(item) for item in value) + keys = tuple((item.container_id, item.destination) for item in mounts) + if keys != tuple(sorted(set(keys))) or {item.container_id for item in mounts} != set(containers): + raise RecoveryContainmentUnavailable("recovery bind inventory is ambiguous or foreign") + for item in mounts: + _path(item.source) + _path(item.destination) + return mounts + + +@dataclass(frozen=True) +class RecoveryOriginal: + generation_id: str + instance_id: str + management_id: str + management_image_id: str + proxy_id: str + proxy_image_id: str + state: str + boot_id: str + bootstrap_config: RecoveryFile + management_config: RecoveryFile + proxy_config: RecoveryFile + mounts: tuple[DeploymentRootBinding, ...] + owner: RecoveryLock | None + prior_evidence: RecoveryFile | None + namespace_pin: OriginalNetworkPin | None + + @classmethod + def parse(cls, value: object) -> RecoveryOriginal: + raw = _object(value, cls.__dataclass_fields__, "recovery original") + manager = _text(raw["management_id"], _HASH, "original manager") + proxy = _text(raw["proxy_id"], _HASH, "original proxy") + if manager == proxy: + raise RecoveryContainmentUnavailable("original manager and proxy alias") + state = raw["state"] + if state not in ("started", "never-started"): + raise RecoveryContainmentUnavailable("original current-state selection differs") + if state == "started": + boot = _boot(raw["boot_id"]) + owner = RecoveryLock.parse(raw["owner"]) + prior = RecoveryFile.parse(raw["prior_evidence"]) + pin = OriginalNetworkPin.parse(raw["namespace_pin"]) + if pin.holder_boot_id != boot: + raise RecoveryContainmentUnavailable("original capture and namespace boot differ") + else: + if raw["boot_id"] != "" or any(raw[name] is not None for name in ("owner", "prior_evidence", "namespace_pin")): + raise RecoveryContainmentUnavailable("never-started object cannot invent old resources") + boot, owner, prior, pin = "", None, None, None + return cls( + _text(raw["generation_id"], _IDENTIFIER, "generation identity"), + _text(raw["instance_id"], _IDENTIFIER, "original instance identity"), + manager, + _text(raw["management_image_id"], _IMAGE, "original manager image"), + proxy, + _text(raw["proxy_image_id"], _IMAGE, "original proxy image"), + str(state), + boot, + RecoveryFile.parse(raw["bootstrap_config"]), + RecoveryFile.parse(raw["management_config"]), + RecoveryFile.parse(raw["proxy_config"]), + _mounts(raw["mounts"], (manager, proxy)), + owner, + prior, + pin, + ) + + +@dataclass(frozen=True) +class RecoveryTarget: + instance_id: str + management_id: str + management_image_id: str + bootstrap_config: RecoveryFile + management_config: RecoveryFile + mounts: tuple[DeploymentRootBinding, ...] + owner: RecoveryLock + published_port: int + + @classmethod + def parse(cls, value: object) -> RecoveryTarget: + raw = _object(value, cls.__dataclass_fields__, "recovery target") + manager = _text(raw["management_id"], _HASH, "target manager") + port = raw["published_port"] + if type(port) is not int or not 0 < port <= 65535: + raise RecoveryContainmentUnavailable("target published port differs") + return cls( + _text(raw["instance_id"], _IDENTIFIER, "target instance"), + manager, + _text(raw["management_image_id"], _IMAGE, "target image"), + RecoveryFile.parse(raw["bootstrap_config"]), + RecoveryFile.parse(raw["management_config"]), + _mounts(raw["mounts"], (manager,)), + RecoveryLock.parse(raw["owner"]), + port, + ) + + +@dataclass(frozen=True) +class RecoveryWatcher: + unit: str + configuration: RecoveryFile + source: RecoveryFile + unit_source: RecoveryFile + lock: RecoveryLock + + @classmethod + def parse(cls, value: object) -> RecoveryWatcher: + raw = _object(value, cls.__dataclass_fields__, "recovery watcher") + unit = _text(raw["unit"], re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,119}\.service"), "watcher unit") + fragment = RecoveryFile.parse(raw["unit_source"]) + if Path(fragment.path).name != unit: + raise RecoveryContainmentUnavailable("recovery watcher retained unit source name differs") + return cls(unit, RecoveryFile.parse(raw["configuration"]), RecoveryFile.parse(raw["source"]), fragment, RecoveryLock.parse(raw["lock"])) + + +@dataclass(frozen=True) +class RecoveryEnrollment: + schema: str + deployment_id: str + enrollment_id: str + authorization_id: str + profile: str + engine_id: str + source_identity: str + originals: tuple[RecoveryOriginal, ...] + target: RecoveryTarget + watchers: tuple[RecoveryWatcher, ...] + approved_observation_only: bool + + @classmethod + def parse(cls, value: object) -> RecoveryEnrollment: + raw = _object(value, cls.__dataclass_fields__, "recovery enrollment") + if raw["schema"] != "approved-management-recovery-containment-v1" or raw["profile"] != RECOVERY_PROFILE or raw["approved_observation_only"] is not True: + raise RecoveryContainmentUnavailable("recovery enrollment profile or authority differs") + values, controllers = raw["originals"], raw["watchers"] + if not isinstance(values, list) or not 1 <= len(values) <= _MAX_GENERATIONS or not isinstance(controllers, list) or not 1 <= len(controllers) <= 32: + raise RecoveryContainmentUnavailable("recovery original/controller set is absent or oversized") + originals = tuple(RecoveryOriginal.parse(item) for item in values) + target = RecoveryTarget.parse(raw["target"]) + watchers = tuple(RecoveryWatcher.parse(item) for item in controllers) + if 3 + 2 * sum(item.state == "started" for item in originals) + len(watchers) > 64: + raise RecoveryContainmentUnavailable("recovery scope exceeds the complete peer resource bound") + names = tuple(item.generation_id for item in originals) + units = tuple(item.unit for item in watchers) + if names != tuple(sorted(set(names))) or units != tuple(sorted(set(units))): + raise RecoveryContainmentUnavailable("recovery members or controllers are duplicate or unsorted") + ids = [target.management_id] + [identifier for item in originals for identifier in (item.management_id, item.proxy_id)] + instances = [target.instance_id] + [item.instance_id for item in originals] + if len(ids) != len(set(ids)) or len(instances) != len(set(instances)): + raise RecoveryContainmentUnavailable("recovery container or instance identities alias") + locks = [item.owner for item in originals if item.owner is not None] + [target.owner] + [item.lock for item in watchers] + if len({item.path for item in locks}) != len(locks) or len({(item.device, item.inode) for item in locks}) != len(locks): + raise RecoveryContainmentUnavailable("recovery owner/controller objects alias") + return cls( + str(raw["schema"]), + _text(raw["deployment_id"], _IDENTIFIER, "deployment"), + _text(raw["enrollment_id"], _IDENTIFIER, "enrollment"), + _text(raw["authorization_id"], _IDENTIFIER, "authorization"), + RECOVERY_PROFILE, + _text(raw["engine_id"], _IDENTIFIER, "engine"), + _text(raw["source_identity"], _IMAGE, "package source"), + originals, + target, + watchers, + True, + ) + + @property + def container_ids(self) -> tuple[str, ...]: + return tuple(sorted((self.target.management_id, *(identifier for item in self.originals for identifier in (item.management_id, item.proxy_id))))) + + +@dataclass(frozen=True) +class RecoveryEnrollmentObservation: + enrollment: RecoveryEnrollment + sha256: str + device: int + inode: int + ctime_ns: int + + +@dataclass(frozen=True) +class NamedDescriptor: + name: str + descriptor: int + + @classmethod + def parse(cls, value: object) -> NamedDescriptor: + raw = _object(value, cls.__dataclass_fields__, "recovery descriptor") + descriptor = raw["descriptor"] + if type(descriptor) is not int or not 0 <= descriptor <= 1048576: + raise RecoveryContainmentUnavailable("recovery descriptor differs") + return cls(_text(raw["name"], _IDENTIFIER, "resource name"), descriptor) + + +@dataclass(frozen=True) +class RecoveryResources: + holder_pid: int + holder_start_token: str + holder_boot_id: str + original_owner_fds: tuple[NamedDescriptor, ...] + controller_fds: tuple[NamedDescriptor, ...] + namespace_fds: tuple[NamedDescriptor, ...] + + @classmethod + def parse(cls, value: object) -> RecoveryResources: + raw = _object(value, cls.__dataclass_fields__, "recovery resources") + pid = _positive(raw["holder_pid"]) + if pid <= 1 or pid > 2147483647: + raise RecoveryContainmentUnavailable("recovery resource holder PID differs") + groups = [] + for name in ("original_owner_fds", "controller_fds", "namespace_fds"): + selected = raw[name] + if not isinstance(selected, list) or len(selected) > 32: + raise RecoveryContainmentUnavailable("recovery resource set is oversized") + group = tuple(NamedDescriptor.parse(item) for item in selected) + names = tuple(item.name for item in group) + if names != tuple(sorted(set(names))) or len({item.descriptor for item in group}) != len(group): + raise RecoveryContainmentUnavailable("recovery resource set is duplicate or unsorted") + groups.append(group) + if len({item.descriptor for group in groups for item in group}) != sum(map(len, groups)): + raise RecoveryContainmentUnavailable("recovery resource descriptor groups alias") + return cls(pid, _text(raw["holder_start_token"], re.compile(r"[1-9][0-9]{0,23}"), "holder lifetime"), _boot(raw["holder_boot_id"]), *groups) diff --git a/src/auto_coder/management/recovery_containment_profile.py b/src/auto_coder/management/recovery_containment_profile.py new file mode 100644 index 000000000..7164ddebf --- /dev/null +++ b/src/auto_coder/management/recovery_containment_profile.py @@ -0,0 +1,309 @@ +"""Actual source, configuration and bind profiles for ordinary recovery only.""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import stat +from dataclasses import dataclass +from pathlib import Path + +from .archived_exclusions import unique_pairs +from .containment_enrollment import _file_identity, _protected, _trusted_ancestors +from .containment_route_manifest import DeploymentRootBinding +from .installed_bootstrap_mount_profile import _mounts, _unshadowed_bootstrap_execution +from .original_attempts import source_identity +from .recovery_containment_model import RecoveryContainmentUnavailable, RecoveryEnrollment, RecoveryFile, RecoveryLock, RecoveryOriginal, RecoveryTarget +from .recovery_source_profile import read_image_source_identity + +_COMMAND = ["-m", "auto_coder.management.bootstrap_supervisor", "--config", "/bootstrap/config.json"] + + +@dataclass(frozen=True) +class RecoveryManagerFact: + generation_id: str + instance_id: str + container_id: str + image_id: str + state: str + source_identity: str + inspection_sha256: str + bootstrap_config_sha256: str + management_config_sha256: str + state_root: str + journal_root: str + oauth_root: str + owner: RecoveryLock | None + published_port: int + mounts: tuple[DeploymentRootBinding, ...] + + +def digest(value: object) -> str: + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + + +def read_bound_file(item: RecoveryFile, *, source: bool = False, expected_uid: int = 0) -> bytes: + path = Path(item.path) + if not source: + data, info = _protected(path, uid=expected_uid, size_limit=1024 * 1024) + else: + if path != path.resolve(strict=True): + raise RecoveryContainmentUnavailable("recovery controller source path is not canonical") + _trusted_ancestors(path.parent, uid=expected_uid) + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC | os.O_NONBLOCK) + try: + info = os.fstat(fd) + if not stat.S_ISREG(info.st_mode) or info.st_uid != expected_uid or info.st_nlink != 1 or info.st_mode & (stat.S_IWGRP | stat.S_IWOTH): + raise RecoveryContainmentUnavailable("recovery controller source is not protected") + with os.fdopen(os.dup(fd), "rb") as stream: + data = stream.read(1024 * 1024 + 1) + if len(data) > 1024 * 1024 or _file_identity(os.fstat(fd)) != _file_identity(info) or _file_identity(path.lstat()) != _file_identity(info): + raise RecoveryContainmentUnavailable("recovery source changed during readback") + finally: + os.close(fd) + if info.st_nlink != 1 or (info.st_dev, info.st_ino, hashlib.sha256(data).hexdigest()) != (item.device, item.inode, item.sha256): + raise RecoveryContainmentUnavailable("protected recovery source/configuration changed") + return data + + +def _security(inspected: dict[str, object]) -> tuple[dict[str, object], dict[str, object]]: + host, config = inspected.get("HostConfig"), inspected.get("Config") + if not isinstance(host, dict) or not isinstance(config, dict): + raise RecoveryContainmentUnavailable("recovery Docker command/security observation is absent") + if ( + host.get("Privileged") is not False + or host.get("PidMode") not in ("", None) + or host.get("CapAdd") not in (None, []) + or host.get("Devices") not in (None, []) + or host.get("SecurityOpt") not in (None, []) + or host.get("NetworkMode") in ("host", "none", None, "") + or config.get("User") != "" + or config.get("Healthcheck") is not None + ): + raise RecoveryContainmentUnavailable("recovery ordinary Docker security profile differs") + return host, config + + +def bind_mounts(container: dict[str, object], selected: tuple[DeploymentRootBinding, ...], *, expected_uid: int = 0) -> tuple[DeploymentRootBinding, ...]: + identifier = container.get("Id") + observed = _mounts(container) + expected = tuple(item for item in selected if item.container_id == identifier) + if tuple(sorted((item.source, item.destination, item.writable) for item in expected)) != observed: + raise RecoveryContainmentUnavailable("recovery actual and protected bind inventories differ") + for item in expected: + path = Path(item.source) + if path != path.resolve(strict=True) or path == Path("/") or path == Path("/proc") or path.is_relative_to("/proc") or path == Path("/sys") or path.is_relative_to("/sys") or path == Path("/dev") or path.is_relative_to("/dev"): + raise RecoveryContainmentUnavailable("recovery bind source is noncanonical or unsupported") + _trusted_ancestors(path.parent, uid=expected_uid) + info = path.lstat() + if info.st_uid != expected_uid or info.st_mode & (stat.S_IWGRP | stat.S_IWOTH) or (info.st_dev, info.st_ino, info.st_mode) != (item.device, item.inode, item.mode): + raise RecoveryContainmentUnavailable("recovery bind root was replaced or is unprotected") + return expected + + +def _mapped_file(path: str, mounts: tuple[DeploymentRootBinding, ...], *, writable: bool) -> str: + destination = Path(path) + matched = [item for item in mounts if destination == Path(item.destination) or Path(item.destination) in destination.parents] + if len(matched) != 1 or matched[0].writable != writable: + raise RecoveryContainmentUnavailable("recovery effective configuration is absent, shadowed or writable") + binding = matched[0] + return str(Path(binding.source) / destination.relative_to(binding.destination)) + + +def _root(path: object, mounts: tuple[DeploymentRootBinding, ...], *, writable: bool) -> str: + if not isinstance(path, str) or not Path(path).is_absolute() or ".." in Path(path).parts: + raise RecoveryContainmentUnavailable("recovery configured root differs") + found = [item for item in mounts if item.destination == path] + if len(found) != 1 or found[0].writable != writable or not stat.S_ISDIR(found[0].mode): + raise RecoveryContainmentUnavailable("recovery effective root mapping differs") + if any(item.destination != path and (Path(path) in Path(item.destination).parents or Path(item.destination) in Path(path).parents) for item in mounts): + raise RecoveryContainmentUnavailable("recovery effective root is shadowed") + return found[0].source + + +def _ordinary_mount_effects(config: dict[str, object], mounts: tuple[DeploymentRootBinding, ...], *, target: bool) -> None: + """Keep extra mounts within the explicit ordinary data/custody profile.""" + effects = {"/bootstrap": False, "/bootstrap/config.json": False, "/bootstrap/management.yml": False} + for name in ("state_root", "journal_root", "oauth_root"): + value = config.get(name) + if isinstance(value, str): + effects[value] = True + for name in ("management_version_root", "management_config", "original_genesis", "recovery_epoch_config"): + value = config.get(name) + if isinstance(value, str): + effects[value] = False + runtimes = config.get("runtimes", []) + exclusions = config.get("archived_exclusions", []) + if not isinstance(runtimes, list) or not isinstance(exclusions, list): + raise RecoveryContainmentUnavailable("recovery runtime/exclusion profile is ambiguous") + for runtime in runtimes: + if isinstance(runtime, dict) and isinstance(runtime.get("arguments"), list): + arguments = runtime["arguments"] + if "--config" in arguments and arguments.index("--config") + 1 < len(arguments) and isinstance(arguments[arguments.index("--config") + 1], str): + effects[arguments[arguments.index("--config") + 1]] = False + if target: + effects.update({"/management-recovery": False, "/management-recovery-history/history.lock": False}) + else: + if config.get("original_genesis") is not None: + effects["/management-original-history"] = False + if config.get("fresh_guard_path") == "/management-initialization/fresh-guard.sock": + effects["/management-initialization"] = False + if config.get("fresh_prior_lock_path") == "/management-prior-owner.lock": + effects.update({"/management-prior-owner.lock": False, "/management-prior-journal": False, "/management-prior-operations": False}) + for exclusion in exclusions: + if isinstance(exclusion, dict): + for name in ("journal_root", "fence"): + if isinstance(exclusion.get(name), str): + effects[exclusion[name]] = False + for item in mounts: + declared_owner = target and re.fullmatch(r"/management-recovery-originals/(?:0|[1-9][0-9]?)/owner\.lock", item.destination) is not None + worker_secret = re.fullmatch(r"/run/secrets/(?:beta|release)_worker_bridge", item.destination) is not None + if worker_secret and not item.writable and stat.S_ISREG(item.mode): + continue + if declared_owner: + if item.writable or not stat.S_ISREG(item.mode): + raise RecoveryContainmentUnavailable("recovery original custody mount is not a read-only file") + elif effects.get(item.destination) is not item.writable: + raise RecoveryContainmentUnavailable("recovery ordinary profile has an undeclared mount effect") + + +def validate_manager_profile(selected: RecoveryOriginal | RecoveryTarget, inspected: dict[str, object], *, source: str, expected_uid: int = 0) -> RecoveryManagerFact: + """Inspect current source/config shape without reading or promoting outcomes.""" + target = isinstance(selected, RecoveryTarget) + if inspected.get("Id") != selected.management_id or inspected.get("Image") != selected.management_image_id: + raise RecoveryContainmentUnavailable("recovery manager image or identity differs") + host, command = _security(inspected) + expected_commands = [_COMMAND + ["--recovery-epoch"]] if target else [_COMMAND, _COMMAND + ["--fresh-initialization"]] + if command.get("Entrypoint") != ["python"] or command.get("Cmd") not in expected_commands or command.get("WorkingDir") != "/workspace" or str(host.get("NetworkMode")).startswith("container:"): + raise RecoveryContainmentUnavailable("recovery ordinary bootstrap command differs") + mounts = bind_mounts(inspected, selected.mounts, expected_uid=expected_uid) + _unshadowed_bootstrap_execution(command, _mounts(inspected)) + environment = dict(item.split("=", 1) for item in cast_environment(command.get("Env"))) + supported_python = {"PYTHON_VERSION", "PYTHON_SHA256", "PYTHONUNBUFFERED", "PYTHONDONTWRITEBYTECODE"} + if environment.get("PYTHONDONTWRITEBYTECODE") != "1" or any(name.startswith("LD_") or name.startswith("PYTHON") and name not in supported_python for name in environment): + raise RecoveryContainmentUnavailable("recovery source execution environment is unsupported") + config_bytes = read_bound_file(selected.bootstrap_config, expected_uid=expected_uid) + from .bootstrap_supervisor import BootstrapConfig + + BootstrapConfig.read(Path(selected.bootstrap_config.path)) + read_bound_file(selected.management_config, expected_uid=expected_uid) + config = json.loads(config_bytes, object_pairs_hook=unique_pairs) + if ( + not isinstance(config, dict) + or config.get("instance_id") != selected.instance_id + or config.get("state_root") != "/management-child-state" + or config.get("oauth_root") != "/management-oauth" + or config.get("new_instance_initialization", False) is not False + or config.get("new_docker_network") is not None + or config.get("new_docker_host_port") is not None + ): + raise RecoveryContainmentUnavailable("recovery ordinary effective bootstrap identity differs") + _ordinary_mount_effects(config, mounts, target=target) + if _mapped_file("/bootstrap/config.json", mounts, writable=False) != selected.bootstrap_config.path: + raise RecoveryContainmentUnavailable("recovery bootstrap configuration is not actually mounted") + state_root = _root(config.get("state_root"), mounts, writable=True) + journal_root = _root(config.get("journal_root"), mounts, writable=True) + oauth_root = _root(config.get("oauth_root"), mounts, writable=True) + roots = (state_root, journal_root, oauth_root) + if len(set(roots)) != 3 or any(Path(left) == Path(right) or Path(left) in Path(right).parents or Path(right) in Path(left).parents for index, left in enumerate(roots) for right in roots[index + 1 :]): + raise RecoveryContainmentUnavailable("recovery mutable roots overlap") + if selected.owner is not None and selected.owner.path != str(Path(state_root) / ".supervisor-owner.lock"): + raise RecoveryContainmentUnavailable("recovery owner is not the effective supervisor owner file") + if isinstance(selected, RecoveryOriginal) and selected.state == "never-started" and (Path(state_root) / ".supervisor-owner.lock").exists(): + raise RecoveryContainmentUnavailable("never-started original already has an owner artifact") + runtimes = config.get("runtimes") + if not isinstance(runtimes, list) or not 1 <= len(runtimes) <= 64 or any(not isinstance(item, dict) for item in runtimes): + raise RecoveryContainmentUnavailable("recovery runtime configuration is absent or ambiguous") + configured = config.get("management_config") + version_root = config.get("management_version_root") + for runtime in runtimes: + if not isinstance(runtime, dict): + raise RecoveryContainmentUnavailable("recovery runtime configuration differs") + arguments = runtime.get("arguments") + if runtime.get("execution_kind") == "management-installed": + if not isinstance(configured, str) or not isinstance(version_root, str) or _root(version_root, mounts, writable=False) != version_root or arguments != ["-I", "-B", "-m", "auto_coder.cli", "management", "serve", "--config", configured]: + raise RecoveryContainmentUnavailable("recovery installed runtime mapping differs") + executable, source_root = runtime.get("executable"), runtime.get("source_root") + if not isinstance(executable, str) or not isinstance(source_root, str) or not Path(executable).is_relative_to(source_root) or not Path(source_root).is_relative_to(version_root) or source_root == version_root or any(".." in Path(item).parts for item in (executable, source_root)): + raise RecoveryContainmentUnavailable("recovery installed source path is unsupported") + management_path = configured + else: + if target or runtime.get("executable") != "/opt/auto-coder/.venv/bin/auto-coder" or not isinstance(arguments, list) or len(arguments) != 4 or arguments[:3] != ["management", "serve", "--config"] or not isinstance(arguments[3], str): + raise RecoveryContainmentUnavailable("recovery ordinary runtime command differs") + management_path = arguments[3] + if _mapped_file(management_path, mounts, writable=False) != selected.management_config.path: + raise RecoveryContainmentUnavailable("recovery runtime uses another management configuration") + if target: + if config.get("recovery_epoch_config") != "/bootstrap/recovery-epoch.json" or config.get("original_genesis") is not None or any(config.get(name) is not None for name in ("fresh_guard_path", "fresh_prior_lock_path", "fresh_runtime_binding")): + raise RecoveryContainmentUnavailable("recovery target lacks its separate prelaunch profile") + else: + if config.get("recovery_epoch_config") is not None: + raise RecoveryContainmentUnavailable("an original cannot be relabeled as the recovery target") + genesis = config.get("original_genesis") + if genesis is not None: + if genesis != "/bootstrap/original-genesis.json" or command.get("Cmd") != _COMMAND: + raise RecoveryContainmentUnavailable("ordinary original genesis configuration differs") + _mapped_file(genesis, mounts, writable=False) + _root("/management-original-history", mounts, writable=False) + elif command.get("Cmd") != _COMMAND + ["--fresh-initialization"]: + raise RecoveryContainmentUnavailable("ordinary original lacks genesis/history-capable entrypoint") + binding = host.get("PortBindings") + if not isinstance(binding, dict) or set(binding) != {"8080/tcp"} or not isinstance(binding["8080/tcp"], list) or len(binding["8080/tcp"]) != 1: + raise RecoveryContainmentUnavailable("recovery published endpoint is ambiguous") + endpoint = binding["8080/tcp"][0] + if not isinstance(endpoint, dict) or set(endpoint) != {"HostIp", "HostPort"} or endpoint["HostIp"] != "127.0.0.1" or not isinstance(endpoint["HostPort"], str) or not endpoint["HostPort"].isdecimal() or not 0 < int(endpoint["HostPort"]) <= 65535: + raise RecoveryContainmentUnavailable("recovery published endpoint is not exact loopback TCP") + port = int(endpoint["HostPort"]) + if isinstance(selected, RecoveryTarget) and port != selected.published_port: + raise RecoveryContainmentUnavailable("recovery target published endpoint differs") + if source != source_identity() or read_image_source_identity(selected.management_id) != source: + raise RecoveryContainmentUnavailable("recovery actual complete package source differs") + generation = "target" if isinstance(selected, RecoveryTarget) else selected.generation_id + state = "prelaunch-target" if isinstance(selected, RecoveryTarget) else selected.state + return RecoveryManagerFact(generation, selected.instance_id, selected.management_id, selected.management_image_id, state, source, digest([inspected]), selected.bootstrap_config.sha256, selected.management_config.sha256, state_root, journal_root, oauth_root, selected.owner, port, mounts) + + +def cast_environment(value: object) -> tuple[str, ...]: + if not isinstance(value, list) or any(not isinstance(item, str) or "=" not in item for item in value): + raise RecoveryContainmentUnavailable("recovery process environment is ambiguous") + return tuple(value) + + +def validate_proxy_profile(selected: RecoveryOriginal, inspected: dict[str, object], *, expected_uid: int = 0) -> str: + if inspected.get("Id") != selected.proxy_id or inspected.get("Image") != selected.proxy_image_id: + raise RecoveryContainmentUnavailable("original recovery proxy identity differs") + host, config = _security(inspected) + if host.get("NetworkMode") != "container:" + selected.management_id or config.get("Entrypoint") != ["caddy"] or config.get("Cmd") != ["run", "--config", "/etc/caddy/Caddyfile", "--adapter", "caddyfile"] or config.get("WorkingDir") != "/srv": + raise RecoveryContainmentUnavailable("original recovery proxy command or network differs") + mounts = bind_mounts(inspected, selected.mounts, expected_uid=expected_uid) + effects = {item.destination: item.writable for item in mounts} + if effects not in ({"/etc/caddy/Caddyfile": False}, {"/etc/caddy/Caddyfile": False, "/data": True, "/config": True}) or len(effects) != len(mounts): + raise RecoveryContainmentUnavailable("original recovery proxy mount effects differ") + if _mapped_file("/etc/caddy/Caddyfile", mounts, writable=False) != selected.proxy_config.path: + raise RecoveryContainmentUnavailable("original recovery proxy configuration is not actually mounted") + for item in mounts: + destination = Path(item.destination) + if any(destination == Path(root) or destination in Path(root).parents or Path(root) in destination.parents for root in ("/usr/bin", "/usr/local/bin", "/bin", "/lib", "/usr/lib", "/srv")): + raise RecoveryContainmentUnavailable("original recovery Caddy executable is shadowed") + read_bound_file(selected.proxy_config, expected_uid=expected_uid) + return digest([inspected]) + + +def assert_disjoint_target(enrollment: RecoveryEnrollment, facts: tuple[RecoveryManagerFact, ...]) -> None: + target = facts[-1] + if target.container_id != enrollment.target.management_id: + raise RecoveryContainmentUnavailable("recovery target fact is not exact") + for old in facts[:-1]: + if target.published_port == old.published_port: + raise RecoveryContainmentUnavailable("recovery target cannot hide an old published listener") + if target.oauth_root != old.oauth_root: + raise RecoveryContainmentUnavailable("recovery target does not retain the original OAuth root") + for left in (target.state_root, target.journal_root): + for right in (old.state_root, old.journal_root, old.oauth_root): + if Path(left) == Path(right) or Path(left) in Path(right).parents or Path(right) in Path(left).parents: + raise RecoveryContainmentUnavailable("recovery target mutable root overlaps an original") + old_root_ids = {(item.device, item.inode) for original in enrollment.originals for item in original.mounts if item.writable and item.source != target.oauth_root} + if any((item.device, item.inode) in old_root_ids for item in enrollment.target.mounts if item.writable): + raise RecoveryContainmentUnavailable("recovery target aliases an original mutable root") diff --git a/src/auto_coder/management/recovery_containment_resources.py b/src/auto_coder/management/recovery_containment_resources.py new file mode 100644 index 000000000..73f22da6f --- /dev/null +++ b/src/auto_coder/management/recovery_containment_resources.py @@ -0,0 +1,136 @@ +"""Read existing recovery fences and retained namespaces without acquiring them.""" + +from __future__ import annotations + +import os +import select +import stat +from contextlib import ExitStack, contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Iterator + +from .containment_enrollment import _trusted_ancestors +from .containment_kernel_subobservation import _external_namespace_pin, _holder_start_token +from .fresh_host_observation import _assert_pid_flock, assert_prior_network_pin +from .recovery_containment_model import RecoveryContainmentUnavailable, RecoveryEnrollment, RecoveryLock, RecoveryResources +from .recovery_epoch_peer import KernelLock, inspect_kernel_lock + + +@dataclass(frozen=True) +class RecoveryResourceFact: + name: str + kind: str + holder_descriptor: int + path: str + device: int + inode: int + mode: int + uid: int + kernel_lock: KernelLock | None + + +@dataclass(frozen=True) +class RecoveryResourceObservation: + holder_pid: int + holder_start_token: str + holder_boot_id: str + resources: tuple[RecoveryResourceFact, ...] + + +def _private_lock(path: Path, selected: RecoveryLock, *, uid: int) -> os.stat_result: + if path != path.resolve(strict=True) or str(path) != selected.path: + raise RecoveryContainmentUnavailable("recovery fence path is not canonical") + _trusted_ancestors(path.parent, uid=uid) + info = path.lstat() + if not stat.S_ISREG(info.st_mode) or info.st_uid != uid or stat.S_IMODE(info.st_mode) != 0o600 or info.st_nlink != 1 or (info.st_dev, info.st_ino) != (selected.device, selected.inode): + raise RecoveryContainmentUnavailable("original recovery fence is missing, replaced or unprotected") + return info + + +@contextmanager +def hold_observed_resources(enrollment: RecoveryEnrollment, resources: RecoveryResources, *, expected_uid: int = 0) -> Iterator[RecoveryResourceObservation]: + """Retain observation references and recheck every supplied actual holder. + + This takes no flock and creates no file. Original SH attribution must name + the issuer process itself, so an inherited shared OFD is not independent + issuer acquisition. Controller EX must name that same current issuer. + """ + originals = {item.generation_id: item for item in enrollment.originals if item.state == "started"} + watchers = {item.unit: item for item in enrollment.watchers} + owner_fds = {item.name: item.descriptor for item in resources.original_owner_fds} + controller_fds = {item.name: item.descriptor for item in resources.controller_fds} + namespace_fds = {item.name: item.descriptor for item in resources.namespace_fds} + if set(owner_fds) != set(originals) or set(namespace_fds) != set(originals) or set(controller_fds) != set(watchers): + raise RecoveryContainmentUnavailable("recovery resource membership differs from the protected scope") + with ExitStack() as stack: + lifetime = os.pidfd_open(resources.holder_pid, 0) + stack.callback(os.close, lifetime) + poller = select.poll() + poller.register(lifetime, select.POLLIN) + held: list[tuple[RecoveryResourceFact, int, RecoveryLock | None]] = [] + + def check_holder() -> None: + boot = Path("/proc/sys/kernel/random/boot_id").read_text(encoding="ascii").strip() + if boot != resources.holder_boot_id or _holder_start_token(resources.holder_pid) != resources.holder_start_token or poller.poll(0): + raise RecoveryContainmentUnavailable("recovery resource holder changed or exited") + proc = Path("/proc") / str(resources.holder_pid) + if proc.stat().st_uid != expected_uid or any(os.readlink(proc / "ns" / name) != os.readlink("/proc/1/ns/" + name) for name in ("pid", "user", "mnt", "net")): + raise RecoveryContainmentUnavailable("recovery resource holder is not the trusted host process") + + def check_one(fact: RecoveryResourceFact, descriptor: int, selected: RecoveryLock | None) -> None: + target = Path("/proc") / str(resources.holder_pid) / "fd" / str(fact.holder_descriptor) + info, remote = os.fstat(descriptor), target.stat() + expected = (fact.device, fact.inode, fact.mode, fact.uid) + if (info.st_dev, info.st_ino, info.st_mode, info.st_uid) != expected or (remote.st_dev, remote.st_ino, remote.st_mode, remote.st_uid) != expected or os.readlink(target) != fact.path: + raise RecoveryContainmentUnavailable("held recovery resource changed") + if selected is None: + assert_prior_network_pin(descriptor, fact.path) + return + _private_lock(Path(fact.path), selected, uid=expected_uid) + observed = inspect_kernel_lock(resources.holder_pid, fact.holder_descriptor, fact.device, fact.inode) + if observed != fact.kernel_lock or observed is None or observed.pid != resources.holder_pid: + raise RecoveryContainmentUnavailable("recovery fence was inherited, released or changed") + _assert_pid_flock(resources.holder_pid, descriptor, observed.mode) + + check_holder() + for kind, selected_fds in (("original-sh", owner_fds), ("controller-ex", controller_fds)): + for name, remote_fd in sorted(selected_fds.items()): + selected = originals[name].owner if kind == "original-sh" else watchers[name].lock + if selected is None: + raise RecoveryContainmentUnavailable("original owner identity is absent") + expected = _private_lock(Path(selected.path), selected, uid=expected_uid) + remote_path = Path("/proc") / str(resources.holder_pid) / "fd" / str(remote_fd) + descriptor = os.open(remote_path, os.O_RDONLY | os.O_CLOEXEC | os.O_NONBLOCK) + stack.callback(os.close, descriptor) + info = os.fstat(descriptor) + if (info.st_dev, info.st_ino, info.st_mode, info.st_uid, info.st_nlink) != (expected.st_dev, expected.st_ino, expected.st_mode, expected.st_uid, expected.st_nlink): + raise RecoveryContainmentUnavailable("recovery issuer fence names another object") + lock = inspect_kernel_lock(resources.holder_pid, remote_fd, info.st_dev, info.st_ino) + mode = "READ" if kind == "original-sh" else "WRITE" + if lock is None or lock.mode != mode or lock.pid != resources.holder_pid: + raise RecoveryContainmentUnavailable("recovery issuer lacks independently acquired " + kind) + fact = RecoveryResourceFact(name, kind, remote_fd, selected.path, info.st_dev, info.st_ino, info.st_mode, info.st_uid, lock) + check_one(fact, descriptor, selected) + held.append((fact, descriptor, selected)) + for name, remote_fd in sorted(namespace_fds.items()): + original = originals[name] + pin = original.namespace_pin + if pin is None or original.boot_id != resources.holder_boot_id: + raise RecoveryContainmentUnavailable("original namespace boot or pin is absent") + stack.enter_context(_external_namespace_pin(pin, pin.namespace)) + target = Path("/proc") / str(resources.holder_pid) / "fd" / str(remote_fd) + descriptor = os.open(target, os.O_RDONLY | os.O_CLOEXEC | os.O_NONBLOCK) + stack.callback(os.close, descriptor) + info = os.fstat(descriptor) + if (info.st_dev, info.st_ino) != (pin.device, pin.inode): + raise RecoveryContainmentUnavailable("recovery namespace descriptor is not the retained original object") + fact = RecoveryResourceFact(name, "network-namespace", remote_fd, pin.namespace, info.st_dev, info.st_ino, info.st_mode, info.st_uid, None) + check_one(fact, descriptor, None) + held.append((fact, descriptor, None)) + check_holder() + result = RecoveryResourceObservation(resources.holder_pid, resources.holder_start_token, resources.holder_boot_id, tuple(item[0] for item in held)) + yield result + for fact, descriptor, selected in held: + check_one(fact, descriptor, selected) + check_holder() diff --git a/src/auto_coder/management/recovery_diagnostics.py b/src/auto_coder/management/recovery_diagnostics.py new file mode 100644 index 000000000..0117f942b --- /dev/null +++ b/src/auto_coder/management/recovery_diagnostics.py @@ -0,0 +1,53 @@ +"""Bounded non-authorizing code locations for recovery refusal diagnostics.""" + +from __future__ import annotations + +import re +from pathlib import Path + +_SYMBOL = r"[A-Za-z_][A-Za-z0-9_]{0,95}" +_FILENAME = r"[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}\.py" +_FUNCTION = _SYMBOL + r"|" + + +def refusal_causes(error: BaseException) -> list[dict[str, str | int]]: + """Describe bounded code locations, never exception data or frame locals.""" + causes: list[dict[str, str | int]] = [] + visited: set[int] = set() + current: BaseException | None = error + while current is not None and id(current) not in visited and len(causes) < 8: + visited.add(id(current)) + name = type(current).__name__ + row: dict[str, str | int] = {"type": name if re.fullmatch(_SYMBOL, name) else "Exception"} + if isinstance(current, OSError) and type(current.errno) is int: + row["errno"] = current.errno + trace = current.__traceback__ + if trace is not None: + while trace.tb_next is not None: + trace = trace.tb_next + code = trace.tb_frame.f_code + filename = Path(code.co_filename).name + if re.fullmatch(_FILENAME, filename) and re.fullmatch(_FUNCTION, code.co_name): + row.update(file=filename, function=code.co_name, line=trace.tb_lineno) + causes.append(row) + current = current.__cause__ + return causes + + +def safe_refusal_causes(value: object) -> list[dict[str, str | int]]: + """Copy only the same finite metadata from a subprocess's JSON result.""" + if not isinstance(value, list) or len(value) > 8: + return [] + causes: list[dict[str, str | int]] = [] + for row in value: + if not isinstance(row, dict) or not set(row) <= {"type", "errno", "file", "function", "line"}: + return [] + if not isinstance(row.get("type"), str) or re.fullmatch(_SYMBOL, row["type"]) is None: + return [] + if "errno" in row and type(row["errno"]) is not int: + return [] + location = {"file", "function", "line"} & row.keys() + if location and (location != {"file", "function", "line"} or not isinstance(row["file"], str) or re.fullmatch(_FILENAME, row["file"]) is None or not isinstance(row["function"], str) or re.fullmatch(_FUNCTION, row["function"]) is None or type(row["line"]) is not int or row["line"] <= 0): + return [] + causes.append(dict(row)) + return causes diff --git a/src/auto_coder/management/recovery_epoch.py b/src/auto_coder/management/recovery_epoch.py new file mode 100644 index 000000000..262cf96ad --- /dev/null +++ b/src/auto_coder/management/recovery_epoch.py @@ -0,0 +1,380 @@ +"""Deployment-fixed one-shot host issuer for management recovery ownership. + +Original evidence and current containment are independently re-derived here. +Only this issuer combines those facts with exact target READY and physically +verified consumer custody, durably consumes the epoch, and sends one grant. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import signal +import socket +import stat +import sys +import threading +from contextlib import ExitStack, contextmanager +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Iterator + +from auto_coder.logger_config import get_logger, setup_logger + +from .bootstrap_supervisor import ManagementBootstrapSupervisor +from .containment_kernel_subobservation import _require_capture_host +from .installed_bootstrap_mount_profile import _inspect as inspect_container +from .original_attempt_evidence import OriginalLineageEvidence, derive_lineage, verify_lineage +from .original_attempts import canonical, source_identity +from .quarantine_history import InspectionReport, QuarantineHistory, QuarantineJournal +from .recovery_containment import RecoveryContainmentReport, load_enrollment +from .recovery_containment import observe as observe_containment +from .recovery_containment import verify as verify_containment +from .recovery_containment_model import RecoveryEnrollment, RecoveryEnrollmentObservation +from .recovery_diagnostics import refusal_causes as _refusal_causes +from .recovery_epoch_authorization import ( + PEER_ROOT, + PEER_SOCKET, + AuthorizationObservation, + EpochAuthorization, + TargetReadyObservation, + check_authorization, + load_authorization, + observe_target_ready, + read_bound_file, + verify_target_ready, +) +from .recovery_epoch_consumer import GRANT_SCHEMA, parse_consumer_configuration +from .recovery_epoch_peer import ExpectedPeer, HeldFence, HeldResource, PeerCustody, PeerSession, encode_peer_message, inspect_held_resource +from .recovery_epoch_resources import EpochResources, hold_epoch_resources +from .recovery_epoch_state import EpochBinding, EpochInventory, EpochReceipt, EpochUnavailable, RecoveryEpochState, hold_recovery_epoch, issuer_lifetime + +logger = get_logger(__name__) +_OVERRIDES = ("AUTO_CODER_RECOVERY_EPOCH_ROOT", "AUTO_CODER_RECOVERY_EPOCH_INVENTORY", "AUTO_CODER_RECOVERY_AUTHORIZATION_ROOT", "AUTO_CODER_RECOVERY_CONTAINMENT_ROOT") + + +@dataclass(frozen=True) +class VerifiedEpochEvidence: + lineage: OriginalLineageEvidence + containment: RecoveryContainmentReport + target: TargetReadyObservation + inspection: InspectionReport + custody: PeerCustody + + +@dataclass(frozen=True) +class EpochDispatch: + receipt: EpochReceipt + evidence_sha256: str + dispatch_attempted: bool + # A successful send is not acknowledgment, child launch or current readiness. + current_ready: bool = False + + +def join_lineage(authorization: EpochAuthorization, inventory: EpochInventory, enrollment: RecoveryEnrollmentObservation, lineage: OriginalLineageEvidence, containment: RecoveryContainmentReport) -> None: + """Match actual complete original derivation to the entire observed scope.""" + selected = enrollment.enrollment + if lineage.unavailable or lineage.genesis is None or not lineage.generations or lineage.binding != authorization.history: + raise EpochUnavailable("complete original genesis and retained lineage are unavailable") + old_attempts = {generation.request_id for generation in lineage.generations} + if authorization.request_id in old_attempts or authorization.authorization_id == lineage.genesis.authorization_id: + raise EpochUnavailable("recovery requires new authorization and a new request identity") + if containment.status != "observed-current-recovery-containment" or containment.grant_issued or containment.history_asserted or containment.admission_created or containment.enrollment != enrollment: + raise EpochUnavailable("current containment is not the exact non-authorizing prerequisite") + if selected.deployment_id != lineage.binding.deployment or selected.source_identity != lineage.binding.source or selected.source_identity != source_identity() or selected.source_identity != "sha256:" + inventory.source_sha256: + raise EpochUnavailable("original and current source/deployment identities differ") + if selected.target.management_id != inventory.target_id or selected.target.bootstrap_config.sha256 != inventory.config_sha256: + raise EpochUnavailable("exact target differs from deployment-fixed inventory") + if selected.engine_id != lineage.genesis.engine_id: + raise EpochUnavailable("current Docker engine differs from original genesis") + original_by_id = {item.management_id: item for item in selected.originals} + generations = {item.management_id: item for item in lineage.generations} + facts = {item.container_id: item for item in containment.managers} + if len(generations) != len(lineage.generations) or len(facts) != len(containment.managers) or set(original_by_id) - set(generations): + raise EpochUnavailable("current recovery enrollment omits or aliases original lineage") + if set(facts) != {*original_by_id, selected.target.management_id}: + raise EpochUnavailable("current effective manager scope differs from the exact enrollment") + for identifier, generation in generations.items(): + original = original_by_id.get(identifier) + if original is None: + if generation.outcome != "never-created" or identifier in containment.docker_ids or generation.owner_inode: + raise EpochUnavailable("original generation is absent from current full-scope containment") + continue + fact = facts[identifier] + if (original.instance_id, original.management_image_id, original.bootstrap_config.sha256, original.management_config.sha256) != (generation.instance_id, generation.image_id, generation.bootstrap_config_sha256, generation.management_config_sha256): + raise EpochUnavailable("current original configuration differs from authoritative lineage") + if generation.proxy_id and (original.proxy_id, original.proxy_image_id) != (generation.proxy_id, generation.proxy_image_id): + raise EpochUnavailable("original proxy differs from genesis provenance") + if (fact.state_root, fact.journal_root, fact.oauth_root) != (generation.state_root, generation.journal_root, generation.oauth_root): + raise EpochUnavailable("original root mapping differs from current effective configuration") + for path, device, inode in ((generation.state_root, generation.state_device, generation.state_inode), (generation.journal_root, generation.journal_device, generation.journal_inode), (generation.oauth_root, generation.oauth_device, generation.oauth_inode)): + roots = [mount for mount in fact.mounts if mount.source == path and mount.writable] + if len(roots) != 1 or (roots[0].device, roots[0].inode) != (device, inode): + raise EpochUnavailable("original root inode differs from current Docker mapping") + if generation.outcome == "never-created": + if original.state != "never-started" or original.owner is not None or generation.owner_inode: + raise EpochUnavailable("original never-created boundary conflicts with current facts") + elif original.state != "started" or original.owner is None or (original.owner.path, original.owner.device, original.owner.inode) != (generation.owner_path, generation.owner_device, generation.owner_inode): + raise EpochUnavailable("current original owner differs from native original provenance") + target = facts[selected.target.management_id] + oauth = lineage.genesis.root("oauth") + roots = [item for item in target.mounts if item.writable and item.source == oauth.host_path] + if target.oauth_root != oauth.host_path or len(roots) != 1 or (roots[0].device, roots[0].inode) != (oauth.device, oauth.inode): + raise EpochUnavailable("recovery target does not preserve the exact original OAuth store") + original_paths = [Path(value) for generation in lineage.generations for value in (generation.state_root, generation.journal_root, generation.oauth_root)] + for target_root in (Path(target.state_root), Path(target.journal_root)): + if any(target_root == original or target_root in original.parents or original in target_root.parents for original in original_paths): + raise EpochUnavailable("recovery target reuses or overlaps an original root") + + +def _quarantine_entries(lineage: OriginalLineageEvidence) -> tuple[QuarantineJournal, ...]: + originals = {item.path: item for item in lineage.originals} + result = [] + for generation in lineage.generations: + if generation.outcome == "never-created": + continue + stores = dict(generation.expected_stores) + original = originals.get(stores.get("operations", "")) + owner = originals.get(generation.owner_path) + if original is None or owner is None: + raise EpochUnavailable("original journal or owner fingerprint is missing") + database, fence = Path(original.path), Path(owner.path) + database_info, fence_info = database.lstat(), fence.lstat() + result.append(QuarantineJournal(generation.instance_id, database, fence, original.device, original.inode, original.sha256, database_info.st_ctime_ns, owner.device, owner.inode, fence_info.st_ctime_ns)) + return tuple(result) + + +def _check_consumer_scope(authorization: EpochAuthorization, enrollment: RecoveryEnrollment, resources: EpochResources) -> None: + selected = parse_consumer_configuration(read_bound_file(authorization.consumer_config)) + if (selected.request_id, selected.authorization_id, selected.target_id, selected.issuer_uid, selected.socket_directory_device, selected.socket_directory_inode) != ( + authorization.request_id, + authorization.authorization_id, + authorization.target_id, + 0, + authorization.socket_directory_device, + authorization.socket_directory_inode, + ): + raise EpochUnavailable("consumer's pinned authorization differs") + peer_mounts = [item for item in enrollment.target.mounts if item.destination == "/management-recovery"] + if len(peer_mounts) != 1 or peer_mounts[0].source != str(PEER_ROOT) or peer_mounts[0].writable or (peer_mounts[0].device, peer_mounts[0].inode) != (authorization.socket_directory_device, authorization.socket_directory_inode): + raise EpochUnavailable("consumer lacks the exact read-only issuer endpoint mount") + original_paths = {"history": ("/management-recovery-history/history.lock", str(Path(authorization.history.root) / "history.lock"))} + original_paths.update({f"owner-{index}": (f"/management-recovery-originals/{index}/owner.lock", item.owner.path) for index, item in enumerate(enrollment.originals) if item.owner is not None}) + held: tuple[HeldFence | HeldResource, ...] = (*resources.fences, *resources.namespaces) + observed = tuple(inspect_held_resource(item) for item in held) + originals = {(item.name, item.device, item.inode) for item in observed if item.kind == "original-sh"} + if {(item.name, item.device, item.inode) for item in selected.original_fences} != originals or len(selected.original_fences) != len(originals): + raise EpochUnavailable("consumer omitted or changed an independently acquired original fence") + for original in selected.original_fences: + container_path, host_path = original_paths[original.name] + mounts = [item for item in enrollment.target.mounts if item.destination == container_path] + if original.path != container_path or len(mounts) != 1 or mounts[0].source != host_path or mounts[0].writable or (mounts[0].device, mounts[0].inode) != (original.device, original.inode): + raise EpochUnavailable("consumer original fence is not the exact read-only mounted inode") + target = next(item for item in observed if item.kind == "target-ex") + if (selected.target_owner.name, selected.target_owner.device, selected.target_owner.inode) != (target.name, target.device, target.inode): + raise EpochUnavailable("consumer's independent target owner differs") + transfers = {(item.name, item.kind, item.device, item.inode) for item in observed if item.kind in ("transferred-ex", "network-namespace")} + if {(item.name, item.kind, item.device, item.inode) for item in selected.transferred_resources} != transfers or len(selected.transferred_resources) != len(transfers): + raise EpochUnavailable("consumer omitted or changed controller, deployment or namespace custody") + + +def _expected_peer(authorization: EpochAuthorization) -> ExpectedPeer: + actual = inspect_container(authorization.target_id) + state = actual.get("State") + if actual.get("Id") != authorization.target_id or not isinstance(state, dict) or state.get("Running") is not True or type(state.get("Pid")) is not int or state["Pid"] <= 1: + raise EpochUnavailable("exact recovery target is not a live bootstrap consumer") + pid = state["Pid"] + token = ManagementBootstrapSupervisor._start_token(pid) + if not token: + raise EpochUnavailable("exact recovery target process lifetime is unavailable") + return ExpectedPeer(pid, token, issuer_lifetime()[0], 0, authorization.request_id, authorization.authorization_id, authorization.target_id) + + +def _match_peer_report(session: PeerSession, report: RecoveryContainmentReport) -> None: + if (report.target_pid, report.target_start_token, report.facts.target.boot_id) != (session.expected.pid, session.expected.start_token, session.expected.boot_id): + raise EpochUnavailable("authenticated peer differs from currently observed Docker target") + + +def _binding(state: RecoveryEpochState, authorization: AuthorizationObservation, evidence: VerifiedEpochEvidence) -> EpochBinding: + boot, pid, start = issuer_lifetime() + # Retain all original file fingerprints and current scope/custody identities; + # the independently verified controller/kernel report is bound in full by + # its canonical evidence digest, not by an unverified success flag. + original = { + "schema": "verified-management-recovery-evidence-v1", + "authorization": asdict(authorization), + "lineage": asdict(evidence.lineage), + "containment_sha256": evidence.containment.evidence_sha256, + "containment_enrollment": asdict(evidence.containment.enrollment), + "controller_fences": [asdict(item) for item in evidence.containment.facts.watchers], + "current_holder": asdict(evidence.containment.facts.resources), + "target": asdict(evidence.target), + "inspection": asdict(evidence.inspection), + "custody": asdict(evidence.custody), + } + encoded = canonical(original) + inventory = state.inventory + return EpochBinding( + inventory.deployment_id, + inventory.authorization_id, + inventory.enrollment_ids, + inventory.request_id, + inventory.preparation_id, + inventory.target_id, + inventory.source_sha256, + inventory.config_sha256, + hashlib.sha256(encoded).hexdigest(), + boot, + pid, + start, + inventory.authorization_sha256, + encoded.decode(), + ) + + +def issue_epoch_once(state: RecoveryEpochState, authorization: AuthorizationObservation, enrollment: RecoveryEnrollmentObservation, resources: EpochResources, session: PeerSession) -> tuple[EpochDispatch, PeerCustody]: + """Verify, irreversibly consume and attempt exactly one current peer grant.""" + selected = enrollment.enrollment + approved = authorization.authorization + check_authorization(authorization, state.inventory) + _check_consumer_scope(approved, selected, resources) + target = observe_target_ready(approved, selected) + lineage = derive_lineage(approved.history, resources.history_descriptor, resources.original_descriptors) + lineage = verify_lineage(lineage, approved.history, resources.history_descriptor, resources.original_descriptors) + with QuarantineHistory(_quarantine_entries(lineage)) as history: + inspection = history.inspect() + containment = observe_containment(approved.enrollment_id, resources.containment) + join_lineage(approved, state.inventory, enrollment, lineage, containment) + _match_peer_report(session, containment) + custody = session.offer_and_verify_custody(resources.fences, resources.namespaces) + evidence = VerifiedEpochEvidence(lineage, containment, target, inspection, custody) + binding = _binding(state, authorization, evidence) + state.reserve(binding) + state.record_intent(binding) + + def final_verification() -> None: + check_authorization(authorization, state.inventory) + if load_enrollment(approved.enrollment_id) != enrollment: + raise EpochUnavailable("protected recovery enrollment changed at issuance") + verify_target_ready(approved, selected, target) + verify_lineage(lineage, approved.history, resources.history_descriptor, resources.original_descriptors) + if history.inspect() != inspection: + raise EpochUnavailable("quarantined original exclusions changed before grant") + current = verify_containment(approved.enrollment_id, containment, resources.containment) + join_lineage(approved, state.inventory, enrollment, lineage, current) + _match_peer_report(session, current) + _check_consumer_scope(approved, selected, resources) + resources.check() + session.check(custody) + state.check() + + final_verification() + receipt = state.consume(binding) + # Persistence can block or be interrupted. Its completion is never + # substituted for a current containment/custody/READY observation. + final_verification() + packet = encode_peer_message( + { + "schema": GRANT_SCHEMA, + "granted": "recovery-epoch-consumed", + **{name: getattr(approved, name) for name in ("request_id", "authorization_id", "target_id")}, + "challenge": custody.challenge, + "receipt_nonce": custody.receipt_nonce, + **{name: getattr(receipt, name) for name in ("inventory_sha256", "binding_sha256", "consumed_sha256", "receipt_sha256")}, + "host_boot_id": binding.host_boot_id, + "host_pid": binding.host_pid, + "host_start_token": binding.host_start_token, + } + ) + session.check(custody) + state.check() + # A partial/failed send, lost reply or exception has no retry path. + session.connection.settimeout(10) + session.connection.sendall(packet) + logger.bind(recovery_epoch_phase="grant-send-completed", request_id=approved.request_id, target_id=approved.target_id, current_ready=False).info("One consumed recovery epoch grant send completed; delivery and readiness are not inferred") + return EpochDispatch(receipt, binding.evidence_sha256, True), custody + + +@contextmanager +def _listen(authorization: AuthorizationObservation) -> Iterator[socket.socket]: + root = PEER_ROOT.stat() + expected = authorization.authorization + if (root.st_dev, root.st_ino) != (expected.socket_directory_device, expected.socket_directory_inode) or PEER_SOCKET.exists() or PEER_SOCKET.is_symlink(): + raise EpochUnavailable("fixed recovery socket directory or original endpoint is unavailable") + listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + bound = None + try: + listener.bind(str(PEER_SOCKET)) + os.chmod(PEER_SOCKET, 0o600) + bound = PEER_SOCKET.lstat() + if not stat.S_ISSOCK(bound.st_mode) or bound.st_uid != 0: + raise EpochUnavailable("recovery local endpoint is not protected") + listener.listen(1) + listener.settimeout(600) + logger.bind(recovery_epoch_phase="listening", request_id=expected.request_id, target_id=expected.target_id).info("Fixed recovery issuer endpoint is listening for its exact target") + yield listener + finally: + listener.close() + if bound is not None: + try: + current = PEER_SOCKET.lstat() + if (current.st_dev, current.st_ino) == (bound.st_dev, bound.st_ino): + PEER_SOCKET.unlink() + except FileNotFoundError: + pass + + +def run_issuer() -> EpochDispatch: + """Own one current epoch until the target exits or its custody is lost.""" + if any(name in os.environ for name in _OVERRIDES): + raise EpochUnavailable("request/environment recovery inventory overrides are unsupported") + _require_capture_host() + stopped = threading.Event() + handlers: dict[int, object] = {} + with ExitStack() as held: + state = held.enter_context(hold_recovery_epoch()) + authorization = load_authorization(state.inventory) + enrollment = load_enrollment(authorization.authorization.enrollment_id) + resources = held.enter_context(hold_epoch_resources(authorization.authorization, enrollment.enrollment, state)) + listener = held.enter_context(_listen(authorization)) + connection, _ = listener.accept() + try: + session = held.enter_context(PeerSession(connection, _expected_peer(authorization.authorization), timeout=60)) + except BaseException: + connection.close() + raise + listener.close() + dispatch, custody = issue_epoch_once(state, authorization, enrollment, resources, session) + for signum in (signal.SIGTERM, signal.SIGINT): + handlers[signum] = signal.signal(signum, lambda *_: stopped.set()) + try: + while not stopped.wait(0.2): + check_authorization(authorization, state.inventory) + resources.check() + session.check(custody) + state.check() + return dispatch + finally: + for saved_signal, handler in handlers.items(): + signal.signal(saved_signal, handler) # type: ignore[arg-type] + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Issue one approved management ownership epoch from fixed host authority") + parser.parse_args(argv) + setup_logger(stream=sys.stdout, log_file="/var/log/auto-coder/recovery-epoch.log") + try: + result = run_issuer() + except (OSError, ValueError, TypeError, KeyError, RuntimeError) as error: + logger.bind(recovery_epoch_phase="unavailable", current_ready=False).error("Recovery epoch unavailable; preserved evidence is not repaired: {}", error) + for index, cause in enumerate(_refusal_causes(error), start=1): + logger.bind(recovery_epoch_phase="unavailable", current_ready=False).error("Recovery epoch refusal cause {}: {}", index, json.dumps(cause, sort_keys=True, separators=(",", ":"))) + return 2 + print(json.dumps(asdict(result), sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/auto_coder/management/recovery_epoch_authorization.py b/src/auto_coder/management/recovery_epoch_authorization.py new file mode 100644 index 000000000..09c37a65e --- /dev/null +++ b/src/auto_coder/management/recovery_epoch_authorization.py @@ -0,0 +1,243 @@ +"""Protected fixed recovery authorization and independently checked READY. + +An authorization selects a finite scope. It neither establishes original +history nor replaces the issuer's current Docker, kernel and custody checks. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +from dataclasses import asdict, dataclass +from pathlib import Path + +from auto_coder.operating_policy import MANAGEMENT_REPOSITORY + +from .archived_exclusions import unique_pairs +from .bootstrap_supervisor import BootstrapConfig +from .bootstrap_upgrade import _source_authority +from .containment_enrollment import _approved_directory +from .original_attempts import HistoryBinding, canonical, read_original +from .recovery_containment_model import RecoveryEnrollment, RecoveryFile +from .recovery_epoch_state import EpochInventory, EpochUnavailable +from .self_update_handoff import runtime_for_receipt +from .version_staging import ManagementReadyReceipt, ManagementRuntimeConfig, ManagementVersionStager + +AUTHORIZATION_ROOT = Path("/etc/auto-coder/recovery-epoch") +AUTHORIZATION_FILE = AUTHORIZATION_ROOT / "authorization.json" +PEER_ROOT = Path("/var/lib/auto-coder/recovery-epoch-peer") +PEER_SOCKET = PEER_ROOT / "issuer.sock" +AUTHORIZATION_SCHEMA = "management-recovery-authorization-v1" +_SHA = re.compile(r"[0-9a-f]{40}\Z", re.ASCII) +_TOKEN = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}\Z", re.ASCII) +_UV_VERSION = re.compile(r"[0-9]+\.[0-9]+\.[0-9]+(?: \([A-Za-z0-9_.-](?:[A-Za-z0-9_. -]{0,126}[A-Za-z0-9_.-])?\))?\Z", re.ASCII) + + +def validate_uv_version(value: object) -> str: + """Retain the exact installer pin, including its bounded build descriptor.""" + if not isinstance(value, str) or len(value) > 128 or not _UV_VERSION.fullmatch(value): + raise EpochUnavailable("exact configured installer version required") + return value + + +@dataclass(frozen=True) +class EpochAuthorization: + schema: str + deployment_id: str + authorization_id: str + request_id: str + preparation_id: str + target_id: str + enrollment_id: str + history: HistoryBinding + consumer_config: RecoveryFile + ready_receipt: RecoveryFile + source_repository: str + source_token_file: str + python_executable: str + uv_executable: str + uv_version: str + source_sha: str + socket_directory_device: int + socket_directory_inode: int + + @classmethod + def parse(cls, value: object) -> EpochAuthorization: + if not isinstance(value, dict) or set(value) != set(cls.__dataclass_fields__): + raise EpochUnavailable("complete fixed recovery authorization required") + raw = dict(value) + history = raw["history"] + if not isinstance(history, dict) or set(history) != set(HistoryBinding.__dataclass_fields__) or any(not isinstance(item, str) for item in history.values()): + raise EpochUnavailable("original history authorization is incomplete") + raw["history"] = HistoryBinding(**history) + raw["consumer_config"] = RecoveryFile.parse(raw["consumer_config"]) + raw["ready_receipt"] = RecoveryFile.parse(raw["ready_receipt"]) + result = cls(**raw) + if result.schema != AUTHORIZATION_SCHEMA: + raise EpochUnavailable("unsupported recovery authorization schema") + for field in ("deployment_id", "authorization_id", "request_id", "preparation_id", "enrollment_id"): + token = getattr(result, field) + if not isinstance(token, str) or not _TOKEN.fullmatch(token): + raise EpochUnavailable("invalid recovery authorization " + field) + if not isinstance(result.target_id, str) or not re.fullmatch(r"[0-9a-f]{64}", result.target_id) or not isinstance(result.source_sha, str) or not _SHA.fullmatch(result.source_sha): + raise EpochUnavailable("exact recovery target and source required") + if result.history.deployment != result.deployment_id: + raise EpochUnavailable("history names another recovery deployment") + for path in (result.history.root, result.source_repository, result.source_token_file, result.python_executable, result.uv_executable): + _path(path) + validate_uv_version(result.uv_version) + for field in ("socket_directory_device", "socket_directory_inode"): + if type(getattr(result, field)) is not int or getattr(result, field) <= 0: + raise EpochUnavailable("fixed recovery peer directory identity required") + return result + + +@dataclass(frozen=True) +class AuthorizationObservation: + authorization: EpochAuthorization + original: RecoveryFile + + +@dataclass(frozen=True) +class TargetReadyObservation: + receipt: ManagementReadyReceipt + bootstrap_config: RecoveryFile + management_config: RecoveryFile + consumer_config: RecoveryFile + ready_receipt: RecoveryFile + runtime_sha256: str + + +def _path(value: object) -> Path: + if not isinstance(value, str) or not value or len(value) > 4096: + raise EpochUnavailable("invalid recovery authority path") + path = Path(value) + if not path.is_absolute() or str(path) != value or path == Path("/") or ".." in path.parts or "\x00" in value: + raise EpochUnavailable("recovery authority path is not canonical") + return path + + +def read_bound_file(bound: RecoveryFile) -> bytes: + """Read the original protected file, retaining its actual inode and bytes.""" + path = _path(bound.path) + _approved_directory(path.parent, uid=os.geteuid()) + data, identity = read_original(path) + if identity != (bound.device, bound.inode) or hashlib.sha256(data).hexdigest() != bound.sha256: + raise EpochUnavailable("recovery original file changed: " + bound.path) + return data + + +def _load_authorization(path: Path, inventory: EpochInventory) -> AuthorizationObservation: + """Explicit component seam; the production factory uses one fixed path.""" + if inventory.schema_version != 2 or not inventory.authorization_sha256: + raise EpochUnavailable("complete issuer requires the fixed version-two inventory") + data, (device, inode) = read_original(path) + digest = hashlib.sha256(data).hexdigest() + if digest != inventory.authorization_sha256: + raise EpochUnavailable("recovery authorization differs from fixed inventory") + raw = json.loads(data, object_pairs_hook=unique_pairs) + if canonical(raw) != data: + raise EpochUnavailable("recovery authorization is not canonical") + authorization = EpochAuthorization.parse(raw) + for field in ("deployment_id", "authorization_id", "request_id", "preparation_id", "target_id"): + if getattr(authorization, field) != getattr(inventory, field): + raise EpochUnavailable("recovery authorization differs from inventory: " + field) + if inventory.enrollment_ids != (authorization.enrollment_id,): + raise EpochUnavailable("recovery enrollment differs from complete fixed scope") + return AuthorizationObservation(authorization, RecoveryFile(str(path), digest, device, inode)) + + +def load_authorization(inventory: EpochInventory) -> AuthorizationObservation: + """Read the deployment-fixed authorization; no CLI or environment root.""" + _approved_directory(AUTHORIZATION_ROOT, uid=0) + result = _load_authorization(AUTHORIZATION_FILE, inventory) + check_authorization(result, inventory) + return result + + +def check_authorization(previous: AuthorizationObservation, inventory: EpochInventory) -> None: + if _load_authorization(Path(previous.original.path), inventory) != previous: + raise EpochUnavailable("recovery authorization changed after observation") + if Path(previous.original.path) != AUTHORIZATION_FILE: + raise EpochUnavailable("recovery authorization did not use the fixed production path") + _approved_directory(PEER_ROOT, uid=0) + peer = PEER_ROOT.stat() + if (peer.st_dev, peer.st_ino) != (previous.authorization.socket_directory_device, previous.authorization.socket_directory_inode): + raise EpochUnavailable("fixed recovery peer directory was replaced") + + +def _same_path_version_mount(enrollment: RecoveryEnrollment, config: BootstrapConfig) -> None: + version = config.management_version_root + if version is None: + raise EpochUnavailable("recovery target has no installed runtime root") + selected = [mount for mount in enrollment.target.mounts if mount.container_id == enrollment.target.management_id and mount.destination == str(version)] + if len(selected) != 1 or selected[0].source != str(version) or selected[0].writable: + raise EpochUnavailable("recovery READY requires its exact original same-path read-only version mount") + + +def observe_target_ready(authorization: EpochAuthorization, enrollment: RecoveryEnrollment) -> TargetReadyObservation: + """Invoke the actual stager verifier against the retained installed READY.""" + target = enrollment.target + if (enrollment.deployment_id, enrollment.authorization_id, enrollment.enrollment_id, target.management_id) != (authorization.deployment_id, authorization.authorization_id, authorization.enrollment_id, authorization.target_id): + raise EpochUnavailable("current target enrollment differs from recovery authorization") + read_bound_file(target.bootstrap_config) + read_bound_file(target.management_config) + read_bound_file(authorization.consumer_config) + ready_bytes = read_bound_file(authorization.ready_receipt) + config = BootstrapConfig.read(Path(target.bootstrap_config.path)) + if config.instance_id != target.instance_id or config.management_config is None or config.management_version_root is None or config.recovery_epoch_config != Path("/bootstrap/recovery-epoch.json"): + raise EpochUnavailable("exact recovery bootstrap profile is unavailable") + _same_path_version_mount(enrollment, config) + mounts = {mount.destination: mount for mount in target.mounts} + consumer_mount = mounts.get("/bootstrap/recovery-epoch.json") + if consumer_mount is None or consumer_mount.source != authorization.consumer_config.path or consumer_mount.writable: + raise EpochUnavailable("target recovery consumer configuration mount differs") + runtime = next(item for item in config.runtimes if item.reference == config.initial_runtime) + if runtime.execution_kind != "management-installed" or runtime.ready_receipt != Path(authorization.ready_receipt.path): + raise EpochUnavailable("recovery target requires the authorized original installed READY") + # Host verification and the exact same-path RO mount must refer to the + # same complete interpreter distribution. A host-only symlink target can + # otherwise produce valid host READY while the Docker child is unusable. + resolved_interpreter = runtime.executable.resolve(strict=True) + version_root = config.management_version_root.resolve(strict=True) + if not resolved_interpreter.is_relative_to(version_root) or resolved_interpreter != Path(authorization.python_executable).resolve(strict=True): + raise EpochUnavailable("recovery interpreter escapes the exactly mounted runtime closure") + safe_python = {"PYTHONDONTWRITEBYTECODE": "1", "PYTHONNOUSERSITE": "1", "PYTHONUNBUFFERED": "1"} + if any(name.startswith("LD_") or (name.startswith("PYTHON") and safe_python.get(name) != value) for name, value in runtime.environment): + raise EpochUnavailable("recovery target runtime contains unsupported executable hooks") + authority = _source_authority(Path(authorization.source_token_file)) + staging = ManagementVersionStager( + ManagementRuntimeConfig( + config.instance_id, + MANAGEMENT_REPOSITORY, + Path(authorization.source_repository), + config.management_version_root, + Path(authorization.python_executable), + Path(authorization.uv_executable), + authorization.uv_version, + protected_roots=(config.state_root, config.oauth_root, config.journal_root), + management_config=Path(target.management_config.path), + source_authority=authority, + ) + ) + receipt = staging.lookup(Path(authorization.ready_receipt.path)) + if receipt.source_sha != authorization.source_sha or receipt.instance_id != target.instance_id or json.loads(ready_bytes, object_pairs_hook=unique_pairs) != asdict(receipt): + raise EpochUnavailable("genuine installed READY differs from exact authorization") + if Path(receipt.environment_path).parent.name != authorization.preparation_id: + raise EpochUnavailable("installed runtime belongs to another exact preparation") + expected = runtime_for_receipt(config, receipt) + if runtime != expected: + raise EpochUnavailable("target invocation differs from its original installed runtime") + for original in (target.bootstrap_config, target.management_config, authorization.consumer_config, authorization.ready_receipt): + read_bound_file(original) + runtime_bytes = json.dumps(asdict(runtime), sort_keys=True, separators=(",", ":"), default=str, allow_nan=False).encode() + return TargetReadyObservation(receipt, target.bootstrap_config, target.management_config, authorization.consumer_config, authorization.ready_receipt, hashlib.sha256(runtime_bytes).hexdigest()) + + +def verify_target_ready(authorization: EpochAuthorization, enrollment: RecoveryEnrollment, previous: TargetReadyObservation) -> TargetReadyObservation: + current = observe_target_ready(authorization, enrollment) + if current != previous: + raise EpochUnavailable("target source, configuration or READY changed before grant") + return current diff --git a/src/auto_coder/management/recovery_epoch_bootstrap.py b/src/auto_coder/management/recovery_epoch_bootstrap.py new file mode 100644 index 000000000..4706c5ed7 --- /dev/null +++ b/src/auto_coder/management/recovery_epoch_bootstrap.py @@ -0,0 +1,83 @@ +"""Explicit bootstrap custody endpoint for the one-use recovery issuer. + +This stage owns no child launch, durable consumer launch claim, application +readiness or operation admission. Those belong to the separate epoch consumer. +""" + +from __future__ import annotations + +import fcntl +import os +import signal +import stat +import sys +import threading +from pathlib import Path +from typing import TYPE_CHECKING + +from auto_coder.logger_config import get_logger, setup_logger + +from .original_attempts import identity +from .recovery_epoch_consumer import CONSUMER_CONFIG, connect_recovery_epoch + +if TYPE_CHECKING: + from .bootstrap_supervisor import BootstrapConfig + +logger = get_logger(__name__) + + +class RecoveryBootstrapCancelled(ValueError): + """Terminate custody immediately; this endpoint never owns a live child.""" + + +def _open_target_owner(config: BootstrapConfig) -> int: + config.validate() + if config.recovery_epoch_config != CONSUMER_CONFIG: + raise ValueError("recovery custody requires its fixed selected bootstrap profile") + identity(config.state_root, directory=True) + # The separately authorized preparation must already provide the exact + # target file. This path cannot create or repair an old or missing owner. + path = config.state_root / ".supervisor-owner.lock" + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC) + try: + actual = os.fstat(descriptor) + if identity(path) != (actual.st_dev, actual.st_ino) or not stat.S_ISREG(actual.st_mode) or stat.S_IMODE(actual.st_mode) != 0o600 or actual.st_uid != os.geteuid() or actual.st_nlink != 1: + raise ValueError("recovery target owner is not the protected original file") + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + if set(os.listdir(config.state_root)) - {".supervisor-owner.lock", "logs"} or os.listdir(config.journal_root): + raise ValueError("recovery target contains prior or uncertain launch state") + return descriptor + except BaseException: + os.close(descriptor) + raise + + +def run_recovery_epoch_bootstrap(config: BootstrapConfig) -> int: + """Retain acknowledged fences while the authenticated host remains live.""" + owner = -1 + pause = threading.Event() + previous_handlers: dict[int, object] = {} + try: + owner = _open_target_owner(config) + setup_logger(stream=sys.stdout, log_file=str(config.state_root / "logs" / "recovery-epoch-bootstrap.log")) + + def cancel(*_: object) -> None: + raise RecoveryBootstrapCancelled("recovery custody was cancelled") + + for signum in (signal.SIGTERM, signal.SIGINT): + previous_handlers[signum] = signal.signal(signum, cancel) + with connect_recovery_epoch(CONSUMER_CONFIG, owner, timeout=config.verification_timeout) as consumer: + logger.bind(recovery_epoch_phase="consumer-custody", current_ready=False, child_started=False).info("Recovery custody acknowledged; no child launch or readiness is established") + while True: + consumer.check_current() + pause.wait(0.2) + except (OSError, ValueError, TypeError, KeyError) as error: + logger.bind(recovery_epoch_phase="consumer-unavailable", current_ready=False, child_started=False).error("Recovery custody unavailable: {}", error) + return 1 + finally: + for saved_signal, handler in previous_handlers.items(): + signal.signal(saved_signal, handler) # type: ignore[arg-type] + if owner >= 0: + # This endpoint never launches a child, so it has no live child + # whose containment could outlast these retained fences. + os.close(owner) diff --git a/src/auto_coder/management/recovery_epoch_consumer.py b/src/auto_coder/management/recovery_epoch_consumer.py new file mode 100644 index 000000000..779d4324a --- /dev/null +++ b/src/auto_coder/management/recovery_epoch_consumer.py @@ -0,0 +1,414 @@ +"""Retain recovery issuer/consumer custody before any child launch authority. + +This helper does not claim a child, create stores, launch, or declare readiness. +The owning Supervisor must retain the returned handle until it can safely end +custody. A current-host failure never implicitly releases already received FDs. +""" + +from __future__ import annotations + +import fcntl +import json +import os +import re +import secrets +import select +import socket +import stat +import struct +import time +import uuid +from dataclasses import dataclass +from pathlib import Path +from typing import Literal, cast + +from .archived_exclusions import unique_pairs +from .original_attempts import read_original +from .recovery_epoch_peer import CHALLENGE_SCHEMA, MAX_MESSAGE_BYTES, MAX_RESOURCES, OFFER_SCHEMA, RECEIPT_SCHEMA, HeldFence, KernelLock, PeerUnavailable, ResourceIdentity, _name, _reject_constant, _token, encode_peer_message, inspect_held_resource, inspect_kernel_lock, receive_peer_message + +CONSUMER_CONFIG = Path("/bootstrap/recovery-epoch.json") +ISSUER_DIRECTORY = Path("/management-recovery") +ISSUER_SOCKET = ISSUER_DIRECTORY / "issuer.sock" +GRANT_SCHEMA = "management-recovery-epoch-grant-v1" +# Linux UAPI asm-generic/socket.h; unavailable kernels refuse without a fallback. +_SO_PEERPIDFD = getattr(socket, "SO_PEERPIDFD", 77) + + +@dataclass(frozen=True) +class OriginalFence: + name: str + path: str + device: int + inode: int + + +@dataclass(frozen=True) +class TargetOwner: + name: str + device: int + inode: int + + +@dataclass(frozen=True) +class TransferredResource: + name: str + kind: Literal["transferred-ex", "network-namespace"] + device: int + inode: int + + +@dataclass(frozen=True) +class ConsumerConfiguration: + schema_version: int + request_id: str + authorization_id: str + target_id: str + issuer_uid: int + socket_directory_device: int + socket_directory_inode: int + original_fences: tuple[OriginalFence, ...] + target_owner: TargetOwner + transferred_resources: tuple[TransferredResource, ...] + + +@dataclass(frozen=True) +class RecoveryGrant: + request_id: str + authorization_id: str + target_id: str + challenge: str + receipt_nonce: str + inventory_sha256: str + binding_sha256: str + consumed_sha256: str + receipt_sha256: str + host_boot_id: str + host_pid: int + host_start_token: str + + +@dataclass(frozen=True) +class ConsumerTransfer: + expected: TransferredResource + descriptor: int + mode: int + uid: int + link: str + lock: KernelLock | None + + +def _identity(value: object) -> bool: + return type(value) is int and value > 0 + + +def parse_consumer_configuration(encoded: bytes) -> ConsumerConfiguration: + """Parse a complete pinned target configuration, not host authorization.""" + if len(encoded) > MAX_MESSAGE_BYTES: + raise PeerUnavailable("consumer configuration exceeds its bound") + value = json.loads(encoded, object_pairs_hook=unique_pairs, parse_constant=_reject_constant) + if not isinstance(value, dict) or set(value) != set(ConsumerConfiguration.__dataclass_fields__): + raise PeerUnavailable("consumer configuration schema differs") + if type(value["schema_version"]) is not int or value["schema_version"] != 1 or type(value["issuer_uid"]) is not int or value["issuer_uid"] < 0: + raise PeerUnavailable("consumer configuration version or issuer differs") + if not all(_name(value[name]) for name in ("request_id", "authorization_id", "target_id")) or not all(_identity(value[name]) for name in ("socket_directory_device", "socket_directory_inode")): + raise PeerUnavailable("consumer binding or socket directory identity is invalid") + originals, transferred, target = value["original_fences"], value["transferred_resources"], value["target_owner"] + if not isinstance(originals, list) or not originals or not isinstance(transferred, list) or not transferred or len(originals) + len(transferred) + 1 > MAX_RESOURCES: + raise PeerUnavailable("consumer resource set is incomplete or oversized") + if not isinstance(target, dict) or set(target) != set(TargetOwner.__dataclass_fields__): + raise PeerUnavailable("consumer target owner binding is incomplete") + for item in originals: + if not isinstance(item, dict) or set(item) != set(OriginalFence.__dataclass_fields__) or not isinstance(item.get("path"), str) or not Path(item["path"]).is_absolute() or "\x00" in item["path"]: + raise PeerUnavailable("consumer original fence binding is incomplete") + for item in transferred: + if not isinstance(item, dict) or set(item) != set(TransferredResource.__dataclass_fields__) or item.get("kind") not in ("transferred-ex", "network-namespace"): + raise PeerUnavailable("consumer transferred resource binding is incomplete") + entries = [*originals, target, *transferred] + if any(not _name(item["name"]) or not _identity(item["device"]) or not _identity(item["inode"]) for item in entries): + raise PeerUnavailable("consumer resource identity is invalid") + if len({item["name"] for item in entries}) != len(entries) or len({(item["device"], item["inode"]) for item in entries}) != len(entries) or len({item["path"] for item in originals}) != len(originals): + raise PeerUnavailable("consumer resource aliases are conflicting") + if not any(item["kind"] == "transferred-ex" for item in transferred): + raise PeerUnavailable("consumer deployment/controller exclusive custody is missing") + return ConsumerConfiguration( + value["schema_version"], + value["request_id"], + value["authorization_id"], + value["target_id"], + value["issuer_uid"], + value["socket_directory_device"], + value["socket_directory_inode"], + tuple(OriginalFence(**item) for item in originals), + TargetOwner(**target), + tuple(TransferredResource(**item) for item in transferred), + ) + + +def _acquire_originals(configuration: ConsumerConfiguration) -> tuple[HeldFence, ...]: + fences: list[HeldFence] = [] + try: + for expected in configuration.original_fences: + path = Path(expected.path) + if path != path.resolve(strict=True): + raise PeerUnavailable("consumer original fence path is not canonical") + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC) + fences.append(HeldFence(expected.name, descriptor, "original-sh")) + actual = os.fstat(descriptor) + if (actual.st_dev, actual.st_ino) != (expected.device, expected.inode): + raise PeerUnavailable("consumer original fence object differs") + fcntl.flock(descriptor, fcntl.LOCK_SH | fcntl.LOCK_NB) + inspect_held_resource(fences[-1]) + return tuple(fences) + except BaseException: + for fence in fences: + os.close(fence.descriptor) + raise + + +def _target(configuration: ConsumerConfiguration, descriptor: int) -> HeldFence: + # Using the independent EX inspection also rejects an inherited target lock. + fence = HeldFence(configuration.target_owner.name, descriptor, "transferred-ex") + actual = inspect_held_resource(fence) + if (actual.device, actual.inode) != (configuration.target_owner.device, configuration.target_owner.inode): + raise PeerUnavailable("consumer target owner differs from the pinned configuration") + return fence + + +def _transfer(expected: TransferredResource, descriptor: int, issuer_pid: int, issuer_uid: int) -> ConsumerTransfer: + info = os.fstat(descriptor) + link = os.readlink(f"/proc/self/fd/{descriptor}") + if (info.st_dev, info.st_ino) != (expected.device, expected.inode) or os.get_inheritable(descriptor): + raise PeerUnavailable("received resource identity or close-on-exec flag differs") + if expected.kind == "network-namespace": + if link != f"net:[{info.st_ino}]": + raise PeerUnavailable("received resource is not the pinned network namespace") + lock = None + else: + if not stat.S_ISREG(info.st_mode) or stat.S_IMODE(info.st_mode) != 0o600 or info.st_nlink != 1 or info.st_uid != issuer_uid: + raise PeerUnavailable("received exclusive fence protection differs") + # A host outside the consumer PID namespace maps to PID 0 in fdinfo. + # SO_PEERPIDFD, obtained independently from this socket, pins that peer. + lock = inspect_kernel_lock(os.getpid(), descriptor, info.st_dev, info.st_ino, allow_unmapped_owner=issuer_pid == 0) + if lock is None or lock.mode != "WRITE" or lock.pid != issuer_pid: + raise PeerUnavailable("received exclusive fence is not held by the authenticated issuer") + return ConsumerTransfer(expected, descriptor, info.st_mode, info.st_uid, link, lock) + + +class RecoveryEpochConsumer: + """Retained prelaunch custody; the owner alone decides when it may close.""" + + def __init__(self, connection: socket.socket, configuration: ConsumerConfiguration, target_owner_fd: int, *, timeout: float = 30, configuration_file: Path | None = None) -> None: + self.connection = connection + self.configuration = configuration + self.grant: RecoveryGrant | None = None + self.challenge = secrets.token_hex(32) + self.receipt_nonce = "" + self._timeout = timeout + self._configuration_file = configuration_file + self._configuration_bytes: bytes | None = None + self._configuration_identity: tuple[int, int] | None = None + self._issuer_pidfd = -1 + self._issuer_pid = -1 + self._closed = False + self._originals: tuple[HeldFence, ...] = () + self._target: HeldFence | None = None + self._owned: list[int] = [] + self._initial: tuple[ResourceIdentity, ...] = () + self._transfers: tuple[ConsumerTransfer, ...] = () + try: + if type(timeout) not in (int, float) or not 0 < timeout <= 3600 or connection.family != socket.AF_UNIX or connection.type not in (socket.SOCK_STREAM, socket.SOCK_SEQPACKET): + raise PeerUnavailable("consumer requires a bounded authenticated local exchange") + if configuration_file is not None: + self._configuration_bytes, self._configuration_identity = read_original(configuration_file) + if parse_consumer_configuration(self._configuration_bytes) != configuration: + raise PeerUnavailable("consumer configuration changed before connection") + self._target = _target(configuration, target_owner_fd) + self._originals = _acquire_originals(configuration) + self._initial = tuple(inspect_held_resource(fence) for fence in (*self._originals, self._target)) + self._issuer_pid, uid, _ = struct.unpack("3i", connection.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, struct.calcsize("3i"))) + if uid != configuration.issuer_uid or self._issuer_pid < 0: + raise PeerUnavailable("connected issuer credentials differ") + self._issuer_pidfd = connection.getsockopt(socket.SOL_SOCKET, _SO_PEERPIDFD) + os.set_inheritable(self._issuer_pidfd, False) + if os.readlink(f"/proc/self/fd/{self._issuer_pidfd}") != "anon_inode:[pidfd]": + raise PeerUnavailable("connected issuer has no kernel peer lifetime pin") + self._check_issuer() + self._exchange() + except BaseException as error: + self.close() + if isinstance(error, (OSError, ValueError, TypeError, KeyError)) and not isinstance(error, PeerUnavailable): + raise PeerUnavailable("recovery consumer custody is unavailable") from error + raise + + def __enter__(self) -> RecoveryEpochConsumer: + return self + + def __exit__(self, *_: object) -> None: + self.close() + + @property + def issuer_pidfd(self) -> int: + return self._issuer_pidfd + + @property + def retained_descriptors(self) -> tuple[int, ...]: + return (*[fence.descriptor for fence in self._originals], *self._owned) + + def close(self) -> None: + """Release only when the owning lifecycle has established it is safe.""" + if not self._closed: + self._closed = True + self.connection.close() + for descriptor in self.retained_descriptors: + os.close(descriptor) + self._originals = () + self._owned.clear() + if self._issuer_pidfd >= 0: + os.close(self._issuer_pidfd) + self._issuer_pidfd = -1 + + def _check_issuer(self) -> None: + if self._closed or self._issuer_pidfd < 0: + raise PeerUnavailable("issuer custody handle is closed") + pid, uid, _ = struct.unpack("3i", self.connection.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, struct.calcsize("3i"))) + poll = select.poll() + poll.register(self._issuer_pidfd, select.POLLIN) + if (pid, uid) != (self._issuer_pid, self.configuration.issuer_uid) or poll.poll(0): + raise PeerUnavailable("the authenticated issuer lifetime ended") + channel = select.poll() + channel.register(self.connection, select.POLLERR | select.POLLHUP | select.POLLRDHUP) + if channel.poll(0): + raise PeerUnavailable("the authenticated issuer channel ended") + + def _binding(self) -> dict[str, object]: + return {"request_id": self.configuration.request_id, "authorization_id": self.configuration.authorization_id, "target_id": self.configuration.target_id, "challenge": self.challenge} + + def _exchange(self) -> None: + deadline = time.monotonic() + self._timeout + self.connection.settimeout(min(self._timeout, 60)) + self.connection.sendall(encode_peer_message({"schema": CHALLENGE_SCHEMA, **self._binding()})) + offer, descriptors = receive_peer_message(self.connection, timeout=self._timeout, allow_rights=True, deadline=deadline) + self._owned.extend(descriptors) + if set(offer) != {"schema", *self._binding(), "receipt_nonce", "resources"} or offer["schema"] != OFFER_SCHEMA or any(offer[name] != value for name, value in self._binding().items()) or not _token(offer["receipt_nonce"]): + raise PeerUnavailable("issuer custody offer is stale or bound to another target") + self.receipt_nonce = str(offer["receipt_nonce"]) + resources = offer["resources"] + expected: dict[str, tuple[str, int, int]] = {fence.name: ("original-sh", fence.device, fence.inode) for fence in self.configuration.original_fences} + owner = self.configuration.target_owner + expected[owner.name] = ("target-ex", owner.device, owner.inode) + expected.update({resource.name: (resource.kind, resource.device, resource.inode) for resource in self.configuration.transferred_resources}) + if not isinstance(resources, list) or len(resources) != len(expected): + raise PeerUnavailable("issuer omitted required custody resources") + seen: set[str] = set() + transferred: dict[str, int] = {} + for resource in resources: + if not isinstance(resource, dict) or set(resource) != {"name", "kind", "device", "inode", "transfer_index"} or not isinstance(resource["name"], str): + raise PeerUnavailable("issuer custody resource schema differs") + name = resource["name"] + if name in seen or name not in expected or (resource["kind"], resource["device"], resource["inode"]) != expected[name] or type(resource["device"]) is not int or type(resource["inode"]) is not int: + raise PeerUnavailable("issuer custody resource differs from the pinned configuration") + seen.add(name) + index = resource["transfer_index"] + if resource["kind"] in ("transferred-ex", "network-namespace"): + if type(index) is not int or not 0 <= index < len(descriptors) or index in transferred.values(): + raise PeerUnavailable("issuer transfer descriptor mapping is incomplete or repeated") + transferred[name] = index + elif index is not None: + raise PeerUnavailable("original shared or target exclusive custody must be independently acquired") + if seen != set(expected) or set(transferred.values()) != set(range(len(descriptors))) or len(descriptors) != len(self.configuration.transferred_resources): + raise PeerUnavailable("issuer delivered an incomplete or unexpected descriptor set") + self._transfers = tuple(_transfer(resource, descriptors[transferred[resource.name]], self._issuer_pid, self.configuration.issuer_uid) for resource in self.configuration.transferred_resources) + self.check_current() + assert self._target is not None + received = [{"name": fence.name, "fd": fence.descriptor} for fence in (*self._originals, self._target)] + received.extend({"name": item.expected.name, "fd": item.descriptor} for item in self._transfers) + remaining = deadline - time.monotonic() + if remaining <= 0: + raise PeerUnavailable("consumer verification exceeded its bounded deadline") + self.connection.settimeout(min(remaining, 60)) + self.connection.sendall(encode_peer_message({"schema": RECEIPT_SCHEMA, **self._binding(), "receipt_nonce": self.receipt_nonce, "descriptors": received})) + grant, _ = receive_peer_message(self.connection, timeout=self._timeout, deadline=deadline) + expected_fields = {"schema", "granted", *RecoveryGrant.__dataclass_fields__} + if set(grant) != expected_fields or grant["schema"] != GRANT_SCHEMA or grant["granted"] != "recovery-epoch-consumed" or any(grant[name] != value for name, value in self._binding().items()) or grant["receipt_nonce"] != self.receipt_nonce: + raise PeerUnavailable("issuer grant is not the exact current consumed challenge") + if not all(_token(grant[name]) for name in ("inventory_sha256", "binding_sha256", "consumed_sha256", "receipt_sha256")): + raise PeerUnavailable("issuer grant durable identities are incomplete") + boot, pid, started = grant["host_boot_id"], grant["host_pid"], grant["host_start_token"] + if not isinstance(boot, str) or str(uuid.UUID(boot)) != boot or boot != Path("/proc/sys/kernel/random/boot_id").read_text(encoding="ascii").strip() or not _identity(pid) or not isinstance(started, str) or re.fullmatch(r"[1-9][0-9]*", started) is None: + raise PeerUnavailable("issuer grant original lifetime is malformed or from another boot") + if self._issuer_pid > 0: + fields = Path(f"/proc/{self._issuer_pid}/stat").read_text(encoding="ascii").rsplit(")", 1)[1].split() + if pid != self._issuer_pid or len(fields) < 20 or fields[19] != started or fields[0] in ("Z", "X", "x"): + raise PeerUnavailable("issuer grant differs from its currently visible lifetime") + self.check_current() + self.grant = RecoveryGrant( + self.configuration.request_id, + self.configuration.authorization_id, + self.configuration.target_id, + self.challenge, + self.receipt_nonce, + cast(str, grant["inventory_sha256"]), + cast(str, grant["binding_sha256"]), + cast(str, grant["consumed_sha256"]), + cast(str, grant["receipt_sha256"]), + boot, + cast(int, pid), + started, + ) + + def check_current(self) -> None: + """Refuse stale readiness without releasing any already retained custody.""" + try: + self._check_issuer() + assert self._target is not None + if tuple(inspect_held_resource(fence) for fence in (*self._originals, self._target)) != self._initial: + raise PeerUnavailable("consumer independent fence custody changed") + if tuple(_transfer(item.expected, item.descriptor, self._issuer_pid, self.configuration.issuer_uid) for item in self._transfers) != self._transfers: + raise PeerUnavailable("consumer transferred custody changed") + if self._configuration_file is not None: + current, identity = read_original(self._configuration_file) + if current != self._configuration_bytes or identity != self._configuration_identity: + raise PeerUnavailable("consumer configuration changed while holding custody") + self._check_issuer() + except (OSError, ValueError, TypeError, KeyError) as error: + if isinstance(error, PeerUnavailable): + raise + raise PeerUnavailable("current consumer custody could not be verified") from error + + +def _validate_socket_directory(configuration: ConsumerConfiguration) -> None: + if ISSUER_DIRECTORY != ISSUER_DIRECTORY.resolve(strict=True): + raise PeerUnavailable("recovery issuer directory is not canonical") + directory = os.open(ISSUER_DIRECTORY, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC) + try: + info = os.fstat(directory) + if (info.st_dev, info.st_ino) != (configuration.socket_directory_device, configuration.socket_directory_inode) or info.st_uid != configuration.issuer_uid or stat.S_IMODE(info.st_mode) != 0o700: + raise PeerUnavailable("recovery issuer directory differs from the target configuration") + entry = ISSUER_SOCKET.lstat() + if not stat.S_ISSOCK(entry.st_mode) or entry.st_uid != configuration.issuer_uid or stat.S_IMODE(entry.st_mode) != 0o600: + raise PeerUnavailable("recovery issuer socket protection differs") + current = ISSUER_DIRECTORY.lstat() + if (current.st_dev, current.st_ino, current.st_uid, current.st_mode) != (info.st_dev, info.st_ino, info.st_uid, info.st_mode): + raise PeerUnavailable("recovery issuer directory changed during inspection") + finally: + os.close(directory) + + +def connect_recovery_epoch(config_path: Path, target_owner_fd: int, *, timeout: float = 30) -> RecoveryEpochConsumer: + """Connect only the fixed production origin; no request or environment root.""" + if config_path != CONSUMER_CONFIG: + raise PeerUnavailable("recovery consumer configuration must use its fixed bootstrap path") + if type(timeout) not in (int, float) or not 0 < timeout <= 3600: + raise PeerUnavailable("recovery consumer verification deadline must be positive and at most one hour") + encoded, _ = read_original(config_path) + configuration = parse_consumer_configuration(encoded) + _validate_socket_directory(configuration) + connection = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + try: + connection.settimeout(min(timeout, 60)) + connection.connect(str(ISSUER_SOCKET)) + _validate_socket_directory(configuration) + return RecoveryEpochConsumer(connection, configuration, target_owner_fd, timeout=timeout, configuration_file=config_path) + except BaseException: + connection.close() + raise diff --git a/src/auto_coder/management/recovery_epoch_peer.py b/src/auto_coder/management/recovery_epoch_peer.py new file mode 100644 index 000000000..f2e6fd665 --- /dev/null +++ b/src/auto_coder/management/recovery_epoch_peer.py @@ -0,0 +1,415 @@ +"""Authenticated recovery peer custody evidence, without grant authority. + +The issuer supplies the expected Docker-bound process and the complete fence +set. This transport neither decides that scope nor consumes or issues an epoch. +Caller-owned fences stay open; the session owns only its socket and PID pin. +""" + +from __future__ import annotations + +import array +import json +import os +import re +import secrets +import select +import socket +import stat +import struct +import time +import uuid +from dataclasses import dataclass +from pathlib import Path +from typing import Literal + +from .archived_exclusions import unique_pairs +from .fresh_host_observation import _assert_pid_flock + +MAX_MESSAGE_BYTES = 64 * 1024 +MAX_RESOURCES = 64 +CHALLENGE_SCHEMA = "management-recovery-peer-challenge-v1" +OFFER_SCHEMA = "management-recovery-custody-offer-v1" +RECEIPT_SCHEMA = "management-recovery-custody-receipt-v1" +FenceKind = Literal["original-sh", "target-ex", "transferred-ex"] + + +class PeerUnavailable(ValueError): + """Peer custody is unavailable; this result never permits a grant.""" + + +@dataclass(frozen=True) +class ExpectedPeer: + pid: int + start_token: str + boot_id: str + uid: int + request_id: str + authorization_id: str + target_id: str + + +@dataclass(frozen=True) +class HeldFence: + name: str + descriptor: int + kind: FenceKind + + +@dataclass(frozen=True) +class HeldResource: + name: str + descriptor: int + kind: Literal["network-namespace"] = "network-namespace" + + +@dataclass(frozen=True) +class KernelLock: + mode: str + pid: int + device_major: int + device_minor: int + inode: int + + +@dataclass(frozen=True) +class ResourceIdentity: + name: str + kind: str + path: str + device: int + inode: int + mode: int + uid: int + lock: KernelLock | None + + +@dataclass(frozen=True) +class ReceivedResource: + identity: ResourceIdentity + peer_descriptor: int + peer_lock: KernelLock | None + + +@dataclass(frozen=True) +class PeerCustody: + peer: ExpectedPeer + challenge: str + receipt_nonce: str + resources: tuple[ReceivedResource, ...] + observed_monotonic_ns: int + + +def _token(value: object) -> bool: + return isinstance(value, str) and re.fullmatch(r"[0-9a-f]{64}", value) is not None + + +def _name(value: object) -> bool: + return isinstance(value, str) and re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}", value) is not None + + +def _fd_path(pid: int, descriptor: int, *, info: bool = False) -> Path: + if type(pid) is not int or pid <= 0 or type(descriptor) is not int or not 0 <= descriptor < 1_048_576: + raise PeerUnavailable("peer descriptor or process identity is invalid") + return Path("/proc") / str(pid) / ("fdinfo" if info else "fd") / str(descriptor) + + +def inspect_kernel_lock(pid: int, descriptor: int, device: int, inode: int, *, allow_unmapped_owner: bool = False) -> KernelLock | None: + """Read the exact FD's normalized kernel flock record without changing it. + + The optional PID-zero allowance is only for an authenticated host outside + a consumer's PID namespace; independent holder checks must use the default. + """ + path = _fd_path(pid, descriptor, info=True) + with path.open("rb") as stream: + data = stream.read(MAX_MESSAGE_BYTES + 1) + if len(data) > MAX_MESSAGE_BYTES: + raise PeerUnavailable("kernel descriptor observation exceeds its bound") + matches: list[KernelLock] = [] + for line in data.decode("ascii").splitlines(): + if not line.startswith("lock:"): + continue + fields = line.removeprefix("lock:").split() + if len(fields) != 8 or fields[1:3] != ["FLOCK", "ADVISORY"] or fields[3] not in ("READ", "WRITE") or fields[6:] != ["0", "EOF"]: + raise PeerUnavailable("descriptor kernel lock profile differs") + parts = fields[5].split(":") + if len(parts) != 3 or not fields[4].isdecimal() or int(fields[4]) < (0 if allow_unmapped_owner else 1): + raise PeerUnavailable("descriptor kernel lock identity is malformed") + observed = KernelLock(fields[3], int(fields[4]), int(parts[0], 16), int(parts[1], 16), int(parts[2])) + if (observed.device_major, observed.device_minor, observed.inode) != (os.major(device), os.minor(device), inode): + raise PeerUnavailable("descriptor kernel lock names another file") + matches.append(observed) + if len(matches) > 1: + raise PeerUnavailable("descriptor kernel lock is ambiguous") + return matches[0] if matches else None + + +def inspect_held_resource(resource: HeldFence | HeldResource) -> ResourceIdentity: + """Inspect a caller-owned descriptor without acquiring or releasing it.""" + if not _name(resource.name): + raise PeerUnavailable("resource name is invalid") + descriptor_path = _fd_path(os.getpid(), resource.descriptor) + info = os.fstat(resource.descriptor) + path = os.readlink(descriptor_path) + if isinstance(resource, HeldResource): + if resource.kind != "network-namespace" or re.fullmatch(r"net:\[[0-9]+\]", path) is None or path != f"net:[{info.st_ino}]": + raise PeerUnavailable("held resource is not the exact network namespace") + return ResourceIdentity(resource.name, resource.kind, path, info.st_dev, info.st_ino, info.st_mode, info.st_uid, None) + if resource.kind not in ("original-sh", "target-ex", "transferred-ex"): + raise PeerUnavailable("held fence kind is unsupported") + original = Path(path) + if not original.is_absolute() or original != original.resolve(strict=True): + raise PeerUnavailable("held fence path is not canonical") + current = original.lstat() + if not stat.S_ISREG(info.st_mode) or stat.S_IMODE(info.st_mode) != 0o600 or info.st_nlink != 1 or info.st_uid != os.geteuid() or (current.st_dev, current.st_ino, current.st_mode, current.st_uid, current.st_nlink) != (info.st_dev, info.st_ino, info.st_mode, info.st_uid, info.st_nlink): + raise PeerUnavailable("held fence identity or protection differs") + lock = inspect_kernel_lock(os.getpid(), resource.descriptor, info.st_dev, info.st_ino) + if resource.kind != "target-ex": + mode = "READ" if resource.kind == "original-sh" else "WRITE" + if lock is None or lock.mode != mode or lock.pid != os.getpid(): + raise PeerUnavailable("issuer does not independently hold the required fence") + _assert_pid_flock(os.getpid(), resource.descriptor, mode) + return ResourceIdentity(resource.name, resource.kind, path, info.st_dev, info.st_ino, info.st_mode, info.st_uid, lock) + + +def _received(identity: ResourceIdentity, peer: ExpectedPeer, descriptor: int) -> ReceivedResource: + path = _fd_path(peer.pid, descriptor) + before = path.stat() + # /proc/PID/fd intentionally follows the authenticated process's descriptor. + # NONBLOCK prevents a concurrently replaced FIFO from blocking inspection. + reopened = os.open(path, os.O_RDONLY | os.O_CLOEXEC | os.O_NONBLOCK) + try: + actual = os.fstat(reopened) + after = path.stat() + expected = (identity.device, identity.inode, identity.mode, identity.uid) + for info in (before, actual, after): + if (info.st_dev, info.st_ino, info.st_mode, info.st_uid) != expected: + raise PeerUnavailable("received descriptor names another resource") + if identity.kind == "network-namespace": + if os.readlink(path) != identity.path: + raise PeerUnavailable("received network namespace object differs") + observed = None + else: + observed = inspect_kernel_lock(peer.pid, descriptor, actual.st_dev, actual.st_ino) + if identity.kind == "transferred-ex": + if observed is None or observed != identity.lock or observed.mode != "WRITE": + raise PeerUnavailable("consumer did not receive the held exclusive open file description") + else: + mode = "READ" if identity.kind == "original-sh" else "WRITE" + if observed is None or observed.pid != peer.pid or observed.mode != mode: + raise PeerUnavailable("consumer did not independently acquire the required fence") + _assert_pid_flock(peer.pid, reopened, mode) + final = path.stat() + if (final.st_dev, final.st_ino, final.st_mode, final.st_uid) != expected: + raise PeerUnavailable("consumer descriptor changed during custody observation") + return ReceivedResource(identity, descriptor, observed) + finally: + os.close(reopened) + + +def _reject_constant(value: str) -> object: + raise PeerUnavailable("non-finite peer JSON value: " + value) + + +def encode_peer_message(value: dict[str, object]) -> bytes: + """Encode one bounded protocol object; this does not authorize its content.""" + packet = json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False).encode() + b"\n" + if len(packet) > MAX_MESSAGE_BYTES: + raise PeerUnavailable("peer message exceeds its bound") + return packet + + +def receive_peer_message(connection: socket.socket, *, timeout: float, allow_rights: bool = False, deadline: float | None = None) -> tuple[dict[str, object], tuple[int, ...]]: + """Read one message, closing all kernel-delivered FDs on any refusal.""" + deadline = time.monotonic() + timeout if deadline is None else deadline + content = bytearray() + received: list[int] = [] + try: + while b"\n" not in content: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise PeerUnavailable("peer exchange exceeded its bounded deadline") + connection.settimeout(remaining) + data, ancillary, flags, _ = connection.recvmsg(MAX_MESSAGE_BYTES + 1, socket.CMSG_SPACE(MAX_RESOURCES * array.array("i").itemsize), socket.MSG_CMSG_CLOEXEC) + for level, kind, payload in ancillary: + if level == socket.SOL_SOCKET and kind == socket.SCM_RIGHTS: + values = array.array("i") + values.frombytes(payload[: len(payload) - len(payload) % values.itemsize]) + received.extend(values) + if (ancillary and not allow_rights) or any((level, kind) != (socket.SOL_SOCKET, socket.SCM_RIGHTS) for level, kind, _ in ancillary) or flags & (socket.MSG_TRUNC | socket.MSG_CTRUNC): + raise PeerUnavailable("peer unexpectedly supplied descriptors or a truncated message") + if len(received) > MAX_RESOURCES: + raise PeerUnavailable("peer supplied too many descriptors") + if not data: + raise PeerUnavailable("peer closed before complete custody evidence") + content.extend(data) + if len(content) > MAX_MESSAGE_BYTES: + raise PeerUnavailable("peer message exceeds its bound") + line, remainder = bytes(content).split(b"\n", 1) + if remainder: + raise PeerUnavailable("peer supplied an unsolicited additional message") + value = json.loads(line, object_pairs_hook=unique_pairs, parse_constant=_reject_constant) + if not isinstance(value, dict): + raise PeerUnavailable("peer message is not an object") + return value, tuple(received) + except BaseException: + for descriptor in received: + os.close(descriptor) + raise + + +class PeerSession: + """One challenge and independently observed custody, never a launch grant.""" + + def __init__(self, connection: socket.socket, expected: ExpectedPeer, *, timeout: float = 10) -> None: + self.connection = connection + self.expected = expected + self._pidfd = -1 + self._closed = False + self._offered = False + self._held: tuple[HeldFence | HeldResource, ...] = () + self._identities: tuple[ResourceIdentity, ...] = () + self._custody: PeerCustody | None = None + self._timeout = timeout + self.challenge = "" + self.receipt_nonce = secrets.token_hex(32) + try: + if connection.family != socket.AF_UNIX or connection.type not in (socket.SOCK_STREAM, socket.SOCK_SEQPACKET): + raise PeerUnavailable("recovery custody requires an authenticated local socket") + if type(timeout) not in (int, float) or not 0 < timeout <= 60: + raise PeerUnavailable("peer timeout must be positive and at most sixty seconds") + if type(expected.pid) is not int or expected.pid <= 0 or type(expected.uid) is not int or expected.uid < 0: + raise PeerUnavailable("expected peer identity is invalid") + if not expected.start_token.isascii() or not expected.start_token.isdecimal() or str(int(expected.start_token)) != expected.start_token or int(expected.start_token) <= 0: + raise PeerUnavailable("expected peer lifetime is invalid") + if str(uuid.UUID(expected.boot_id)) != expected.boot_id or not all(_name(value) for value in (expected.request_id, expected.authorization_id, expected.target_id)): + raise PeerUnavailable("expected peer binding is invalid") + connection.settimeout(timeout) + self._pidfd = os.pidfd_open(expected.pid, 0) + self._check_lifetime() + message = self._read_message() + if set(message) != {"schema", "request_id", "authorization_id", "target_id", "challenge"} or message.get("schema") != CHALLENGE_SCHEMA: + raise PeerUnavailable("peer challenge schema differs") + if any(message[name] != getattr(expected, name) for name in ("request_id", "authorization_id", "target_id")) or not _token(message["challenge"]): + raise PeerUnavailable("peer challenge binding differs") + self.challenge = str(message["challenge"]) + self._check_lifetime() + except BaseException: + self.close() + raise + + def __enter__(self) -> PeerSession: + return self + + def __exit__(self, *_: object) -> None: + self.close() + + def close(self) -> None: + if not self._closed: + self._closed = True + self.connection.close() + if self._pidfd >= 0: + os.close(self._pidfd) + self._pidfd = -1 + + def _check_lifetime(self) -> None: + if self._closed or self._pidfd < 0: + raise PeerUnavailable("peer session is closed") + pid, uid, _ = struct.unpack("3i", self.connection.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, struct.calcsize("3i"))) + if (pid, uid) != (self.expected.pid, self.expected.uid): + raise PeerUnavailable("local peer credentials differ from the expected target") + poll = select.poll() + poll.register(self._pidfd, select.POLLIN) + prefix, separator, tail = (Path("/proc") / str(pid) / "stat").read_text(encoding="ascii").rpartition(")") + fields = tail.split() + if not separator or prefix.partition(" (")[0] != str(pid) or len(fields) < 20 or fields[0] in ("Z", "X", "x") or fields[19] != self.expected.start_token or Path("/proc/sys/kernel/random/boot_id").read_text(encoding="ascii").strip() != self.expected.boot_id or poll.poll(0): + raise PeerUnavailable("exact target peer lifetime exited or changed") + + def _set_remaining_timeout(self, deadline: float) -> None: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise PeerUnavailable("peer exchange exceeded its bounded deadline") + self.connection.settimeout(remaining) + + def _read_message(self, *, deadline: float | None = None) -> dict[str, object]: + value, _ = receive_peer_message(self.connection, timeout=self._timeout, deadline=deadline) + return value + + def offer_and_verify_custody(self, fences: tuple[HeldFence, ...], resources: tuple[HeldResource, ...] = ()) -> PeerCustody: + """Transfer declared host resources, then require physical receipt proof.""" + try: + if self._offered: + raise PeerUnavailable("this peer session already offered custody") + self._offered = True + self._check_lifetime() + held: tuple[HeldFence | HeldResource, ...] = (*fences, *resources) + if not 3 <= len(held) <= MAX_RESOURCES or not any(fence.kind == "original-sh" for fence in fences) or sum(fence.kind == "target-ex" for fence in fences) != 1 or not any(fence.kind == "transferred-ex" for fence in fences): + raise PeerUnavailable("complete original, target and deployment fence classes are required") + identities = tuple(inspect_held_resource(resource) for resource in held) + if len({item.name for item in identities}) != len(identities) or len({(item.device, item.inode) for item in identities}) != len(identities): + raise PeerUnavailable("resource names or kernel objects are duplicated") + transfer = array.array("i") + offered: list[dict[str, object]] = [] + for resource, identity in zip(held, identities): + transfer_index = None + if identity.kind in ("transferred-ex", "network-namespace"): + transfer_index = len(transfer) + transfer.append(resource.descriptor) + offered.append({"name": identity.name, "kind": identity.kind, "device": identity.device, "inode": identity.inode, "transfer_index": transfer_index}) + binding = {name: getattr(self.expected, name) for name in ("request_id", "authorization_id", "target_id")} + binding.update(challenge=self.challenge, receipt_nonce=self.receipt_nonce) + packet = encode_peer_message({"schema": OFFER_SCHEMA, **binding, "resources": offered}) + deadline = time.monotonic() + self._timeout + self._set_remaining_timeout(deadline) + sent = self.connection.sendmsg([packet], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, transfer)]) + if sent <= 0: + raise PeerUnavailable("custody offer was not sent") + if sent < len(packet): + self._set_remaining_timeout(deadline) + self.connection.sendall(packet[sent:]) + receipt = self._read_message(deadline=deadline) + if set(receipt) != {"schema", *binding, "descriptors"} or receipt.get("schema") != RECEIPT_SCHEMA or any(receipt[name] != value for name, value in binding.items()): + raise PeerUnavailable("custody receipt is stale or bound to another challenge") + descriptors = receipt["descriptors"] + if not isinstance(descriptors, list) or len(descriptors) != len(held): + raise PeerUnavailable("custody receipt omits required descriptors") + peer_descriptors: dict[str, int] = {} + for item in descriptors: + if not isinstance(item, dict) or set(item) != {"name", "fd"} or not _name(item.get("name")) or type(item.get("fd")) is not int: + raise PeerUnavailable("custody receipt descriptor is malformed") + name, descriptor = str(item["name"]), int(item["fd"]) + _fd_path(self.expected.pid, descriptor) + if name in peer_descriptors or descriptor in peer_descriptors.values(): + raise PeerUnavailable("custody receipt duplicates a descriptor") + peer_descriptors[name] = descriptor + if set(peer_descriptors) != {item.name for item in identities}: + raise PeerUnavailable("custody receipt names another resource set") + self._held, self._identities = held, identities + self._check_lifetime() + observed = tuple(_received(identity, self.expected, peer_descriptors[identity.name]) for identity in identities) + result = PeerCustody(self.expected, self.challenge, self.receipt_nonce, observed, time.monotonic_ns()) + self._custody = result + self.check(result) + return result + except BaseException as error: + self.close() + if isinstance(error, (OSError, ValueError, TypeError, KeyError)) and not isinstance(error, PeerUnavailable): + raise PeerUnavailable("current peer custody could not be verified") from error + raise + + def check(self, custody: PeerCustody) -> None: + """Repeat actual custody checks; the caller still owns grant ordering.""" + try: + if self._custody is None or custody != self._custody: + raise PeerUnavailable("custody belongs to another session or was not verified") + self._check_lifetime() + if tuple(inspect_held_resource(resource) for resource in self._held) != self._identities: + raise PeerUnavailable("issuer resource identity or lock changed after receipt") + observed = tuple(_received(item.identity, self.expected, item.peer_descriptor) for item in self._custody.resources) + if observed != self._custody.resources: + raise PeerUnavailable("consumer resource custody changed after receipt") + self._check_lifetime() + except BaseException as error: + self.close() + if isinstance(error, (OSError, ValueError, TypeError, KeyError)) and not isinstance(error, PeerUnavailable): + raise PeerUnavailable("current peer custody could not be verified") from error + raise diff --git a/src/auto_coder/management/recovery_epoch_resources.py b/src/auto_coder/management/recovery_epoch_resources.py new file mode 100644 index 000000000..78b5d83df --- /dev/null +++ b/src/auto_coder/management/recovery_epoch_resources.py @@ -0,0 +1,132 @@ +"""Acquire and retain the actual fences used by one recovery issuer lifetime.""" + +from __future__ import annotations + +import fcntl +import os +import select +import stat +from contextlib import ExitStack, contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Iterator + +from .containment_enrollment import OriginalNetworkPin, _approved_directory +from .original_attempt_evidence import observe as observe_original_history +from .recovery_containment_model import NamedDescriptor, RecoveryEnrollment, RecoveryLock, RecoveryResources +from .recovery_epoch_authorization import EpochAuthorization +from .recovery_epoch_peer import HeldFence, HeldResource, inspect_held_resource +from .recovery_epoch_state import EpochUnavailable, RecoveryEpochState, issuer_lifetime + + +@dataclass(frozen=True) +class EpochResources: + history_descriptor: int + containment: RecoveryResources + fences: tuple[HeldFence, ...] + namespaces: tuple[HeldResource, ...] + + @property + def original_descriptors(self) -> tuple[int, ...]: + return tuple(item.descriptor for item in self.containment.original_owner_fds) + + def check(self) -> None: + if (self.containment.holder_boot_id, self.containment.holder_pid, self.containment.holder_start_token) != issuer_lifetime(): + raise EpochUnavailable("recovery resources escaped their original issuer lifetime") + held: tuple[HeldFence | HeldResource, ...] = (*self.fences, *self.namespaces) + for resource in held: + inspect_held_resource(resource) + + +def _open_bound_lock(selected: RecoveryLock, *, mode: int | None) -> int: + path = Path(selected.path) + _approved_directory(path.parent, uid=os.geteuid()) + if path != path.resolve(strict=True): + raise EpochUnavailable("recovery owner/controller path is not canonical") + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC | os.O_NONBLOCK) + try: + info, named = os.fstat(descriptor), path.lstat() + expected = (selected.device, selected.inode) + if (info.st_dev, info.st_ino) != expected or (named.st_dev, named.st_ino) != expected or info.st_mode != named.st_mode or not stat.S_ISREG(info.st_mode) or stat.S_IMODE(info.st_mode) != 0o600 or info.st_nlink != 1 or info.st_uid != os.geteuid(): + raise EpochUnavailable("recovery original fence identity or protection differs") + if mode is not None: + fcntl.flock(descriptor, mode | fcntl.LOCK_NB) + return descriptor + except BaseException: + os.close(descriptor) + raise + + +def _pin_current_namespace(selected: OriginalNetworkPin) -> int: + """Duplicate a currently held nsfs object after pinning its exact holder.""" + boot = issuer_lifetime()[0] + if selected.holder_boot_id != boot: + raise EpochUnavailable("original namespace holder belongs to another boot") + pidfd = os.pidfd_open(selected.holder_pid, 0) + descriptor = -1 + try: + poll = select.poll() + poll.register(pidfd, select.POLLIN) + holder = Path("/proc") / str(selected.holder_pid) + + def check_holder() -> None: + fields = (holder / "stat").read_text(encoding="ascii").rsplit(")", 1)[1].split() + if len(fields) < 20 or fields[0] in ("Z", "X", "x") or fields[19] != selected.holder_start_token or poll.poll(0): + raise EpochUnavailable("original namespace holder exited or changed") + + check_holder() + path = holder / "fd" / str(selected.holder_fd) + before = path.stat() + if os.readlink(path) != selected.namespace or selected.namespace != f"net:[{selected.inode}]" or (before.st_dev, before.st_ino) != (selected.device, selected.inode): + raise EpochUnavailable("original namespace pin no longer names its observed object") + # Following the authenticated process's descriptor is intentional; + # ordinary pathname symlinks never select an original namespace. + descriptor = os.open(path, os.O_RDONLY | os.O_CLOEXEC | os.O_NONBLOCK) + actual, after = os.fstat(descriptor), path.stat() + if any((item.st_dev, item.st_ino, item.st_mode) != (before.st_dev, before.st_ino, before.st_mode) for item in (actual, after)) or os.readlink(path) != selected.namespace: + raise EpochUnavailable("original namespace object changed during acquisition") + check_holder() + inspect_held_resource(HeldResource("acquired-namespace", descriptor)) + return descriptor + except BaseException: + if descriptor >= 0: + os.close(descriptor) + raise + finally: + os.close(pidfd) + + +@contextmanager +def hold_epoch_resources(authorization: EpochAuthorization, enrollment: RecoveryEnrollment, state: RecoveryEpochState) -> Iterator[EpochResources]: + """Acquire independent original SH and real controller EX; create nothing.""" + with ExitStack() as held: + history = held.enter_context(observe_original_history(authorization.history)) + fences = [HeldFence("history", history, "original-sh")] + owners: list[NamedDescriptor] = [] + controllers: list[NamedDescriptor] = [] + namespaces: list[NamedDescriptor] = [] + transfers: list[HeldResource] = [] + for index, original in enumerate(enrollment.originals): + if original.owner is not None: + owner = _open_bound_lock(original.owner, mode=fcntl.LOCK_SH) + held.callback(os.close, owner) + owners.append(NamedDescriptor(original.generation_id, owner)) + fences.append(HeldFence(f"owner-{index}", owner, "original-sh")) + if original.namespace_pin is not None: + namespace = _pin_current_namespace(original.namespace_pin) + held.callback(os.close, namespace) + namespaces.append(NamedDescriptor(original.generation_id, namespace)) + transfers.append(HeldResource(f"namespace-{index}", namespace)) + for index, watcher in enumerate(enrollment.watchers): + controller = _open_bound_lock(watcher.lock, mode=fcntl.LOCK_EX) + held.callback(os.close, controller) + controllers.append(NamedDescriptor(watcher.unit, controller)) + fences.append(HeldFence(f"controller-{index}", controller, "transferred-ex")) + target = _open_bound_lock(enrollment.target.owner, mode=None) + held.callback(os.close, target) + fences.extend((HeldFence("target-owner", target, "target-ex"), HeldFence("epoch", state.descriptor, "transferred-ex"))) + boot, pid, start = issuer_lifetime() + result = EpochResources(history, RecoveryResources(pid, start, boot, tuple(owners), tuple(controllers), tuple(namespaces)), tuple(fences), tuple(transfers)) + result.check() + yield result + result.check() diff --git a/src/auto_coder/management/recovery_epoch_state.py b/src/auto_coder/management/recovery_epoch_state.py new file mode 100644 index 000000000..f1adcc146 --- /dev/null +++ b/src/auto_coder/management/recovery_epoch_state.py @@ -0,0 +1,336 @@ +"""Deployment-fixed, single-use recovery-epoch persistence. + +These receipts record only durable state. They are not a grant, containment +proof, historical completeness verdict, or permission to launch. The issuer +must independently verify those facts before consuming this state. +""" + +from __future__ import annotations + +import fcntl +import hashlib +import json +import os +import re +import uuid +from contextlib import contextmanager +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Iterator + +from ..logger_config import get_logger +from .containment_enrollment import _approved_directory +from .fresh_host_observation import _assert_pid_flock +from .fresh_preflight import ObservedIdentity, _file, _identity, _open +from .original_attempts import canonical, identity + +EPOCH_ROOT = Path("/var/lib/auto-coder/recovery-epoch") +logger = get_logger(__name__) +_PHASES = ("reserved", "intent", "consumed", "receipt") +_IDENTIFIER = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}\Z", re.ASCII) +_SHA256 = re.compile(r"[0-9a-f]{64}\Z", re.ASCII) +_MAX_BYTES = 1024 * 1024 + + +class EpochUnavailable(ValueError): + """An absent, conflicting or uncertain state is never repaired or replayed.""" + + +@dataclass(frozen=True) +class EpochInventory: + schema_version: int + root: str + root_device: int + root_inode: int + lock_device: int + lock_inode: int + deployment_id: str + authorization_id: str + enrollment_ids: tuple[str, ...] + request_id: str + preparation_id: str + target_id: str + source_sha256: str + config_sha256: str + authorization_sha256: str = "" + + +@dataclass(frozen=True) +class EpochBinding: + deployment_id: str + authorization_id: str + enrollment_ids: tuple[str, ...] + request_id: str + preparation_id: str + target_id: str + source_sha256: str + config_sha256: str + evidence_sha256: str + host_boot_id: str + host_pid: int + host_start_token: str + authorization_sha256: str = "" + evidence_json: str = "" + + +@dataclass(frozen=True) +class EpochReceipt: + inventory_sha256: str + binding_sha256: str + consumed_sha256: str + receipt_sha256: str + # This state machine never observes dispatch or grants launch authority. + grant_delivered: bool = False + + +def issuer_lifetime() -> tuple[str, int, str]: + """Read the actual current issuer's boot, PID and Linux start token.""" + boot = Path("/proc/sys/kernel/random/boot_id").read_text(encoding="ascii").strip() + fields = Path("/proc/self/stat").read_text(encoding="ascii").rsplit(")", 1)[1].split() + if str(uuid.UUID(boot)) != boot or len(fields) < 20 or not fields[19].isascii() or not fields[19].isdecimal() or int(fields[19]) <= 0 or str(int(fields[19])) != fields[19]: + raise EpochUnavailable("recovery issuer lifetime is unavailable") + return boot, os.getpid(), fields[19] + + +def _digest(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _read(path: Path) -> tuple[ObservedIdentity, bytes]: + observed, encoded = _file(path, private=True, limit=_MAX_BYTES) + if identity(path) != (observed.device, observed.inode): + raise EpochUnavailable("recovery epoch file was replaced") + return observed, encoded + + +def _validate_static(value: EpochInventory | EpochBinding) -> None: + for field in ("deployment_id", "authorization_id", "request_id", "preparation_id"): + token = getattr(value, field) + if not isinstance(token, str) or not _IDENTIFIER.fullmatch(token): + raise EpochUnavailable("invalid recovery epoch " + field) + if not isinstance(value.enrollment_ids, tuple) or not value.enrollment_ids or len(value.enrollment_ids) > 256 or any(not isinstance(token, str) or not _IDENTIFIER.fullmatch(token) for token in value.enrollment_ids): + raise EpochUnavailable("invalid recovery epoch enrollments") + if tuple(sorted(set(value.enrollment_ids))) != value.enrollment_ids: + raise EpochUnavailable("recovery epoch enrollments are aliased or unordered") + for field in ("target_id", "source_sha256", "config_sha256"): + digest = getattr(value, field) + if not isinstance(digest, str) or not _SHA256.fullmatch(digest): + raise EpochUnavailable("invalid recovery epoch " + field) + if not isinstance(value.authorization_sha256, str) or (value.authorization_sha256 and not _SHA256.fullmatch(value.authorization_sha256)): + raise EpochUnavailable("invalid recovery epoch authorization digest") + + +def _parse_inventory(encoded: bytes, root: Path, root_id: tuple[int, int], lock_id: tuple[int, int]) -> EpochInventory: + raw = json.loads(encoded) + if not isinstance(raw, dict) or set(raw) != set(EpochInventory.__dataclass_fields__) or canonical(raw) != encoded or not isinstance(raw.get("enrollment_ids"), list): + raise EpochUnavailable("recovery epoch inventory is noncanonical or incomplete") + raw["enrollment_ids"] = tuple(raw["enrollment_ids"]) + inventory = EpochInventory(**raw) + _validate_static(inventory) + if type(inventory.schema_version) is not int or inventory.schema_version not in (1, 2): + raise EpochUnavailable("unsupported recovery epoch inventory schema") + if inventory.schema_version == 2 and not inventory.authorization_sha256: + raise EpochUnavailable("issuer inventory requires its complete authorization digest") + if any(type(getattr(inventory, field)) is not int or getattr(inventory, field) < 0 for field in ("root_device", "root_inode", "lock_device", "lock_inode")): + raise EpochUnavailable("invalid recovery epoch inventory identity") + if inventory.root != str(root) or (inventory.root_device, inventory.root_inode) != root_id or (inventory.lock_device, inventory.lock_inode) != lock_id: + raise EpochUnavailable("recovery epoch inventory root or lock differs") + return inventory + + +class RecoveryEpochState: + """One process's held serialization lifetime, never reusable authorization.""" + + def __init__(self, root: Path, directory: int, lock: int, inventory: EpochInventory, observed: ObservedIdentity, encoded: bytes) -> None: + self.inventory = inventory + self.inventory_sha256 = _digest(encoded) + self._root = root + self._directory = directory + self._lock = lock + self._inventory_observed = observed + self._inventory_encoded = encoded + self._lifetime = issuer_lifetime() + self._records: dict[str, tuple[ObservedIdentity, bytes]] = {} + self._binding: EpochBinding | None = None + self._phase = 0 + self._active = True + self._failed = False + + @property + def descriptor(self) -> int: + """Return the still-held epoch fence for verified receiver custody.""" + self._check() + return self._lock + + def check(self) -> None: + """Revalidate the original inventory and every durable publication.""" + self._check() + + def _check(self) -> None: + if not self._active or self._failed or issuer_lifetime() != self._lifetime: + raise EpochUnavailable("recovery epoch invocation is closed, failed or inherited") + try: + expected_root = (self.inventory.root_device, self.inventory.root_inode) + if identity(self._root, directory=True) != expected_root or (os.fstat(self._directory).st_dev, os.fstat(self._directory).st_ino) != expected_root: + raise EpochUnavailable("recovery epoch root was replaced") + expected_lock = (self.inventory.lock_device, self.inventory.lock_inode) + if identity(self._root / "epoch.lock") != expected_lock or (os.fstat(self._lock).st_dev, os.fstat(self._lock).st_ino) != expected_lock: + raise EpochUnavailable("recovery epoch lock was replaced") + locks = Path(f"/proc/self/fdinfo/{self._lock}").read_text(encoding="ascii").splitlines() + if not any(line.startswith("lock:") and " FLOCK ADVISORY WRITE " in line for line in locks): + raise EpochUnavailable("recovery epoch exclusive lock is not held") + _assert_pid_flock(os.getpid(), self._lock, "WRITE") + if _read(self._root / "inventory.json") != (self._inventory_observed, self._inventory_encoded): + raise EpochUnavailable("recovery epoch inventory changed") + if set(os.listdir(self._directory)) != {"inventory.json", "epoch.lock"} | set(self._records): + raise EpochUnavailable("recovery epoch contains existing, partial or conflicting state") + for name, expected in self._records.items(): + if _read(self._root / name) != expected: + raise EpochUnavailable("recovery epoch record changed") + except (OSError, ValueError, TypeError) as error: + self._failed = True + raise EpochUnavailable("recovery epoch custody unavailable: " + str(error)) from error + + def _require_binding(self, binding: EpochBinding) -> None: + if not isinstance(binding, EpochBinding): + raise EpochUnavailable("typed recovery epoch binding required") + _validate_static(binding) + for field in ("deployment_id", "authorization_id", "enrollment_ids", "request_id", "preparation_id", "target_id", "source_sha256", "config_sha256", "authorization_sha256"): + if getattr(binding, field) != getattr(self.inventory, field): + raise EpochUnavailable("recovery epoch binding differs from fixed inventory: " + field) + if not isinstance(binding.evidence_sha256, str) or not _SHA256.fullmatch(binding.evidence_sha256): + raise EpochUnavailable("invalid independently verified recovery evidence digest") + if not isinstance(binding.evidence_json, str): + raise EpochUnavailable("invalid retained recovery evidence") + if self.inventory.schema_version == 2 or binding.evidence_json: + evidence = binding.evidence_json.encode() + if len(evidence) > _MAX_BYTES // 2 or not isinstance(json.loads(evidence), dict) or canonical(json.loads(evidence)) != evidence or _digest(evidence) != binding.evidence_sha256: + raise EpochUnavailable("retained recovery evidence differs from its complete digest") + if type(binding.host_pid) is not int or (binding.host_boot_id, binding.host_pid, binding.host_start_token) != self._lifetime: + raise EpochUnavailable("recovery epoch binding differs from current issuer lifetime") + if self._binding is not None and binding != self._binding: + raise EpochUnavailable("recovery epoch binding changed after reservation") + + def _create(self, name: str, encoded: bytes) -> None: + descriptor = os.open(name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600, dir_fd=self._directory) + try: + # Publish even an empty/partial attempt before filling it. A later + # invocation refuses the entire namespace instead of repairing it. + os.fsync(self._directory) + with os.fdopen(os.dup(descriptor), "wb") as stream: + stream.write(encoded) + stream.flush() + os.fsync(stream.fileno()) + written = _identity(self._root / name, descriptor, _digest(encoded)) + os.fsync(self._directory) + observed, actual = _read(self._root / name) + # Retain the inode actually fsynced by this writer. Matching bytes + # in a replacement file cannot inherit that durability observation. + if observed != written or _identity(self._root / name, descriptor, _digest(encoded)) != written or os.fstat(descriptor).st_nlink != 1 or actual != encoded: + raise EpochUnavailable("recovery epoch append identity or readback differs") + self._records[name] = (observed, actual) + finally: + os.close(descriptor) + + def _append(self, phase: str, binding: EpochBinding) -> str: + self._check() + try: + self._require_binding(binding) + except BaseException: + # An already reserved epoch cannot recover from conflicting authority + # in the same issuer lifetime, even when the caller catches refusal. + if self._phase: + self._failed = True + raise + if self._phase >= len(_PHASES) or _PHASES[self._phase] != phase: + raise EpochUnavailable("recovery epoch transition is out of order or already consumed") + previous = _digest(self._records[_PHASES[self._phase - 1] + "-head.json"][1]) if self._phase else "" + value = {"schema_version": 1, "phase": phase, "sequence": self._phase, "previous": previous, "inventory_sha256": self.inventory_sha256, "binding": asdict(binding)} + encoded = canonical(value) + try: + self._create(phase + ".json", encoded) + self._check() + head = canonical({"schema_version": 1, "phase": phase, "record_sha256": _digest(encoded), "previous": previous, "inventory_sha256": self.inventory_sha256}) + self._create(phase + "-head.json", head) + self._check() + self._binding = binding + self._phase += 1 + logger.bind(recovery_epoch_phase=phase, request_id=binding.request_id, target_id=binding.target_id, grant_delivered=False).info("Recovery epoch {} durably recorded; no grant delivery observed", phase) + return _digest(encoded) + except BaseException: + # Failure, including cancellation after a successful write, ends + # this state machine. Nothing is deleted or marked retryable. + self._failed = True + raise + + def reserve(self, binding: EpochBinding) -> None: + """Irreversibly reserve this one deployment-wide approved invocation.""" + self._append("reserved", binding) + + def record_intent(self, binding: EpochBinding) -> None: + """Retain the full currently verified binding before final consumption.""" + self._append("intent", binding) + + def consume(self, binding: EpochBinding) -> EpochReceipt: + """Persist consumption and its final receipt before possible dispatch. + + The returned record grants nothing. A caller must retain and verify + actual consumer fence custody and current evidence at its grant boundary. + A lost return value cannot be obtained by another call or invocation. + """ + consumed = self._append("consumed", binding) + receipt = self._append("receipt", binding) + self._check() + return EpochReceipt(self.inventory_sha256, _digest(canonical(asdict(binding))), consumed, receipt) + + +@contextmanager +def _hold_recovery_epoch(root: Path, *, expected_uid: int) -> Iterator[RecoveryEpochState]: + """Internal explicit fixture seam; production never accepts a root input.""" + state: RecoveryEpochState | None = None + if os.geteuid() != expected_uid: + raise EpochUnavailable("recovery epoch requires its approved Unix owner") + try: + root_id = identity(root, directory=True) + with _open(root, directory=True) as directory, _open(root / "epoch.lock") as lock: + if (os.fstat(directory).st_dev, os.fstat(directory).st_ino) != root_id: + raise EpochUnavailable("recovery epoch directory changed while opening") + lock_id = identity(root / "epoch.lock") + if (os.fstat(lock).st_dev, os.fstat(lock).st_ino) != lock_id: + raise EpochUnavailable("recovery epoch lock changed while opening") + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError as error: + raise EpochUnavailable("recovery epoch admission contested") from error + observed, encoded = _read(root / "inventory.json") + inventory = _parse_inventory(encoded, root, root_id, lock_id) + state = RecoveryEpochState(root, directory, lock, inventory, observed, encoded) + state._check() + yield state + state._check() + except (OSError, ValueError, TypeError, KeyError) as error: + if isinstance(error, EpochUnavailable): + raise + raise EpochUnavailable("recovery epoch state unavailable: " + str(error)) from error + finally: + if state is not None: + state._active = False + + +@contextmanager +def hold_recovery_epoch() -> Iterator[RecoveryEpochState]: + """Open only the deployment-fixed root; no request/environment override. + + Provisioning the protected inventory and original empty lock is a separate + reviewed operation. Missing state is unavailable, never auto-initialized. + """ + if os.geteuid() != 0: + raise EpochUnavailable("recovery epoch production issuer requires host root") + try: + _approved_directory(EPOCH_ROOT, uid=0) + with _hold_recovery_epoch(EPOCH_ROOT, expected_uid=0) as state: + yield state + except (OSError, ValueError, TypeError) as error: + if isinstance(error, EpochUnavailable): + raise + raise EpochUnavailable("fixed recovery epoch inventory unavailable: " + str(error)) from error diff --git a/src/auto_coder/management/recovery_source_profile.py b/src/auto_coder/management/recovery_source_profile.py new file mode 100644 index 000000000..75348dd86 --- /dev/null +++ b/src/auto_coder/management/recovery_source_profile.py @@ -0,0 +1,78 @@ +"""Read the actual complete package source of a selected local Docker object. + +This reader does not execute the package, establish original history, admit a +profile or grant authority. Its caller separately verifies commands and mounts. +""" + +from __future__ import annotations + +import hashlib +import io +import json +import os +import re +import subprocess +import tarfile +from pathlib import PurePosixPath + +_PACKAGE = "/opt/auto-coder/.venv/lib/python3.12/site-packages/auto_coder" +_MAX_ARCHIVE = 64 * 1024 * 1024 +_MAX_SOURCE = 8 * 1024 * 1024 +_MAX_FILES = 4096 + + +def _source_archive_identity(encoded: bytes) -> str: + if not encoded or len(encoded) > _MAX_ARCHIVE: + raise ValueError("recovery source archive is absent or oversized") + sources: dict[PurePosixPath, bytes] = {} + names: set[str] = set() + try: + with tarfile.open(fileobj=io.BytesIO(encoded), mode="r:") as archive: + for index, member in enumerate(archive): + if index >= 2 * _MAX_FILES: + raise ValueError("recovery source archive has too many entries") + path = PurePosixPath(member.name) + if path.is_absolute() or ".." in path.parts or not path.parts or path.parts[0] != "auto_coder" or len(member.name) > 4096 or str(path) != member.name.rstrip("/") or str(path) in names: + raise ValueError("recovery source archive path is ambiguous") + names.add(str(path)) + if member.isdir(): + continue + if not member.isfile() or member.size < 0: + raise ValueError("recovery source archive contains an unsupported entry") + relative = path.relative_to("auto_coder") + if relative.suffix in (".pyc", ".pyo", ".so", ".pyd"): + raise ValueError("recovery source package contains alternate executable import bytes") + if relative.suffix != ".py": + continue + if not 0 <= member.size <= _MAX_SOURCE or len(sources) >= _MAX_FILES: + raise ValueError("recovery source package exceeds its bound") + source = archive.extractfile(member) + if source is None: + raise ValueError("recovery source member is unavailable") + data = source.read(_MAX_SOURCE + 1) + if len(data) != member.size: + raise ValueError("recovery source member is truncated") + sources[relative] = data + except (tarfile.TarError, EOFError, OSError) as error: + raise ValueError("recovery source archive cannot be decoded") from error + if not {PurePosixPath("__init__.py"), PurePosixPath("management/bootstrap_supervisor.py"), PurePosixPath("management/original_attempts.py")} <= set(sources): + raise ValueError("recovery original-history package source is incomplete") + digest = hashlib.sha256() + # Match original_attempts.source_identity's Path ordering and byte framing. + for path in sorted(sources): + name, content = path.as_posix().encode(), sources[path] + digest.update(len(name).to_bytes(8, "big") + name + len(content).to_bytes(8, "big") + content) + return "sha256:" + digest.hexdigest() + + +def read_image_source_identity(container_id: str) -> str: + """Hash current package bytes from this exact object on the local engine.""" + if not isinstance(container_id, str) or not re.fullmatch(r"[0-9a-f]{64}", container_id): + raise ValueError("exact recovery source container ID required") + if os.environ.get("DOCKER_HOST") or os.environ.get("DOCKER_CONTEXT"): + raise ValueError("recovery source endpoint override is unsupported") + context = subprocess.run(("docker", "context", "inspect", "--format", "{{json .Endpoints.docker.Host}}"), capture_output=True, text=True, check=True, timeout=10) + if len(context.stdout) > 4096 or json.loads(context.stdout) != "unix:///var/run/docker.sock": + raise ValueError("recovery source requires the local system Docker engine") + archive = subprocess.run(("docker", "cp", container_id + ":" + _PACKAGE, "-"), capture_output=True, check=True, timeout=30) + return _source_archive_identity(archive.stdout) diff --git a/src/auto_coder/management/recovery_watcher_source.py b/src/auto_coder/management/recovery_watcher_source.py new file mode 100644 index 000000000..04af9d647 --- /dev/null +++ b/src/auto_coder/management/recovery_watcher_source.py @@ -0,0 +1,170 @@ +"""Read the finite retained watcher unit behind its actual systemd mask. + +Masked effective units have no executable command to inspect. This recovery-only +profile independently binds the retained fragment selected from the real system +manager's load paths. It never changes a unit, mask, controller or approval. +""" + +from __future__ import annotations + +import os +import re +import stat +import sys +from dataclasses import dataclass +from pathlib import Path + +from .containment_enrollment import _trusted_ancestors +from .containment_route_manifest import SystemdRouteFact, _command +from .recovery_containment_model import RecoveryContainmentUnavailable, RecoveryFile, RecoveryWatcher +from .recovery_containment_profile import read_bound_file + +_UNIT = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,119}\.service") +_LITERAL_PATH = re.compile(r"/[A-Za-z0-9_./:+@=-]+") +_MAX_ENTRIES = 16384 + + +@dataclass(frozen=True) +class RecoveryWatcherUnitFact: + source: RecoveryFile + interpreter: str + load_paths: tuple[str, ...] + load_roots: tuple[tuple[str, int, int, int, int, int], ...] + mask_path: str + mask_identity: tuple[int, ...] + + +def _load_paths() -> tuple[str, ...]: + raw = _command("systemctl", "show", "--property=UnitPath", "--value", max_bytes=16384) + paths = tuple(raw.strip().split()) + if not 1 <= len(paths) <= 32 or len(set(paths)) != len(paths) or any(not _LITERAL_PATH.fullmatch(path) or ".." in Path(path).parts or str(Path(path)) != path for path in paths): + raise RecoveryContainmentUnavailable("recovery watcher system unit load paths are unsupported") + return paths + + +def _drop_in_names(unit: str) -> tuple[str, ...]: + stem = unit.removesuffix(".service") + prefixes = [stem[: index + 1] + ".service.d" for index, character in enumerate(stem) if character == "-"] + return tuple(sorted({"service.d", unit + ".d", *prefixes})) + + +def _literal_command(content: bytes, selected: RecoveryWatcher) -> None: + if len(content) > 16384 or any(value < 32 and value not in (9, 10) or value >= 127 for value in content) or not _UNIT.fullmatch(selected.unit): + raise RecoveryContainmentUnavailable("recovery watcher retained unit is outside the finite profile") + expected = sys.executable + " -I -B -m auto_coder.management.proxy_watch_cli --config " + selected.configuration.path + if any(not _LITERAL_PATH.fullmatch(path) or ".." in Path(path).parts for path in (sys.executable, selected.configuration.path)): + raise RecoveryContainmentUnavailable("recovery watcher literal interpreter/configuration path is unsupported") + sections: dict[str, dict[str, str]] = {} + section = "" + for raw in content.decode("ascii").splitlines(): + line = raw.strip() + if not line or line.startswith("#"): + continue + if line in ("[Unit]", "[Service]"): + section = line[1:-1] + if section in sections: + raise RecoveryContainmentUnavailable("recovery watcher retained unit repeats a section") + sections[section] = {} + continue + key, separator, value = line.partition("=") + allowed = {"Description"} if section == "Unit" else {"Type", "ExecStart", "Restart", "StandardOutput", "StandardError"} if section == "Service" else set() + if not separator or key not in allowed or key in sections[section] or not value or any(character in value for character in ("\\", "%", "\r", "\x00")): + raise RecoveryContainmentUnavailable("recovery watcher retained unit has unsupported directives") + sections[section][key] = value + service = sections.get("Service", {}) + if service.get("Type") != "simple" or service.get("Restart") != "no" or service.get("ExecStart") != expected: + raise RecoveryContainmentUnavailable("recovery watcher command does not bind this interpreter/module/configuration") + for name in ("StandardOutput", "StandardError"): + value = service.get(name, "journal") + if value not in ("journal", "null") and (not value.startswith("append:") or not _LITERAL_PATH.fullmatch(value.removeprefix("append:")) or ".." in Path(value.removeprefix("append:")).parts): + raise RecoveryContainmentUnavailable("recovery watcher retained output selection is unsupported") + + +def _scope(selected: RecoveryWatcher, *, expected_uid: int) -> RecoveryWatcherUnitFact: + paths = _load_paths() + source = Path(selected.unit_source.path) + if source.name != selected.unit or str(source.parent) not in paths: + raise RecoveryContainmentUnavailable("recovery watcher retained source is outside the actual unit load path") + roots = [] + masks: list[tuple[int, str, tuple[int, ...]]] = [] + fragments: list[tuple[int, str]] = [] + aliases: list[tuple[str, str]] = [] + entries = 0 + for priority, name in enumerate(paths): + root = Path(name) + if root.resolve(strict=False) != root: + raise RecoveryContainmentUnavailable("recovery watcher unit load path is redirected") + try: + info = root.lstat() + except FileNotFoundError: + ancestor = root.parent + while not ancestor.exists(): + ancestor = ancestor.parent + _trusted_ancestors(ancestor, uid=expected_uid) + roots.append((name, 0, 0, 0, 0, 0)) + continue + _trusted_ancestors(root, uid=expected_uid) + roots.append((name, info.st_dev, info.st_ino, info.st_mode, info.st_uid, info.st_gid)) + for drop_in in _drop_in_names(selected.unit): + try: + (root / drop_in).lstat() + except FileNotFoundError: + continue + raise RecoveryContainmentUnavailable("recovery watcher retained unit has unsupported drop-ins") + candidate = root / selected.unit + try: + candidate_info = candidate.lstat() + except FileNotFoundError: + candidate_info = None + if candidate_info is not None: + if candidate_info.st_uid != expected_uid or candidate_info.st_nlink != 1: + raise RecoveryContainmentUnavailable("recovery watcher fragment/mask is not independently protected") + if stat.S_ISLNK(candidate_info.st_mode) and os.readlink(candidate) == "/dev/null": + identity = (candidate_info.st_dev, candidate_info.st_ino, candidate_info.st_ctime_ns, candidate_info.st_mode, candidate_info.st_uid, candidate_info.st_gid) + masks.append((priority, str(candidate), identity)) + elif stat.S_ISREG(candidate_info.st_mode) and candidate == source: + fragments.append((priority, str(candidate))) + else: + raise RecoveryContainmentUnavailable("recovery watcher retained fragment selection is ambiguous") + with os.scandir(root) as listing: + for entry in listing: + entries += 1 + if entries > _MAX_ENTRIES: + raise RecoveryContainmentUnavailable("recovery watcher unit alias inventory exceeds its bound") + if entry.name == selected.unit or not entry.name.endswith(".service") or not entry.is_symlink(): + continue + target = os.readlink(entry.path) + if not target or len(target) > 4096: + raise RecoveryContainmentUnavailable("recovery watcher unit alias target is unsupported") + # systemd resolves aliases inside UnitPath by unit name even + # when their literal target file is absent at that load level. + aliases.append((entry.name, Path(target).name)) + try: + resolved = Path(entry.path).resolve(strict=False) + except RuntimeError as error: + raise RecoveryContainmentUnavailable("recovery watcher unit alias inventory contains a loop") from error + if resolved == source: + raise RecoveryContainmentUnavailable("recovery watcher retained fragment has an unsupported alias") + names = {selected.unit} + for _ in range(len(aliases) + 1): + related = {name for name, target in aliases if target in names} + if related <= names: + break + names.update(related) + if names != {selected.unit}: + raise RecoveryContainmentUnavailable("recovery watcher retained unit name has an unsupported alias") + if len(masks) != 1 or len(fragments) != 1 or masks[0][0] >= fragments[0][0]: + raise RecoveryContainmentUnavailable("recovery watcher requires one real mask above its retained fragment") + return RecoveryWatcherUnitFact(selected.unit_source, sys.executable, paths, tuple(roots), masks[0][1], masks[0][2]) + + +def observe_recovery_watcher_unit(selected: RecoveryWatcher, fact: SystemdRouteFact, *, expected_uid: int = 0) -> RecoveryWatcherUnitFact: + """Repeat actual load-path/source reads; effective inhibition is separate.""" + if fact.unit != selected.unit or not fact.inhibited or fact.main_pid != 0 or fact.needs_daemon_reload: + raise RecoveryContainmentUnavailable("recovery watcher effective unit is not technically inhibited") + first = _scope(selected, expected_uid=expected_uid) + content = read_bound_file(selected.unit_source, source=True, expected_uid=expected_uid) + _literal_command(content, selected) + if _scope(selected, expected_uid=expected_uid) != first or read_bound_file(selected.unit_source, source=True, expected_uid=expected_uid) != content: + raise RecoveryContainmentUnavailable("recovery watcher retained unit source changed during observation") + return first diff --git a/src/auto_coder/management/worker_lifecycle.py b/src/auto_coder/management/worker_lifecycle.py index e15f3be1a..6a3ca1f83 100644 --- a/src/auto_coder/management/worker_lifecycle.py +++ b/src/auto_coder/management/worker_lifecycle.py @@ -713,9 +713,9 @@ def _step(self, record: OperationRecord, version: int, snapshot: dict[str, objec steps = snapshot["steps"] assert isinstance(steps, dict) identity = self._original_supervisor(client, snapshot) - if identity == "CHANGED": - return self._finish(record, version, snapshot, "NEEDS_ATTENTION", "original_supervisor_incarnation_changed") if identity != "SAME": + # Reconstruction can occur after _process_locked's first check. + # Its next iteration must apply the original retained-receipt gate. return record if name not in steps: if not self.runtime.retained_effect_permitted(record): diff --git a/src/auto_coder/worktree_utils.py b/src/auto_coder/worktree_utils.py index 9823d9c61..f38874849 100644 --- a/src/auto_coder/worktree_utils.py +++ b/src/auto_coder/worktree_utils.py @@ -494,7 +494,8 @@ def _capture_source(target: Path, workspace: Path, ownership: Optional[LocalWork def _capture_git_patch(root: Path, destination: Path, *args: str) -> Path: """Spool binary patches to disk instead of retaining their full stdout.""" with destination.open("wb") as stream: - result = subprocess.run(["git", "diff", *args, "--binary", "--full-index"], cwd=root, stdout=stream, stderr=subprocess.PIPE) + # Diff's cached-stat refresh would otherwise rewrite the bound caller index. + result = subprocess.run(["git", "-c", "diff.autoRefreshIndex=false", "diff", *args, "--binary", "--full-index"], cwd=root, stdout=stream, stderr=subprocess.PIPE) if result.returncode != 0: raise WorkspacePreparationError(f"unable to capture Git patch: {os.fsdecode(result.stderr).strip()}") return destination diff --git a/tests/test_build_provenance.py b/tests/test_build_provenance.py index c77a45f73..9f28f8e07 100644 --- a/tests/test_build_provenance.py +++ b/tests/test_build_provenance.py @@ -31,7 +31,7 @@ class Installed: def _dockerfile_embed_command() -> str: """The production Dockerfile's embed RUN instruction, verbatim.""" lines = (REPO_ROOT / "Dockerfile").read_text(encoding="utf-8").splitlines() - match = [m.group(1) for line in lines if (m := re.match(r"RUN (python -m auto_coder\.build_provenance embed)$", line))] + match = [m.group(1) for line in lines if (m := re.match(r"RUN (python -B -m auto_coder\.build_provenance embed)$", line))] assert len(match) == 1 return match[0] diff --git a/tests/test_invocation_process_supervisor.py b/tests/test_invocation_process_supervisor.py index c9b0d8973..5de2c052e 100644 --- a/tests/test_invocation_process_supervisor.py +++ b/tests/test_invocation_process_supervisor.py @@ -365,7 +365,8 @@ def test_timeout_and_confirmation_uncertainty_remain_distinct(tmp_path: Path) -> assert supervisor.settle_retained(result.invocation_id) -def test_cancelling_one_owner_does_not_stop_peer_or_delete_results(tmp_path: Path) -> None: +def test_cancelling_one_owner_does_not_stop_peer_or_delete_results(tmp_path: Path, _use_real_sleep) -> None: + # Actual concurrent owners and readiness polling must yield between observations. supervisor = make_supervisor(ProcessGroupOwner(tmp_path / "owners")) cancel = threading.Event() first_result = tmp_path / "first-result" diff --git a/tests/test_management_archived_cutover_container.py b/tests/test_management_archived_cutover_container.py index ffe48ca40..dc422b96d 100644 --- a/tests/test_management_archived_cutover_container.py +++ b/tests/test_management_archived_cutover_container.py @@ -25,7 +25,7 @@ import urllib.error import urllib.request import uuid -from contextlib import ExitStack +from contextlib import ExitStack, closing from dataclasses import asdict, dataclass, replace from pathlib import Path @@ -147,7 +147,11 @@ def path(self, name: str) -> Path: class Chain: - def __init__(self, root: Path, *, with_workers: bool = False, original_history: bool = False): + def __init__(self, root: Path, *, with_workers: bool = False, original_history: bool = False, original_genesis: bool = False): + if original_genesis and original_history: + raise ValueError("original genesis cannot reenroll a prospective history") + self.original_genesis = original_genesis + self.genesis = None self.root = root self.names: list[str] = [] self.generations: list[Generation] = [] @@ -162,7 +166,8 @@ def __init__(self, root: Path, *, with_workers: bool = False, original_history: reservation.bind(("127.0.0.1", 0)) self.port = reservation.getsockname()[1] self.oauth = root / "oauth" - self.oauth.mkdir(mode=0o700) + if not original_genesis: + self.oauth.mkdir(mode=0o700) self.history_binding = None if original_history: history = root / "original-history" @@ -185,7 +190,7 @@ def __init__(self, root: Path, *, with_workers: bool = False, original_history: def prepare_caddy_image(self) -> str: """Use the real pinned Caddy binary with the supported bind-only profile.""" - source = _resolve_fixture_base("caddy@sha256:504a44d7d74bd2bc28a66afc4774912fc21223ca8a67734af1c54ab2f999b489") + source = _resolve_fixture_base(os.environ["AUTOCODER_RECOVERY_CADDY_IMAGE"] if self.original_genesis else "caddy@sha256:504a44d7d74bd2bc28a66afc4774912fc21223ca8a67734af1c54ab2f999b489") binary = self.root / "caddy" donor = self.docker("create", "--network", "none", source) target = None @@ -238,17 +243,21 @@ def docker(*args: str, timeout: int = 60) -> str: @staticmethod def checkpoint(database: Path) -> None: - with sqlite3.connect(database) as connection: + # Checkpoint an existing producer store, then release its WAL/SHM + # handles before the immutable original-evidence baseline is taken. + with closing(sqlite3.connect(database.as_uri() + "?mode=rw", uri=True)) as connection: assert connection.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone()[0] == 0 def grants(self): - with sqlite3.connect(self.oauth / "oauth.sqlite3") as connection: + with closing(sqlite3.connect(self.oauth / "oauth.sqlite3")) as connection: return {name: connection.execute(f"SELECT * FROM {name} ORDER BY 1").fetchall() for name in ("authority", "grants", "access_tokens", "refresh_tokens")} def prepare(self, name: str, prior: Generation | None = None, *, omit_archive: bool = False) -> Generation: root = self.root / name root.mkdir(mode=0o700) for part in ("state", "journal", "guard", "bootstrap"): + if self.original_genesis and prior is None and part in ("state", "journal"): + continue (root / part).mkdir(mode=0o700) runtime = { "reference": "bootstrap", @@ -275,7 +284,7 @@ def prepare(self, name: str, prior: Generation | None = None, *, omit_archive: b if prior is not None: captured = capture_prior_container(prior.container, self.image, expected_mounts=tuple((str(a), b, c) for a, b, c in prior.mounts), require_owner_lock=True, require_deployment_identity=True) self.ensure_proxy(prior) - proxy = capture_prior_container(prior.proxy, self.proxy_image, expected_mounts=((str(prior.path("Caddyfile")), "/etc/caddy/Caddyfile", False),)) + proxy = capture_prior_container(prior.proxy, self.proxy_image, expected_mounts=self.proxy_mounts(prior)) bindings = required_exclusions(captured) config.update(fresh_guard_path="/management-initialization/fresh-guard.sock", fresh_prior_lock_path="/management-prior-owner.lock", archived_exclusions=[] if omit_archive else [asdict(item) for item in bindings]) if self.worker_environment is not None: @@ -329,7 +338,26 @@ def prepare(self, name: str, prior: Generation | None = None, *, omit_archive: b for item in bindings: mounts += [(Path(item.host_journal_root), item.journal_root, False), (Path(item.host_fence), item.fence, False)] self.configure_profile(name, root, config, runtime, management, mounts, fresh=prior is not None) + if self.original_genesis and prior is None: + config["original_genesis"] = "/bootstrap/original-genesis.json" config_file.write_text(json.dumps(config)) + if self.original_genesis: + management.chmod(0o600) + root_authority = None + if self.original_genesis and prior is None: + from auto_coder.management.original_attempts import canonical + from auto_coder.management.original_genesis import ROOTS_PROFILE, GenesisRootAuthorization, prepare_original_genesis_roots + + caddyfile = self._write_proxy_config(root) + history = self.root / "original-history" + roots_request = GenesisRootAuthorization( + ROOTS_PROFILE, "separately-authorized-new-deployment", "isolated-original-recovery-" + uuid.uuid4().hex, str(uuid.uuid4()), str(config_file), str(management), str(caddyfile), str(root / "state"), str(root / "journal"), str(self.oauth), str(history) + ) + roots_authorization = root / "original-root-authorization.json" + roots_authorization.write_bytes(canonical(asdict(roots_request))) + roots_authorization.chmod(0o600) + root_authority = prepare_original_genesis_roots(roots_authorization) + mounts.append((history, "/management-original-history", False)) container_name = "ancestor-fixture-" + uuid.uuid4().hex[:16] self.names.append(container_name) args = ["create", "--name", container_name, "--read-only", "--publish", f"127.0.0.1:{self.port}:8080"] @@ -338,7 +366,10 @@ def prepare(self, name: str, prior: Generation | None = None, *, omit_archive: b args += ["--network", "bridge"] for source, target, writable in mounts: args += ["--mount", f"type=bind,src={source},dst={target}" + ("" if writable else ",readonly")] - args += [self.image, "--config", "/bootstrap/config.json"] + if self.original_genesis: + args += ["--workdir", "/workspace", "--env", "PYTHONDONTWRITEBYTECODE=1", "--entrypoint", "python", self.image, "-m", "auto_coder.management.bootstrap_supervisor", "--config", "/bootstrap/config.json"] + else: + args += [self.image, "--config", "/bootstrap/config.json"] if prior is not None: args += ["--fresh-initialization"] container = self.docker(*args) @@ -348,6 +379,18 @@ def prepare(self, name: str, prior: Generation | None = None, *, omit_archive: b assert set(connected) == {"bridge", self.worker_network} generation = Generation(name, root, container, mounts) self.generations.append(generation) + if self.original_genesis: + self._create_recovery_proxy(generation) + if root_authority is not None: + from auto_coder.management.original_genesis import PROFILE, GenesisSelection, capture_original_genesis + + assert generation.proxy is not None + selected = GenesisSelection(PROFILE, roots_request.purpose, roots_request.deployment_id, roots_request.authorization_id, generation.container, generation.proxy, roots_request.history_root, str(config_file), str(management), roots_request.proxy_config_path, str(root_authority)) + authorization = root / "original-capture-authorization.json" + authorization.write_bytes(canonical(asdict(selected))) + authorization.chmod(0o600) + self.genesis = capture_original_genesis(authorization) + self.history_binding = self.genesis.binding if captured is not None: generation.network_fd = os.open(f"/proc/{captured.pid}/ns/net", os.O_RDONLY) self.fds.append(generation.network_fd) @@ -440,6 +483,8 @@ def stop(self, generation: Generation, *, stop_proxy: bool = True) -> None: def ensure_proxy(self, generation: Generation) -> None: if generation.proxy is not None: + if self.original_genesis and self.docker("inspect", "--format", "{{.State.Status}}", generation.proxy) == "created": + self.docker("start", generation.proxy) return actual = json.loads(self.docker("inspect", "--format", "{{json .NetworkSettings.Ports}}", generation.container)) assert actual == {"8080/tcp": [{"HostIp": "127.0.0.1", "HostPort": str(self.port)}]}, "actual loopback Caddy port mapping unavailable" @@ -452,6 +497,27 @@ def ensure_proxy(self, generation: Generation) -> None: "run", "-d", "--name", name, "--network", "container:" + generation.container, "--mount", f"type=bind,src={caddyfile},dst=/etc/caddy/Caddyfile,readonly", "--entrypoint", "caddy", self.proxy_image, "run", "--config", "/etc/caddy/Caddyfile", "--adapter", "caddyfile" ) + def proxy_mounts(self, generation: Generation) -> tuple[tuple[str, str, bool], ...]: + return ((str(generation.path("Caddyfile")), "/etc/caddy/Caddyfile", False),) + + @staticmethod + def _write_proxy_config(root: Path) -> Path: + path = root / "Caddyfile" + if not path.exists(): + path.write_text("{\n admin off\n auto_https off\n}\nhttp://127.0.0.1:8080 {\n reverse_proxy 127.0.0.1:8081 {\n header_up Host 127.0.0.1:8081\n }\n}\n") + path.chmod(0o600) + return path + + def _create_recovery_proxy(self, generation: Generation) -> None: + """Create the real paired object before original capture, without start.""" + self._write_proxy_config(generation.root) + name = "original-recovery-proxy-" + uuid.uuid4().hex[:16] + self.names.append(name) + arguments = ["create", "--name", name, "--workdir", "/srv", "--network", "container:" + generation.container, "--entrypoint", "caddy"] + for source, destination, writable in self.proxy_mounts(generation): + arguments += ["--mount", f"type=bind,src={source},dst={destination}" + ("" if writable else ",readonly")] + generation.proxy = self.docker(*arguments, self.proxy_image, "run", "--config", "/etc/caddy/Caddyfile", "--adapter", "caddyfile") + def call(self, generation: Generation, tool: str, arguments: dict[str, object], *, authenticated: bool = True): assert generation.proxy is not None request = {"jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": {"name": tool, "arguments": arguments}} @@ -467,7 +533,47 @@ def call(self, generation: Generation, tool: str, arguments: dict[str, object], return {"http_status": error.code} raise ValueError("synthetic authenticated proxy HTTP unavailable") from None - def ready(self, generation: Generation, *, fresh: bool) -> None: + def fresh_status(self, generation: Generation, *, require_live: bool = False) -> dict[str, object]: + code = "import json;from pathlib import Path;from auto_coder.management.fresh_initialization import read_fresh_status;status=read_fresh_status(Path('/management-child-state'))\n" + if require_live: + code += ( + "import time\n" + "from auto_coder.management.bootstrap_supervisor import BootstrapConfig,ManagementBootstrapSupervisor\n" + "from auto_coder.management.self_update_handoff import observe_supervisor\n" + "config=BootstrapConfig.read(Path('/bootstrap/config.json'))\n" + "observed=observe_supervisor(config)\n" + "if status.get('status') not in {'launched','succeeded'} or observed is None:\n" + " raise ValueError('fresh launch is not live')\n" + "owner,child=observed\n" + "if child.phase.value!='running' or owner!=status['owner_incarnation'] or child.supervisor_incarnation!=owner or ManagementBootstrapSupervisor._identity(child)!=status['child_identity']:\n" + " raise ValueError('fresh launch lifetime differs')\n" + "if not status['started']<=time.time()<=status['deadline'] or not 0 dict[str, object]: + # Source verification precedes dispatch. Give that phase its own finite + # budget, just as the first A child is observed before HTTP readiness. + deadline = time.monotonic() + 60 + last = "not observed" + while time.monotonic() < deadline: + try: + launch = self.fresh_status(generation, require_live=True) + last = str(launch["status"]) + if launch["status"] in {"launched", "succeeded"} and time.monotonic() < deadline: + return launch + except (subprocess.CalledProcessError, ValueError, KeyError, OSError) as error: + last = type(error).__name__ + time.sleep(0.1) + raise AssertionError("exact synthetic fresh launch was not proven; " + last) + + @staticmethod + def same_fresh_launch(first: dict[str, object], second: dict[str, object]) -> bool: + return {key: value for key, value in first.items() if key != "status"} == {key: value for key, value in second.items() if key != "status"} + + def ready(self, generation: Generation, *, fresh: bool, expected_launch: dict[str, object] | None = None) -> None: self.ensure_proxy(generation) deadline = time.monotonic() + 60 last = "not observed" @@ -475,12 +581,13 @@ def ready(self, generation: Generation, *, fresh: bool) -> None: try: observation = "" if fresh: - code = "import json;from pathlib import Path;from auto_coder.management.fresh_initialization import read_fresh_status;print(json.dumps(read_fresh_status(Path('/management-child-state'))))" - status = json.loads(self.docker("exec", generation.container, "python", "-c", code, timeout=10).splitlines()[-1]) + status = self.fresh_status(generation) last = "fresh initialization status=" + str(status["status"]) if status["status"] not in {"launched", "succeeded"}: time.sleep(0.1) continue + if expected_launch is not None and not self.same_fresh_launch(status, expected_launch): + raise ValueError("fresh initialization launch changed after dispatch") observation = status["observation_id"] last = "unauthenticated Caddy challenge" if self.call(generation, "auto_coder_whoami", {}, authenticated=False)["http_status"] != 401: @@ -488,7 +595,16 @@ def ready(self, generation: Generation, *, fresh: bool) -> None: continue last = "authenticated Caddy whoami" result = self.call(generation, "auto_coder_whoami", {"observation_id": observation}) - if result["uid"] == "operator" and (not fresh or status["status"] == "succeeded"): + confirmed = result["uid"] == "operator" + if confirmed and fresh: + # Authenticated I/O can complete readiness after the initial + # sample. Require a current terminal result for that same launch. + terminal = self.fresh_status(generation) + last = "fresh initialization terminal status=" + str(terminal["status"]) + if not self.same_fresh_launch(terminal, status): + raise ValueError("fresh initialization launch changed during authenticated readback") + confirmed = terminal["status"] == "succeeded" + if confirmed: if self.worker_baseline is not None: last = "authenticated worker identity readback" self.assert_worker_reads(generation) @@ -503,7 +619,7 @@ def ready(self, generation: Generation, *, fresh: bool) -> None: @staticmethod def child(generation: Generation): - with sqlite3.connect((generation.path("state") / "management-child.sqlite3").as_uri() + "?mode=ro", uri=True) as connection: + with closing(sqlite3.connect((generation.path("state") / "management-child.sqlite3").as_uri() + "?mode=ro", uri=True)) as connection: return json.loads(connection.execute("SELECT record_json FROM child_state").fetchone()[0]) def prove_child(self, generation: Generation) -> None: @@ -530,7 +646,8 @@ def start_cutover(self, old: Generation, new: Generation) -> None: self.docker("start", new.container) out, err = process.communicate(timeout=60) assert process.returncode == 0, (out, err) - self.ready(new, fresh=True) + launch = self.wait_for_fresh_launch(new) + self.ready(new, fresh=True, expected_launch=launch) self.prove_child(new) def seed(self, generation: Generation): @@ -767,7 +884,7 @@ def test_real_second_cutover_preserves_all_owner_and_key_exclusions(chain: Chain deadline = time.monotonic() + 30 while True: try: - with sqlite3.connect((a.path("state") / "management-child.sqlite3").as_uri() + "?mode=ro", uri=True) as db: + with closing(sqlite3.connect((a.path("state") / "management-child.sqlite3").as_uri() + "?mode=ro", uri=True)) as db: row = db.execute("SELECT record_json FROM child_state").fetchone() record = json.loads(row[0]) if row else {} if record.get("phase") == "running" and record.get("pid", 0) > 0: @@ -827,7 +944,7 @@ def test_real_second_cutover_preserves_all_owner_and_key_exclusions(chain: Chain assert not (c.path("state") / "fresh-launch-claimed.json").exists() ledger = c.path("state") / "management-child.sqlite3" if ledger.exists(): - with sqlite3.connect(ledger) as db: + with closing(sqlite3.connect(ledger)) as db: assert db.execute("SELECT count(*) FROM effects").fetchone()[0] == 0 assert db.execute("SELECT count(*) FROM child_state").fetchone()[0] == 0 assert not list(c.path("journal").iterdir()) @@ -844,7 +961,7 @@ def test_real_second_cutover_preserves_all_owner_and_key_exclusions(chain: Chain assert chain.call(c, "auto_coder_operation_get", {"operation_id": old.operation_id})["status"] == "NOT_FOUND" else: assert response["operation_id"] == old.operation_id and response["history_origin"] == "retained_prior_instance" - with sqlite3.connect(c.path("journal") / "management-operations.sqlite3") as db: + with closing(sqlite3.connect(c.path("journal") / "management-operations.sqlite3")) as db: assert db.execute("SELECT count(*) FROM operations").fetchone()[0] == 0 # Preserve host fences across the deliberate clean C reconstruction. held = [] diff --git a/tests/test_management_archived_cutover_readiness.py b/tests/test_management_archived_cutover_readiness.py new file mode 100644 index 000000000..7e20e20d0 --- /dev/null +++ b/tests/test_management_archived_cutover_readiness.py @@ -0,0 +1,232 @@ +"""Component checks for the native fixture's readiness observation order. + +Controlled replies exercise the fixture oracle only; they are not evidence of +Docker, authenticated application readiness, or a successful native cutover. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from tests import test_management_archived_cutover_container as cutover + + +def _ready_case(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, *, initial_status: str = "launched", terminal_status: str = "succeeded", operator: bool = True, changed_field: str | None = None, challenge: int = 401): + generation = cutover.Generation("manager-c", tmp_path, "exact-manager-c", [], proxy="exact-proxy-c") + chain = cutover.Chain.__new__(cutover.Chain) + chain.worker_baseline = None + launch = {"request_id": "request-c", "owner_incarnation": "owner-c", "child_identity": "child-c", "observation_id": "a" * 32, "started": 100.0, "deadline": 160.0} + current = [{**launch, "status": initial_status}] + reads: list[dict[str, object]] = [] + authenticated_calls: list[bool] = [] + + def docker(*arguments, **kwargs): + if arguments[0] == "exec": + assert arguments[1] == generation.container + reads.append(dict(current[0])) + return json.dumps(current[0]) + assert arguments[0] == "inspect" + return "running" + + def call(selected, tool, arguments, *, authenticated=True): + assert selected is generation and tool == "auto_coder_whoami" + authenticated_calls.append(authenticated) + if not authenticated: + return {"http_status": challenge} + assert arguments == {"observation_id": launch["observation_id"]} + current[0] = {**launch, "status": terminal_status} + if changed_field is not None: + current[0][changed_field] = "different-generation" + return {"uid": "operator" if operator else "different-principal"} + + # A single poll starts within the existing budget. A second poll would be + # too late, reproducing the completed transition during authenticated I/O. + ticks = iter((0.0, 1.0, 61.0)) + monkeypatch.setattr(cutover, "time", SimpleNamespace(monotonic=lambda: next(ticks, 61.0), sleep=lambda _: None)) + monkeypatch.setattr(chain, "ensure_proxy", lambda selected: None) + monkeypatch.setattr(chain, "docker", docker) + monkeypatch.setattr(chain, "call", call) + return chain, generation, reads, authenticated_calls + + +@pytest.mark.parametrize("initial_status", ["launched", "succeeded"]) +def test_terminal_read_follows_authenticated_whoami(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, initial_status: str) -> None: + chain, generation, reads, calls = _ready_case(monkeypatch, tmp_path, initial_status=initial_status) + chain.ready(generation, fresh=True) + assert [value["status"] for value in reads] == [initial_status, "succeeded"] + assert calls == [False, True] + + +@pytest.mark.parametrize("changed_field", ["observation_id", "child_identity"]) +def test_terminal_read_must_bind_the_same_authenticated_launch(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, changed_field: str) -> None: + chain, generation, reads, calls = _ready_case(monkeypatch, tmp_path, changed_field=changed_field) + with pytest.raises(AssertionError, match="exact synthetic child readiness was not proven"): + chain.ready(generation, fresh=True) + assert len(reads) == 2 and calls == [False, True] + + +def test_authenticated_whoami_does_not_replace_terminal_success(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + chain, generation, reads, calls = _ready_case(monkeypatch, tmp_path, terminal_status="launched") + with pytest.raises(AssertionError, match="exact synthetic child readiness was not proven"): + chain.ready(generation, fresh=True) + assert [value["status"] for value in reads] == ["launched", "launched"] + assert calls == [False, True] + + +def test_terminal_success_does_not_replace_operator_identity(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + chain, generation, reads, calls = _ready_case(monkeypatch, tmp_path, operator=False) + with pytest.raises(AssertionError, match="exact synthetic child readiness was not proven"): + chain.ready(generation, fresh=True) + assert len(reads) == 1 and calls == [False, True] + + +def test_unauthenticated_challenge_must_still_refuse_access(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + chain, generation, reads, calls = _ready_case(monkeypatch, tmp_path, challenge=200) + with pytest.raises(AssertionError, match="exact synthetic child readiness was not proven"): + chain.ready(generation, fresh=True) + assert len(reads) == 1 and calls == [False] + + +def _cutover_case(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, *, ready_at: float = 80.0, replaced: str | None = None): + chain, generation, _, _ = _ready_case(monkeypatch, tmp_path) + clock = SimpleNamespace(now=0.0) + launch = {"request_id": "request-c", "owner_incarnation": "owner-c", "child_identity": "child-c", "observation_id": "a" * 32, "started": 100.0, "deadline": 220.0, "status": "launched"} + samples = [] + authenticated = [] + proved = [] + + def status(selected, *, require_live=False): + assert selected is generation + if not samples: + clock.now += 45.0 + samples.append((require_live, clock.now)) + result = {**launch, "status": "succeeded" if authenticated else "launched"} + if not require_live and (replaced == "before-auth" or (replaced == "after-auth" and authenticated)): + result["observation_id"] = "b" * 32 + return result + + def call(selected, tool, arguments, *, authenticated=True): + assert selected is generation and tool == "auto_coder_whoami" + if not authenticated: + return {"http_status": 401} + if clock.now < ready_at: + raise ValueError("application startup is still in progress") + assert arguments == {"observation_id": launch["observation_id"]} + accepted.append(clock.now) + return {"uid": "operator"} + + accepted = authenticated + monkeypatch.setattr(cutover, "time", SimpleNamespace(monotonic=lambda: clock.now, sleep=lambda _: setattr(clock, "now", clock.now + 1.0))) + monkeypatch.setattr(chain, "fresh_status", status) + monkeypatch.setattr(chain, "call", call) + monkeypatch.setattr(chain, "host", lambda selected: SimpleNamespace(communicate=lambda **kwargs: ("", ""), returncode=0)) + monkeypatch.setattr(chain, "status", lambda *args: None) + monkeypatch.setattr(chain, "stop", lambda *args: None) + monkeypatch.setattr(chain, "docker", lambda *args, **kwargs: "running") + monkeypatch.setattr(chain, "prove_child", lambda selected: proved.append(selected)) + return chain, generation, clock, samples, proved + + +def test_prelaunch_source_work_has_a_separate_finite_budget(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + chain, generation, clock, samples, proved = _cutover_case(monkeypatch, tmp_path) + chain.start_cutover(generation, generation) + assert samples[0] == (True, 45.0) + assert clock.now == 80.0 + assert proved == [generation] + + +@pytest.mark.parametrize("replaced", ["before-auth", "after-auth"]) +def test_launch_cannot_change_between_phase_gate_and_readiness(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, replaced: str) -> None: + chain, generation, clock, _, proved = _cutover_case(monkeypatch, tmp_path, replaced=replaced) + with pytest.raises(AssertionError, match="exact synthetic child readiness was not proven"): + chain.start_cutover(generation, generation) + assert clock.now == 105.0 + assert proved == [] + + +def test_readiness_retries_do_not_renew_the_sixty_second_budget(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + chain, generation, clock, _, proved = _cutover_case(monkeypatch, tmp_path, ready_at=106.0) + with pytest.raises(AssertionError, match="exact synthetic child readiness was not proven"): + chain.start_cutover(generation, generation) + assert clock.now == 105.0 + assert proved == [] + + +@pytest.mark.parametrize("state", ["pending", "invalid", "unknown", "late-launch"]) +def test_launch_gate_refuses_without_renewing_its_budget(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, state: str) -> None: + chain, generation, _, _ = _ready_case(monkeypatch, tmp_path) + clock = SimpleNamespace(now=0.0) + samples = [] + + def status(selected, *, require_live=False): + assert selected is generation and require_live + samples.append(clock.now) + if state == "invalid": + raise ValueError("production fresh record or live lifetime is invalid") + if state == "late-launch": + clock.now = 61.0 + return {"status": "launched"} + return {"status": state} + + monkeypatch.setattr(cutover, "time", SimpleNamespace(monotonic=lambda: clock.now, sleep=lambda _: setattr(clock, "now", clock.now + 1.0))) + monkeypatch.setattr(chain, "fresh_status", status) + with pytest.raises(AssertionError, match="exact synthetic fresh launch was not proven"): + chain.wait_for_fresh_launch(generation) + assert 60.0 <= clock.now <= 62.0 + assert 1 <= len(samples) <= 60 + + +@pytest.mark.parametrize("mismatch", [None, "missing", "owner", "child", "phase", "expired", "window", "terminal-failed"]) +def test_live_launch_reader_joins_actual_production_observations(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, mismatch: str | None) -> None: + import io + import time + from contextlib import redirect_stdout + + from auto_coder.management import bootstrap_supervisor, fresh_initialization, self_update_handoff + + chain, generation, _, _ = _ready_case(monkeypatch, tmp_path) + child = SimpleNamespace(pid=42, start_token="start-c", child_generation=1, selected_runtime="runtime-c", supervisor_incarnation="owner-c", phase=SimpleNamespace(value="running")) + now = time.time() + launch = { + "request_id": "request-c", + "owner_incarnation": "owner-c", + "child_identity": bootstrap_supervisor.ManagementBootstrapSupervisor._identity(child), + "observation_id": "a" * 32, + "started": now - 5, + "deadline": now + 115, + "status": "launched", + } + owner = "different-owner" if mismatch == "owner" else "owner-c" + if mismatch == "child": + child.pid = 43 + if mismatch == "phase": + child.phase.value = "stopped" + if mismatch == "expired": + launch.update(started=now - 121, deadline=now - 1) + if mismatch == "window": + launch["deadline"] = now + 500 + if mismatch == "terminal-failed": + launch["status"] = "unknown" + monkeypatch.setattr(fresh_initialization, "read_fresh_status", lambda path: launch) + monkeypatch.setattr(bootstrap_supervisor.BootstrapConfig, "read", classmethod(lambda cls, path: SimpleNamespace(verification_timeout=120))) + monkeypatch.setattr(self_update_handoff, "observe_supervisor", lambda config: None if mismatch == "missing" else (owner, child)) + + def docker(*arguments, **kwargs): + assert arguments[:4] == ("exec", generation.container, "python", "-c") + captured = io.StringIO() + # Execute the actual fixture reader program with controlled production + # observations; these component tests do not claim kernel evidence. + with redirect_stdout(captured): + exec(arguments[4], {}) + return captured.getvalue() + + monkeypatch.setattr(chain, "docker", docker) + if mismatch is None: + assert chain.fresh_status(generation, require_live=True) == launch + else: + with pytest.raises(ValueError, match="fresh launch"): + chain.fresh_status(generation, require_live=True) diff --git a/tests/test_management_containment_cron_inhibition.py b/tests/test_management_containment_cron_inhibition.py index bf6f51939..183d8e825 100644 --- a/tests/test_management_containment_cron_inhibition.py +++ b/tests/test_management_containment_cron_inhibition.py @@ -10,8 +10,10 @@ import json import os import pwd +import shutil import stat import tempfile +from contextlib import contextmanager from dataclasses import asdict, replace from pathlib import Path from types import SimpleNamespace @@ -54,6 +56,35 @@ def _scope(tree, units: tuple[str, ...] = ("cron.service",)) -> cron.CronCgroupS return cron._cgroup_snapshot(root=root, proc=proc, mountinfo=mountinfo, inhibited_units=units) +def _stock_entry(proof: cron.CronInhibitedSource) -> routes.CronEntry: + path = "/etc/cron.daily/apport" + return routes.CronEntry(path, "regular", 1, 2, 3, 0o755, 0, cron.STOCK_INPUTS[path][1], True, inhibited_source=proof) + + +def _recovery_binding(entry: routes.CronEntry) -> tuple[routes.CronEntry, ...]: + return routes._cron_approval_binding((entry,), recovery=True) + + +@contextmanager +def _approved_cron_routes(monkeypatch: pytest.MonkeyPatch, entry: routes.CronEntry, profile: str = "recovery"): + from tests.test_management_containment_route_manifest import _closed_routes, _closed_v2_routes + + fixture_type = _closed_routes if profile == "v1" else _closed_v2_routes + with fixture_type(monkeypatch) as fixture: + fixture.facts["cron.service"] = routes.SystemdRouteFact("cron.service", "a" * 64, "masked", "inactive", 0, True) + fixture.catalog = tuple(sorted(fixture.facts)) + if profile == "recovery": + fixture.ids += ("f" * 64,) + fixture.docker = tuple(SimpleNamespace(container_id=item) for item in fixture.ids) + selected = fixture.approval.enrollment + scope = routes.RecoveryRouteScope(selected.deployment_id, selected.enrollment_id, selected.authorization_id, "ordinary-recovery-v1", fixture.ids, selected.original_files) + fixture.approval = routes.RecoveryRouteSelection(scope, fixture.approval.enrollment_sha256) + fixture.manifest = replace(fixture.manifest, schema="approved-management-recovery-routes-v1", profile=scope.profile, docker_container_ids=fixture.ids) + fixture.manifest = replace(fixture.manifest, cron_entries=(entry,), cron_restarter_paths=(entry.path,), systemd_routes=tuple(sorted((*fixture.manifest.systemd_routes, routes.SystemdRoute("cron.service", "a" * 64, "restarter")), key=lambda item: item.unit))) + fixture.save() + yield fixture + + @pytest.fixture def closed_proof(scope_tree) -> cron.CronInhibitedSource: properties = {name: "" for name in cron._PROPERTIES} @@ -89,6 +120,116 @@ def test_missing_cron_and_user_scopes_are_explicitly_bound(scope_tree) -> None: assert _scope(scope_tree) == first +@pytest.mark.parametrize("change", ("added", "removed", "recreated")) +def test_recovery_approval_allows_only_unrelated_directory_lifecycle(scope_tree, closed_proof, change: str) -> None: + root, _, _ = scope_tree + target = root / "system.slice" / ("docker-" + "a" * 64 + ".scope") + if change != "added": + _write_group(target, processes="1234\n", threads="1234\n", populated="1") + approved = _stock_entry(replace(closed_proof, cgroups=_scope(scope_tree))) + if change != "added": + target.rename(root.parent / "retained-old-target-scope") + if change != "removed": + _write_group(target, processes="1234\n", threads="1234\n", populated="1") + current_scope = _scope(scope_tree) + assert _scope(scope_tree) == current_scope + current = _stock_entry(replace(closed_proof, cgroups=current_scope)) + assert current != approved + assert _recovery_binding(current) == _recovery_binding(approved) + assert routes._cron_approval_binding((current,), recovery=False) != routes._cron_approval_binding((approved,), recovery=False) + assert current.inhibited_source.cgroups == current_scope + + +@pytest.mark.parametrize("relative", ("", "system.slice", "system.slice/cron.service", "system.slice/cron.service/child", "other.slice", "other.slice/cron.service", "user.slice")) +def test_recovery_approval_retains_every_inhibited_directory_and_ancestor(scope_tree, closed_proof, relative: str) -> None: + root, _, _ = scope_tree + for path in ("system.slice/cron.service/child", "other.slice/cron.service", "user.slice"): + _write_group(root / path) + for path in ("system.slice/cron.service", "other.slice"): + _write_group(root / path) + approved = _stock_entry(replace(closed_proof, cgroups=_scope(scope_tree))) + target = root / relative + target.chmod(stat.S_IMODE(target.stat().st_mode) ^ stat.S_IXGRP) + current = _stock_entry(replace(closed_proof, cgroups=_scope(scope_tree))) + assert _recovery_binding(current) != _recovery_binding(approved) + + +@pytest.mark.parametrize("change", ("mount-id", "mount-device", "required-roots", "missing-roots", "empty-groups", "boot", "scheduler", "controller", "vendor", "mask", "inhibited-services", "cron-bytes", "cron-inode", "missing-proof")) +def test_recovery_approval_retains_other_scope_and_source_bindings(closed_proof, change: str) -> None: + approved = _stock_entry(closed_proof) + scope = closed_proof.cgroups + if change == "mount-id": + current = _stock_entry(replace(closed_proof, cgroups=replace(scope, mount_id=scope.mount_id + 1))) + elif change == "mount-device": + current = _stock_entry(replace(closed_proof, cgroups=replace(scope, mount_device="0:999"))) + elif change == "required-roots": + current = _stock_entry(replace(closed_proof, cgroups=replace(scope, required_empty_roots=("/other.slice/cron.service", *scope.required_empty_roots)))) + elif change == "missing-roots": + current = _stock_entry(replace(closed_proof, cgroups=replace(scope, missing_roots=()))) + elif change == "empty-groups": + current = _stock_entry(replace(closed_proof, cgroups=replace(scope, empty_groups=(cron.EmptyCgroupFact("/user.slice", (), (), 0, 0),)))) + elif change == "boot": + current = _stock_entry(replace(closed_proof, boot_id="00000000-0000-4000-8000-000000000001")) + elif change == "scheduler": + current = _stock_entry(replace(closed_proof, scheduler_sha256="b" * 64)) + elif change == "controller": + current = _stock_entry(replace(closed_proof, controller_properties=tuple((key, "system.slice" if key == "Slice" else value) for key, value in closed_proof.controller_properties))) + elif change == "vendor": + current = _stock_entry(replace(closed_proof, sources=(replace(closed_proof.sources[0], inode=99), *closed_proof.sources[1:]))) + elif change == "mask": + current = _stock_entry(replace(closed_proof, mask_identity=(1, 99, *closed_proof.mask_identity[2:]))) + elif change == "inhibited-services": + current = _stock_entry(replace(closed_proof, inhibited_services=(("cron.service", "b" * 64),))) + elif change == "cron-bytes": + current = replace(approved, sha256="f" * 64) + elif change == "cron-inode": + current = replace(approved, inode=99) + else: + current = replace(approved, inhibited_source=None) + assert _recovery_binding(current) != _recovery_binding(approved) + + +@pytest.mark.parametrize("profile", ("recovery", "v1", "v2")) +def test_only_recovery_route_reader_allows_unrelated_scope_and_keeps_full_facts(scope_tree, closed_proof, monkeypatch: pytest.MonkeyPatch, profile: str) -> None: + approved = _stock_entry(closed_proof) + with _approved_cron_routes(monkeypatch, approved, profile) as fixture: + original = fixture.path.read_bytes() + target = scope_tree[0] / "system.slice" / ("docker-" + "a" * 64 + ".scope") + _write_group(target, processes="1234\n", threads="1234\n", populated="1") + + def read_catalog(**kwargs): + return (_stock_entry(replace(closed_proof, cgroups=_scope(scope_tree))),) + + monkeypatch.setattr(routes, "_cron_catalog", read_catalog) + if profile != "recovery": + with pytest.raises(routes.RouteManifestUnavailable, match="changed cron/at route"): + fixture.observe() + else: + observation = fixture.observe() + assert observation.cron_entries == read_catalog() + assert observation.cron_entries != observation.manifest.cron_entries + assert "/system.slice/" + target.name in {item.path for item in observation.cron_entries[0].inhibited_source.cgroups.directories} + assert fixture.path.read_bytes() == original + + +def test_recovery_fresh_route_repeat_still_refuses_unrelated_scope_change(scope_tree, closed_proof, monkeypatch: pytest.MonkeyPatch) -> None: + approved = _stock_entry(closed_proof) + with _approved_cron_routes(monkeypatch, approved) as fixture: + calls = 0 + + def read_catalog(**kwargs): + nonlocal calls + calls += 1 + if calls == 2: + _write_group(scope_tree[0] / "system.slice/docker-new.scope") + return (_stock_entry(replace(closed_proof, cgroups=_scope(scope_tree))),) + + monkeypatch.setattr(routes, "_cron_catalog", read_catalog) + with pytest.raises(routes.RouteManifestUnavailable, match="installed ordinary routes changed during observation"): + fixture.observe() + assert calls == 2 + + @pytest.mark.parametrize("state", ("leader", "thread", "nested", "event-only", "user-session", "alternative-slice")) def test_masked_controller_cannot_hide_real_scope_population(scope_tree, state: str) -> None: root, _, _ = scope_tree @@ -196,6 +337,218 @@ def test_profile_reobserves_complete_scope_and_rejects_races(closed_proof, monke cron.observe_inhibited_stock_cron_scope((unit,)) +def _write_proc_task(proc: Path, *, pid: str = "1089", tid: str = "1089", group: str = "/system.slice/unrelated.service") -> Path: + task = proc / pid / "task" / tid + task.mkdir(parents=True) + (task / "stat").write_text(tid + " (fixture) S " + "0 " * 18 + "777 0\n") + (task / "cgroup").write_text("0::" + group + "\n") + if not (proc / pid / "cmdline").exists(): + (proc / pid / "cmdline").write_bytes(b"fixture\0") + if not (proc / pid / "exe").is_symlink(): + (proc / pid / "exe").symlink_to("/usr/bin/sleep") + return task + + +def _filesystem_cron_observer(scope_tree, closed_proof, monkeypatch: pytest.MonkeyPatch): + """Keep the real paired observer and filesystem scope reader in this component test.""" + root, proc, mountinfo = scope_tree + state = SimpleNamespace(source=cron._CronSourceBinding.from_observation(closed_proof), source_values=[], source_reads=0, scope_reads=0, completed=[], before_scope=lambda: None) + read_scope = cron._cgroup_snapshot + + def source(): + state.source_reads += 1 + return state.source_values.pop(0) if state.source_values else state.source + + def scope(*, inhibited_units): + state.scope_reads += 1 + state.before_scope() + value = read_scope(root=root, proc=proc, mountinfo=mountinfo, inhibited_units=inhibited_units) + state.completed.append(value) + return value + + monkeypatch.setattr(cron, "_source_snapshot", source) + monkeypatch.setattr(cron, "_cgroup_snapshot", scope) + state.facts = (routes.SystemdRouteFact("cron.service", "a" * 64, "masked", "inactive", 0, True),) + return state + + +@pytest.mark.parametrize("incomplete_snapshot", (1, 2)) +def test_disappearing_process_discards_the_entire_observation_pair(scope_tree, closed_proof, monkeypatch: pytest.MonkeyPatch, incomplete_snapshot: int) -> None: + root, proc, _ = scope_tree + _write_proc_task(proc) + state = _filesystem_cron_observer(scope_tree, closed_proof, monkeypatch) + enumerate_directory = Path.iterdir + injected = False + + def disappear(path): + nonlocal injected + if path == proc / "1089/task" and state.scope_reads == incomplete_snapshot and not injected: + injected = True + shutil.rmtree(proc / "1089") + _write_group(root / "system.slice/after-exit.scope") + return enumerate_directory(path) + + monkeypatch.setattr(Path, "iterdir", disappear) + observed = cron.observe_inhibited_stock_cron_scope(state.facts) + assert injected and state.source_reads == state.scope_reads == incomplete_snapshot + 2 + assert state.completed[-2] == state.completed[-1] == observed.cgroups + assert "/system.slice/after-exit.scope" in {item.path for item in observed.cgroups.directories} + if incomplete_snapshot == 2: + assert state.completed[0] != observed.cgroups + + +def test_disappearing_nonleader_is_reobserved_without_skipping_other_tasks(scope_tree, closed_proof, monkeypatch: pytest.MonkeyPatch) -> None: + _, proc, _ = scope_tree + _write_proc_task(proc) + thread = _write_proc_task(proc, tid="1090") + state = _filesystem_cron_observer(scope_tree, closed_proof, monkeypatch) + read = cron._read + injected = False + + def disappear(path, limit=1024 * 1024): + nonlocal injected + if path == thread / "stat" and not injected: + injected = True + shutil.rmtree(thread) + return read(path, limit) + + monkeypatch.setattr(cron, "_read", disappear) + observed = cron.observe_inhibited_stock_cron_scope(state.facts) + assert injected and state.source_reads == state.scope_reads == 3 + assert observed.cgroups == state.completed[-2] == state.completed[-1] + assert (proc / "1089/task/1089/stat").is_file() + + +def test_repeated_process_churn_has_a_finite_complete_observation_budget(scope_tree, closed_proof, monkeypatch: pytest.MonkeyPatch) -> None: + _, proc, _ = scope_tree + state = _filesystem_cron_observer(scope_tree, closed_proof, monkeypatch) + state.before_scope = lambda: _write_proc_task(proc) + enumerate_directory = Path.iterdir + + def disappear(path): + if path == proc / "1089/task": + shutil.rmtree(proc / "1089") + return enumerate_directory(path) + + monkeypatch.setattr(Path, "iterdir", disappear) + with pytest.raises(cron.CronInhibitionUnavailable, match="inventory did not stabilize"): + cron.observe_inhibited_stock_cron_scope(state.facts) + assert state.source_reads == state.scope_reads == 3 + assert not state.completed + + +@pytest.mark.parametrize("fault", ("task-directory", "empty-tasks", "stat", "cgroup", "nonleader-stat", "nonleader-cgroup", "cmdline", "exe", "malformed-stat", "malformed-cgroup", "permission", "eperm")) +def test_live_missing_or_invalid_process_evidence_is_not_retried(scope_tree, closed_proof, monkeypatch: pytest.MonkeyPatch, fault: str) -> None: + _, proc, _ = scope_tree + task = _write_proc_task(proc) + if fault.startswith("nonleader-"): + task = _write_proc_task(proc, tid="1090") + state = _filesystem_cron_observer(scope_tree, closed_proof, monkeypatch) + if fault == "task-directory": + shutil.rmtree(task.parent) + elif fault == "empty-tasks": + shutil.rmtree(task) + elif fault in {"stat", "cgroup", "nonleader-stat", "nonleader-cgroup"}: + (task / fault.removeprefix("nonleader-")).unlink() + elif fault in {"cmdline", "exe"}: + (proc / "1089" / fault).unlink() + elif fault.startswith("malformed-"): + (task / fault.removeprefix("malformed-")).write_text("unavailable\n") + else: + read = cron._read + + def denied(path, limit=1024 * 1024): + if path == task / "stat": + raise PermissionError(1 if fault == "eperm" else 13, "fixture task is inaccessible") + return read(path, limit) + + monkeypatch.setattr(cron, "_read", denied) + with pytest.raises(cron.CronInhibitionUnavailable): + cron.observe_inhibited_stock_cron_scope(state.facts) + assert state.source_reads == state.scope_reads == 1 + + +@pytest.mark.parametrize("kind", ("inhibited-membership", "scheduler")) +def test_positively_observed_inhibited_tasks_are_never_retried(scope_tree, closed_proof, monkeypatch: pytest.MonkeyPatch, kind: str) -> None: + _, proc, _ = scope_tree + _write_proc_task(proc, group="/system.slice/cron.service" if kind == "inhibited-membership" else "/system.slice/unrelated.service") + if kind == "scheduler": + (proc / "1089/exe").unlink() + (proc / "1089/exe").symlink_to("/usr/sbin/cron") + state = _filesystem_cron_observer(scope_tree, closed_proof, monkeypatch) + with pytest.raises(cron.CronScopeRetainsTasks): + cron.observe_inhibited_stock_cron_scope(state.facts) + assert state.source_reads == state.scope_reads == 1 + + +def test_new_nonleader_in_an_inhibited_scope_is_found_by_the_fresh_complete_scan(scope_tree, closed_proof, monkeypatch: pytest.MonkeyPatch) -> None: + _, proc, _ = scope_tree + _write_proc_task(proc) + state = _filesystem_cron_observer(scope_tree, closed_proof, monkeypatch) + enumerate_directory = Path.iterdir + task_reads = 0 + + def add_thread(path): + nonlocal task_reads + if path == proc / "1089/task": + task_reads += 1 + if task_reads == 2: + _write_proc_task(proc, tid="1090", group="/system.slice/cron.service") + return enumerate_directory(path) + + monkeypatch.setattr(Path, "iterdir", add_thread) + with pytest.raises(cron.CronScopeRetainsTasks, match="tid=1090"): + cron.observe_inhibited_stock_cron_scope(state.facts) + assert state.source_reads == state.scope_reads == 2 + assert not state.completed + + +@pytest.mark.parametrize("field", ("boot_id", "controller_properties", "sources", "mask_path", "mask_identity")) +def test_changed_source_binding_cannot_be_hidden_by_a_second_snapshot_disappearance(scope_tree, closed_proof, monkeypatch: pytest.MonkeyPatch, field: str) -> None: + _, proc, _ = scope_tree + _write_proc_task(proc) + state = _filesystem_cron_observer(scope_tree, closed_proof, monkeypatch) + changes = { + "boot_id": "11111111-89ab-cdef-0123-456789abcdef", + "controller_properties": tuple((key, "changed" if key == "FragmentPath" else value) for key, value in state.source.controller_properties), + "sources": (replace(state.source.sources[0], inode=99), *state.source.sources[1:]), + "mask_path": "/run/systemd/system/cron.service", + "mask_identity": (1, 99, 3, stat.S_IFLNK | 0o777, 0, 0), + } + state.source_values = [state.source, replace(state.source, **{field: changes[field]})] + enumerate_directory = Path.iterdir + + def disappear(path): + if path == proc / "1089/task" and state.scope_reads == 2: + shutil.rmtree(proc / "1089") + return enumerate_directory(path) + + monkeypatch.setattr(Path, "iterdir", disappear) + with pytest.raises(cron.CronInhibitionUnavailable, match="source/controller authority changed"): + cron.observe_inhibited_stock_cron_scope(state.facts) + assert state.source_reads == state.scope_reads == 2 + assert len(state.completed) == 1 + + +def test_first_incomplete_snapshot_retains_its_protected_source_refusal_anchor(scope_tree, closed_proof, monkeypatch: pytest.MonkeyPatch) -> None: + _, proc, _ = scope_tree + _write_proc_task(proc) + state = _filesystem_cron_observer(scope_tree, closed_proof, monkeypatch) + state.source_values = [state.source, replace(state.source, mask_identity=(1, 99, 3, stat.S_IFLNK | 0o777, 0, 0))] + enumerate_directory = Path.iterdir + + def disappear(path): + if path == proc / "1089/task" and state.scope_reads == 1: + shutil.rmtree(proc / "1089") + return enumerate_directory(path) + + monkeypatch.setattr(Path, "iterdir", disappear) + with pytest.raises(cron.CronInhibitionUnavailable, match="source/controller authority changed"): + cron.observe_inhibited_stock_cron_scope(state.facts) + assert state.source_reads == state.scope_reads == 2 + assert len(state.completed) == 1 + + def test_vendor_source_read_binds_actual_protected_file_and_refuses_change(native_sources: dict[str, str]) -> None: with tempfile.TemporaryDirectory(prefix="ac-cron-vendor-", dir=pwd.getpwuid(os.geteuid()).pw_dir) as directory: path = Path(directory) / "cron.service" diff --git a/tests/test_management_containment_route_manifest.py b/tests/test_management_containment_route_manifest.py index 789ab74ae..5d17a1f1c 100644 --- a/tests/test_management_containment_route_manifest.py +++ b/tests/test_management_containment_route_manifest.py @@ -39,6 +39,21 @@ def test_systemd_timer_without_service_main_pid_is_fingerprinted(monkeypatch: py assert fact.inhibited is False +def test_generic_docker_restarter_uses_the_enrolled_container_identity(monkeypatch: pytest.MonkeyPatch) -> None: + unit = "auto-coder-recovery-negative-" + "1" * 32 + ".service" + selected = "2" * 64 + shown = f"Id={unit}\nLoadState=loaded\nUnitFileState=static\nActiveState=inactive\nSubState=dead\nFragmentPath=/run/systemd/system/{unit}\nNeedDaemonReload=no\nMainPID=0\nExecStart=/usr/bin/docker start {selected}\n" + source = f"[Service]\nType=oneshot\nExecStart=/usr/bin/docker start {selected}\n" + monkeypatch.setattr(routes, "_command", lambda *args, **kwargs: shown if args[1] == "show" else source) + assert not routes._systemd_unit(unit).management_candidate + assert not routes._systemd_unit(unit, extra_cues=("3" * 64,)).management_candidate + fact = routes._systemd_unit(unit, extra_cues=(selected,)) + assert fact.management_candidate + assert not fact.inhibited + assert fact.main_pid == 0 + assert fact.active_state == "inactive" + + @pytest.mark.parametrize("unit", ("apport-autoreport.path", "multi-user.target", "etc-fstab.mount", "proc-sys-fs-binfmt_misc.automount", "dev-zram0.swap")) def test_other_service_activators_are_fingerprinted_without_main_pid(monkeypatch: pytest.MonkeyPatch, unit: str) -> None: shown = "Id=" + unit + "\nLoadState=loaded\nUnitFileState=enabled\nActiveState=active\nSubState=running\nFragmentPath=/etc/systemd/system/" + unit + "\nNeedDaemonReload=no\n" diff --git a/tests/test_management_cutover_fixture_contracts.py b/tests/test_management_cutover_fixture_contracts.py index 4920a1764..a6e598a65 100644 --- a/tests/test_management_cutover_fixture_contracts.py +++ b/tests/test_management_cutover_fixture_contracts.py @@ -150,6 +150,7 @@ def test_management_ingress_keeps_workers_on_the_internal_network(tmp_path): fixture.oauth.mkdir() fixture.origin = "http://127.0.0.1:8081" fixture.executable_digest = "f" * 64 + fixture.original_genesis = False fixture.image = "sha256:" + "e" * 64 fixture.port = 23456 fixture.worker_network = "private-workers" diff --git a/tests/test_management_cutover_sqlite_lifetimes.py b/tests/test_management_cutover_sqlite_lifetimes.py new file mode 100644 index 000000000..647a439ae --- /dev/null +++ b/tests/test_management_cutover_sqlite_lifetimes.py @@ -0,0 +1,242 @@ +"""Real SQLite lifetimes; supplied Docker metadata is component input only.""" + +from __future__ import annotations + +import fcntl +import gc +import hashlib +import json +import os +import sqlite3 +import stat +import uuid +from contextlib import closing, contextmanager +from dataclasses import replace +from pathlib import Path + +import pytest + +from auto_coder.management import fresh_host_observation as observation +from auto_coder.management.quarantine_history import QuarantineHistory +from tests.test_management_archived_cutover_container import Chain, Generation +from tests.test_management_fresh_initialization import _intent +from tests.test_management_quarantine_history import AUTHORITY, _store + + +@contextmanager +def _without_collection(): + """Expose leaked connection cycles without relying on finalizer timing.""" + enabled = gc.isenabled() + gc.disable() + try: + yield + finally: + if enabled: + gc.enable() + + +def _open_database_handles(database: Path) -> tuple[str, ...]: + selected = {str(database), *(str(database) + suffix for suffix in ("-wal", "-shm", "-journal"))} + result = [] + for descriptor in Path("/proc/self/fd").iterdir(): + try: + target = os.readlink(descriptor) + except FileNotFoundError: + continue + if target.removesuffix(" (deleted)") in selected: + result.append(target) + return tuple(sorted(result)) + + +def _assert_no_sidecars(database: Path) -> None: + assert not _open_database_handles(database) + for suffix in ("-wal", "-shm", "-journal"): + assert not Path(str(database) + suffix).exists() + + +def test_checkpoint_closes_real_sqlite_before_quarantine_inspection(tmp_path: Path) -> None: + journal, old = _store(tmp_path / "original", "original-A", management=True) + before = journal.database.read_bytes() + identity = journal.database.stat() + _assert_no_sidecars(journal.database) + with _without_collection(): + Chain.checkpoint(journal.database) + _assert_no_sidecars(journal.database) + current = journal.database.stat() + assert (current.st_dev, current.st_ino, current.st_uid, stat.S_IMODE(current.st_mode)) == (identity.st_dev, identity.st_ino, identity.st_uid, 0o600) + assert journal.database.read_bytes() == before + with QuarantineHistory((journal,)) as history: + report = history.inspect() + assert (report.inspected_journals, report.historical_keys, report.unresolved_targets) == (1, 1, 1) + assert history.decide(old, AUTHORITY).status == "key_excluded" + assert report.launch_authority is False + _assert_no_sidecars(journal.database) + assert journal.database.read_bytes() == before + + +@pytest.mark.parametrize("reader", ["child", "grants"]) +@pytest.mark.parametrize("missing_table", [False, True]) +def test_fixture_reads_close_connections_on_success_and_error(tmp_path: Path, reader: str, missing_table: bool) -> None: + root = tmp_path / reader + root.mkdir(mode=0o700) + database = root / ("management-child.sqlite3" if reader == "child" else "oauth.sqlite3") + with closing(sqlite3.connect(database)) as producer: + assert producer.execute("PRAGMA journal_mode=WAL").fetchone() == ("wal",) + if reader == "child": + producer.execute("CREATE TABLE " + ("unrelated" if missing_table else "child_state") + " (record_json TEXT)") + producer.execute("INSERT INTO " + ("unrelated" if missing_table else "child_state") + " VALUES (?)", (json.dumps({"phase": "running", "pid": 17}),)) + else: + for table in ("authority", "grants", "access_tokens", "refresh_tokens"): + producer.execute("CREATE TABLE " + ("unrelated" if missing_table and table == "grants" else table) + " (key TEXT)") + producer.commit() + database.chmod(0o600) + before = database.read_bytes() + _assert_no_sidecars(database) + if reader == "child": + generation_root = tmp_path / "generation" + generation_root.mkdir(mode=0o700) + root.rename(generation_root / "state") + database = generation_root / "state" / database.name + generation = Generation("original-A", generation_root, "a" * 64, []) + + def read(): + return Chain.child(generation) + + else: + chain = Chain.__new__(Chain) + chain.oauth = root + read = chain.grants + with _without_collection(): + if missing_table: + with pytest.raises(sqlite3.OperationalError, match="no such table"): + read() + else: + expected = {"phase": "running", "pid": 17} if reader == "child" else {name: [] for name in ("authority", "grants", "access_tokens", "refresh_tokens")} + assert read() == expected + assert not _open_database_handles(database) + assert database.read_bytes() == before + Chain.checkpoint(database) + _assert_no_sidecars(database) + assert database.read_bytes() == before + + +def test_checkpoint_refuses_a_missing_original_without_creating_it(tmp_path: Path) -> None: + database = tmp_path / "missing-original.sqlite3" + with pytest.raises(sqlite3.OperationalError, match="unable to open database file"): + Chain.checkpoint(database) + assert list(tmp_path.iterdir()) == [] + + +def test_failed_checkpoint_closes_the_original_connection(tmp_path: Path) -> None: + database = tmp_path / "corrupt-original.sqlite3" + database.write_bytes(b"retained invalid original database") + database.chmod(0o600) + with _without_collection(): + with pytest.raises(sqlite3.DatabaseError, match="file is not a database"): + Chain.checkpoint(database) + _assert_no_sidecars(database) + assert database.read_bytes() == b"retained invalid original database" + + +@pytest.mark.parametrize("reader", ["capture", "replacement"]) +@pytest.mark.parametrize("missing_table", [False, True]) +def test_actual_host_presence_reader_releases_sqlite_on_success_and_error(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, reader: str, missing_table: bool) -> None: + """Only Docker inspection is supplied; files, process, locks and SQLite are real.""" + intent = replace(_intent(tmp_path), state_root="/management-child-state", journal_root="/management-operations", oauth_root="/management-oauth") + config_root = tmp_path / "bootstrap" + config_root.mkdir(mode=0o700) + config = config_root / "config.json" + config.write_text( + json.dumps( + { + "instance_id": intent.new_instance, + "state_root": intent.state_root, + "journal_root": intent.journal_root, + "oauth_root": intent.oauth_root, + "listener_url": "http://127.0.0.1:8081/mcp", + "initial_runtime": "bootstrap", + "runtimes": [{"reference": "bootstrap", "receipt": "sha256:" + "b" * 64, "executable": "/usr/bin/python3", "arguments": []}], + } + ) + ) + config.chmod(0o600) + lock = tmp_path / "prior-owner.lock" + lock.write_bytes(b"") + lock.chmod(0o600) + intent = replace(intent, bootstrap_config_sha256=hashlib.sha256(config.read_bytes()).hexdigest(), owner_lock_device=lock.stat().st_dev, owner_lock_inode=lock.stat().st_ino) + pid = os.getpid() + token = Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split()[19] + incarnation = str(uuid.uuid4()) + ledger = Path(intent.host_state_root) / "management-child.sqlite3" + with closing(sqlite3.connect(ledger)) as producer: + assert producer.execute("PRAGMA journal_mode=WAL").fetchone() == ("wal",) + table = "unrelated" if missing_table else "supervisor_presence" + producer.execute(f"CREATE TABLE {table}(instance_id TEXT, incarnation TEXT, pid INTEGER, start_token TEXT)") + producer.execute(f"INSERT INTO {table} VALUES (?, ?, ?, ?)", (intent.new_instance, incarnation, 1, token)) + producer.commit() + ledger.chmod(0o600) + mounts = [ + (intent.host_state_root, intent.state_root, True), + (intent.host_journal_root, intent.journal_root, True), + (intent.host_oauth_root, intent.oauth_root, True), + (str(config_root), "/bootstrap", False), + (str(tmp_path / "guard"), "/management-initialization", False), + (str(lock), "/management-prior-owner.lock", False), + ] + values = [ + intent.new_container_id, + intent.image_digest, + True, + pid, + "running", + False, + "", + None, + None, + "bridge", + None, + [{"Type": "bind", "Source": source, "Destination": destination, "RW": writable} for source, destination, writable in mounts], + True, + "original-start", + 0, + ["python"], + [], + {"8080/tcp": [{"HostIp": "127.0.0.1", "HostPort": str(intent.host_listener_port)}]}, + ] + inspected = [] + + def selected_inspection(container_id): + assert container_id == intent.new_container_id + inspected.append(container_id) + return values + + monkeypatch.setattr(observation, "_inspect", selected_inspection) + before = ledger.read_bytes() + identity = ledger.stat() + with lock.open("rb") as fence, _without_collection(): + fcntl.flock(fence, fcntl.LOCK_SH | fcntl.LOCK_NB) + + def read(): + if reader == "replacement": + return observation.assert_replacement_owner(intent, incarnation, pid, fence.fileno(), tmp_path / "guard") + prior = observation.PriorContainer("fixture-engine", intent.new_container_id, intent.image_digest, "original-start", 0, pid, token, "pid:[1]", "net:[2]", "bridge", "a" * 64, "/fixture", 1, 1, (observation.PriorTask(pid, token),)) + return observation._capture_supervisor_identity(prior, values, tuple(mounts), config) + + if missing_table: + with pytest.raises((sqlite3.OperationalError, observation.HostObservationUnavailable)) as error: + read() + cause = error.value if reader == "capture" else error.value.__cause__ + assert isinstance(cause, sqlite3.OperationalError) and "no such table" in str(cause) + else: + result = read() + if reader == "capture": + assert result.instance_id == intent.new_instance and result.pid == pid and result.start_token == token + else: + assert result is None and len(inspected) == 2 + assert not _open_database_handles(ledger) + after = ledger.stat() + assert (after.st_dev, after.st_ino, after.st_uid, stat.S_IMODE(after.st_mode)) == (identity.st_dev, identity.st_ino, identity.st_uid, 0o600) + assert ledger.read_bytes() == before + Chain.checkpoint(ledger) + _assert_no_sidecars(ledger) + assert ledger.read_bytes() == before diff --git a/tests/test_management_original_genesis.py b/tests/test_management_original_genesis.py new file mode 100644 index 000000000..7bb8f6414 --- /dev/null +++ b/tests/test_management_original_genesis.py @@ -0,0 +1,432 @@ +"""Original genesis filesystem and native-store components. + +Docker observations and container bind mapping are explicit component substitutes. +These tests do not claim real-container, first-child or recovery-epoch acceptance. +""" + +from __future__ import annotations + +import fcntl +import hashlib +import json +import os +import sqlite3 +import uuid +from contextlib import closing +from dataclasses import asdict, replace +from pathlib import Path + +import pytest + +from auto_coder.management import original_genesis as producer +from auto_coder.management.bootstrap_supervisor import BootstrapConfig, ManagementBootstrapSupervisor +from auto_coder.management.containment_observation import DockerObservation +from auto_coder.management.fresh_initialization import claim_fresh_initialization +from auto_coder.management.original_attempt_evidence import derive_lineage, observe, verify_lineage +from auto_coder.management.original_attempts import OriginalEvidenceUnavailable, canonical, read_frontier, source_identity +from tests.test_management_original_attempts import _covered, _fingerprint + + +@pytest.fixture +def _use_real_commands(): + pass + + +@pytest.fixture +def _use_real_sleep(): + pass + + +def _write(path, value): + path.write_bytes(canonical(value)) + path.chmod(0o600) + return path + + +@pytest.fixture +def root_authorization(tmp_path, monkeypatch): + # Only the host privilege capability is substituted in this same-UID + # component. Every original file, inode, exclusive create and fsync is real. + monkeypatch.setattr(producer, "_require_host_root", lambda: None) + tmp_path.chmod(0o700) + bootstrap = tmp_path / "bootstrap" + bootstrap.mkdir(mode=0o700) + runtime = {"reference": "bootstrap", "receipt": "sha256:" + "a" * 64, "executable": "/opt/auto-coder/.venv/bin/auto-coder", "arguments": ["management", "serve", "--config", "/bootstrap/management.yaml"]} + _write( + bootstrap / "config.json", + { + "instance_id": "genesis-instance", + "state_root": "/management-child-state", + "journal_root": "/management-operations", + "oauth_root": "/management-oauth", + "listener_url": "http://127.0.0.1:8081/mcp", + "initial_runtime": "bootstrap", + "runtimes": [runtime], + "original_genesis": "/bootstrap/original-genesis.json", + }, + ) + _write( + bootstrap / "management.yaml", + { + "state_root": "/management-oauth", + "operation_state_root": "/management-operations", + "bootstrap_config_path": "/bootstrap/config.json", + "public_origin": "https://management.example.test", + "firebase_project_id": "component-project", + "clients": [{"client_id": "component-client", "redirect_uris": ["https://client.example.test/callback"]}], + "bind_host": "127.0.0.1", + "port": 8081, + }, + ) + proxy = _write(tmp_path / "Caddyfile", {"component": "protected-proxy-config"}) + request = producer.GenesisRootAuthorization( + producer.ROOTS_PROFILE, "separately-authorized-new-deployment", "component-deployment", str(uuid.uuid4()), str(bootstrap / "config.json"), str(bootstrap / "management.yaml"), str(proxy), *(str(tmp_path / role) for role in ("state", "journal", "oauth", "history")) + ) + path = _write(tmp_path / "root-authorization.json", asdict(request)) + return request, path + + +def _capture_component(root_authorization, monkeypatch, tmp_path): + request, authorization = root_authorization + roots_path = producer.prepare_original_genesis_roots(authorization) + selection = producer.GenesisSelection(producer.PROFILE, request.purpose, request.deployment_id, request.authorization_id, "a" * 64, "b" * 64, request.history_root, request.bootstrap_path, request.management_path, request.proxy_config_path, str(roots_path)) + final_authorization = _write(tmp_path / "capture-authorization.json", asdict(selection)) + manager = DockerObservation("component-engine", selection.management_id, "sha256:" + "c" * 64, "d" * 64, "no", 0, False, "created", 0, "0001-01-01T00:00:00Z", "0001-01-01T00:00:00Z", 0, "e" * 64) + proxy = replace(manager, container_id=selection.proxy_id, image_id="sha256:" + "f" * 64) + mounts = [{"Type": "bind", "Source": str(Path(request.bootstrap_path).parent), "Destination": "/bootstrap", "RW": False}] + mounts += [ + {"Type": "bind", "Source": getattr(request, role + "_root"), "Destination": destination, "RW": role != "history"} + for role, destination in (("state", "/management-child-state"), ("journal", "/management-operations"), ("oauth", "/management-oauth"), ("history", "/management-original-history")) + ] + monkeypatch.setattr(producer, "_created_pair", lambda _selection: ((manager, proxy), {"Mounts": mounts}, source_identity())) + anchor = producer.capture_original_genesis(final_authorization) + return anchor, final_authorization + + +@pytest.fixture +def captured(root_authorization, monkeypatch, tmp_path): + return _capture_component(root_authorization, monkeypatch, tmp_path) + + +def _native_supervisor(anchor, monkeypatch): + """Map the tested container-bind boundary onto real isolated local roots.""" + from auto_coder.management import oauth_config + + mapped = replace(anchor, roots=tuple(replace(root, container_path=root.host_path) for root in anchor.roots)) + original_file = producer._file + original_settings = oauth_config.load_management_settings + original_history = producer.validate_history_genesis + bootstrap = Path(anchor.file("bootstrap").path).parent + + def mapped_file(role, path): + if path.parent == Path("/bootstrap"): + path = bootstrap / path.name + return original_file(role, path) + + def mapped_history(root, *, mounted_genesis=None): + if mounted_genesis is not None: + return mapped + with monkeypatch.context() as current: + current.setattr(producer, "HISTORY_MOUNT", Path("/management-original-history")) + return original_history(root) + + def mapped_settings(path): + settings = original_settings(path) + oauth_root = Path(anchor.root("oauth").host_path) + return replace(settings, state_root=oauth_root, oauth=replace(settings.oauth, database_path=oauth_root / "oauth.sqlite3"), operation_state_root=Path(anchor.root("journal").host_path), bootstrap_config_path=bootstrap / "config.json") + + monkeypatch.setattr(oauth_config, "load_management_settings", mapped_settings) + monkeypatch.setattr(producer, "_file", mapped_file) + monkeypatch.setattr(producer, "HISTORY_MOUNT", Path(anchor.binding.root)) + monkeypatch.setattr(producer, "validate_history_genesis", mapped_history) + config = BootstrapConfig.read(bootstrap / "config.json") + # Map only the executable pathname; the actual receipt and wrapper parser + # still verify this real console file before any native store is created. + launcher = bootstrap / "component-console.py" + launcher.write_text( + "#!/opt/auto-coder/.venv/bin/python\n" + "import sys\n" + "from auto_coder.cli import main\n" + "if __name__ == '__main__':\n" + " if sys.argv[0].endswith('-script.pyw'):\n" + " sys.argv[0] = sys.argv[0][:-11]\n" + " elif sys.argv[0].endswith('.exe'):\n" + " sys.argv[0] = sys.argv[0][:-4]\n" + " sys.exit(main())\n" + ) + runtime = replace(config.runtimes[0], executable=launcher, receipt="sha256:" + hashlib.sha256(launcher.read_bytes()).hexdigest()) + config = replace(config, state_root=Path(anchor.root("state").host_path), journal_root=Path(anchor.root("journal").host_path), oauth_root=Path(anchor.root("oauth").host_path), runtimes=(runtime,)) + return ManagementBootstrapSupervisor(config, bootstrap_config_path=bootstrap / "config.json") + + +def _native_component(anchor, monkeypatch): + supervisor = _native_supervisor(anchor, monkeypatch) + with supervisor._owner_file() as owner: + fcntl.flock(owner.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + result = producer.prepare_original_genesis(supervisor, supervisor._await_owner_start_token()) + assert result is not None + assert producer.prepare_original_genesis(supervisor, supervisor._await_owner_start_token()) is None + # The native creators use WAL. This component explicitly settles its own + # stopped test stores before an immutable reader; the verifier never writes. + for path in producer.native_paths(anchor, host=True).values(): + with closing(sqlite3.connect(path)) as connection: + connection.execute("PRAGMA wal_checkpoint(TRUNCATE)") + return supervisor, result + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("state_root", "/wrong-oauth"), + ("operation_state_root", "/wrong-operations"), + ("operation_state_root", None), + ("bootstrap_config_path", "/wrong-bootstrap/config.json"), + ("bootstrap_config_path", None), + ("bind_host", "0.0.0.0"), + ("port", 8082), + ], +) +def test_genesis_capture_refuses_service_using_unrecorded_stores_or_listener(root_authorization, monkeypatch, tmp_path, field, value): + request, _ = root_authorization + management = Path(request.management_path) + settings = json.loads(management.read_bytes()) + settings[field] = value + _write(management, settings) + with pytest.raises(OriginalEvidenceUnavailable, match="effective management"): + _capture_component(root_authorization, monkeypatch, tmp_path) + assert not (management.parent / producer.GENESIS_NAME).exists() + assert all(list(Path(getattr(request, role + "_root")).iterdir()) == [] for role in ("state", "journal", "oauth", "history")) + + +@pytest.mark.parametrize("change", ["receipt", "python-hook", "loader-hook"]) +def test_native_genesis_refuses_unbound_execution_before_original_store_creation(captured, monkeypatch, change): + anchor, _ = captured + supervisor = _native_supervisor(anchor, monkeypatch) + runtime = supervisor.config.runtimes[0] + if change == "receipt": + runtime = replace(runtime, receipt="sha256:" + "f" * 64) + else: + runtime = replace(runtime, environment=(("PYTHONPATH" if change == "python-hook" else "LD_PRELOAD", "/unselected"),)) + supervisor.config = replace(supervisor.config, runtimes=(runtime,)) + with supervisor._owner_file() as owner: + fcntl.flock(owner.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + with pytest.raises(ValueError, match="console receipt differs|environment contains code hooks"): + producer.prepare_original_genesis(supervisor, supervisor._await_owner_start_token()) + assert {path.name for path in supervisor.config.state_root.iterdir()} == {".supervisor-owner.lock"} + assert list(supervisor.config.journal_root.iterdir()) == [] + assert list(supervisor.config.oauth_root.iterdir()) == [] + + +def test_original_roots_are_created_not_inferred_from_emptiness(root_authorization, tmp_path): + request, authorization = root_authorization + Path(request.state_root).mkdir(mode=0o700) + before = _fingerprint(tmp_path) + with pytest.raises(OriginalEvidenceUnavailable, match="no empty-root conversion"): + producer.prepare_original_genesis_roots(authorization) + assert _fingerprint(tmp_path) == before + assert not (Path(request.bootstrap_path).parent / producer.ROOT_CLAIM_NAME).exists() + assert not Path(request.history_root).exists() + + +@pytest.mark.parametrize("boundary", [producer.ROOT_CLAIM_NAME, producer.ROOTS_NAME]) +def test_interrupted_original_root_publication_is_never_reenrolled(root_authorization, tmp_path, monkeypatch, boundary): + request, authorization = root_authorization + original_create = producer._create + + def interrupted(root, name, value): + original_create(root, name, value) + if name == boundary: + raise OSError("interrupted original root publication") + + monkeypatch.setattr(producer, "_create", interrupted) + with pytest.raises(OSError, match="interrupted original"): + producer.prepare_original_genesis_roots(authorization) + monkeypatch.setattr(producer, "_create", original_create) + before = _fingerprint(tmp_path) + with pytest.raises((FileExistsError, OriginalEvidenceUnavailable)): + producer.prepare_original_genesis_roots(authorization) + assert _fingerprint(tmp_path) == before + + +def test_captured_genesis_pins_affirmative_roots_exact_empty_history_and_configs(captured, tmp_path): + anchor, authorization = captured + authority, events = read_frontier(Path(anchor.binding.root), anchor.binding) + assert authority.profile == producer.HISTORY_PROFILE + assert authority.genesis_path == str(Path(anchor.file("bootstrap").path).parent / producer.GENESIS_NAME) + assert events == [] + assert producer.validate_history_genesis(Path(anchor.binding.root)) == anchor + assert {root.role for root in anchor.roots} == {"bootstrap", "state", "journal", "oauth", "history"} + assert {item.role for item in anchor.files} == {"authorization", "root-authorization", "root-claim", "roots", "bootstrap", "management", "proxy"} + before = _fingerprint(tmp_path) + with pytest.raises(OriginalEvidenceUnavailable, match="must be empty|already captured"): + producer.capture_original_genesis(authorization) + assert _fingerprint(tmp_path) == before + for role in ("state", "journal", "oauth"): + assert list(Path(anchor.root(role).host_path).iterdir()) == [] + + +@pytest.mark.parametrize("role", ["bootstrap", "management", "proxy", "root-authorization", "root-claim", "roots", "authorization"]) +def test_genesis_refuses_changed_original_file_without_repair(captured, tmp_path, role): + anchor, _ = captured + original = Path(anchor.file(role).path) + original.write_bytes(original.read_bytes() + b" ") + before = _fingerprint(tmp_path) + with pytest.raises((OriginalEvidenceUnavailable, ValueError)): + read_frontier(Path(anchor.binding.root), anchor.binding) + assert _fingerprint(tmp_path) == before + + +def test_genesis_first_successor_cannot_select_unrelated_baseline(captured, tmp_path): + anchor, _ = captured + intent = _covered(tmp_path, anchor.binding, "foreign-child") + before = _fingerprint(tmp_path) + with pytest.raises(OriginalEvidenceUnavailable, match="uncovered, refused or cyclic"): + claim_fresh_initialization(Path(intent.host_state_root), intent) + assert _fingerprint(tmp_path) == before + assert read_frontier(Path(anchor.binding.root), anchor.binding)[1] == [] + + +def test_native_genesis_unknown_is_complete_evidence_but_missing_store_is_not(captured, tmp_path, monkeypatch): + anchor, _ = captured + _, launch = _native_component(anchor, monkeypatch) + native, first = producer.validate_native_genesis(anchor, Path(anchor.file("bootstrap").path).parent / producer.GENESIS_NAME, host=True) + assert (first["operation_id"], first["attempt_id"]) == launch + assert set(native["stores"]) == {"child", "handoff", "recovery", "operations"} + owner_path = Path(anchor.root("state").host_path) / ".supervisor-owner.lock" + with owner_path.open("rb") as owner: + fcntl.flock(owner.fileno(), fcntl.LOCK_SH | fcntl.LOCK_NB) + with observe(anchor.binding) as history: + evidence = derive_lineage(anchor.binding, history, (owner.fileno(),)) + assert evidence.unavailable == () + assert len(evidence.generations) == 1 + assert evidence.generations[0].outcome == "unknown" + assert verify_lineage(evidence, anchor.binding, history, (owner.fileno(),)) == evidence + before = _fingerprint(tmp_path) + with pytest.raises(OriginalEvidenceUnavailable, match="forged, stale, truncated"): + verify_lineage(replace(evidence, generations=()), anchor.binding, history, (owner.fileno(),)) + assert _fingerprint(tmp_path) == before + database = Path(anchor.root("journal").host_path) / "management-operations.sqlite3" + hidden = database.with_name("removed-original") + database.rename(hidden) + missing = derive_lineage(anchor.binding, history, (owner.fileno(),)) + assert missing.unavailable and any("management-operations.sqlite3" in reason for reason in missing.unavailable) + assert hidden.read_bytes() == before[str(database.relative_to(tmp_path))][2] + + +def test_genesis_full_frontier_retains_original_failed_branch(captured, tmp_path, monkeypatch): + anchor, _ = captured + _native_component(anchor, monkeypatch) + intent = _covered(tmp_path, anchor.binding, "refused-successor") + intent = replace(intent, old_instance=anchor.instance_id, old_container_id=anchor.management_id, oauth_inode=intent.oauth_inode + 1) + with pytest.raises(ValueError, match="retained OAuth root identity differs"): + claim_fresh_initialization(Path(intent.host_state_root), intent) + owner_path = Path(anchor.root("state").host_path) / ".supervisor-owner.lock" + with owner_path.open("rb") as owner: + fcntl.flock(owner.fileno(), fcntl.LOCK_SH | fcntl.LOCK_NB) + with observe(anchor.binding) as history: + evidence = derive_lineage(anchor.binding, history, (owner.fileno(),)) + assert evidence.unavailable == () + assert [(item.instance_id, item.outcome, item.owner_inode > 0) for item in evidence.generations] == [(anchor.instance_id, "unknown", True), (intent.new_instance, "never-created", False)] + assert evidence.attempts[0].request_id == intent.request_id + assert evidence.attempts[0].state == "never-created" + assert verify_lineage(evidence, anchor.binding, history, (owner.fileno(),)) == evidence + + +@pytest.mark.parametrize("missing", [producer.NATIVE_NAME, producer.LAUNCH_NAME]) +def test_interrupted_genesis_native_boundary_cannot_reopen_first_owner(captured, monkeypatch, tmp_path, missing): + anchor, _ = captured + supervisor, _ = _native_component(anchor, monkeypatch) + path = supervisor.config.state_root / missing + path.rename(path.with_suffix(".missing")) + before = _fingerprint(tmp_path) + with pytest.raises(FileExistsError): + with supervisor._owner_file(): + pytest.fail("an incomplete native boundary reopened the original owner") + assert _fingerprint(tmp_path) == before + + +def test_original_genesis_selector_does_not_enable_fresh_or_new(captured): + anchor, _ = captured + config = BootstrapConfig.read(Path(anchor.file("bootstrap").path)) + for changed in (replace(config, new_instance_initialization=True), replace(config, fresh_guard_path=Path("/guard"), fresh_prior_lock_path=Path("/prior"))): + with pytest.raises(ValueError, match="explicitly selected ordinary"): + changed.validate() + assert replace(config, original_genesis=None).original_genesis is None + + +def test_complete_unknown_successor_is_distinct_from_missing_original_claim(captured, tmp_path, monkeypatch): + """Actual original writers/stores; only the host grant transport is substituted.""" + from auto_coder.management import fresh_initialization + from auto_coder.management.bootstrap_store import BootstrapStore + from auto_coder.management.operation_store import ManagementOperationStore + from auto_coder.management.original_attempts import grant_boundary, record_original_stores + from auto_coder.management.recovery_mailbox import RecoveryMailbox + from auto_coder.management.self_update_handoff import HandoffStore + + anchor, _ = captured + _native_component(anchor, monkeypatch) + initial = _covered(tmp_path, anchor.binding, "unknown-successor") + prior_owner_path = Path(anchor.root("state").host_path) / ".supervisor-owner.lock" + prior_info = prior_owner_path.stat() + oauth = anchor.root("oauth") + intent = replace( + initial, + old_instance=anchor.instance_id, + old_container_id=anchor.management_id, + state_root=initial.host_state_root, + journal_root=initial.host_journal_root, + oauth_root=oauth.host_path, + host_oauth_root=oauth.host_path, + oauth_device=oauth.device, + oauth_inode=oauth.inode, + owner_lock_device=prior_info.st_dev, + owner_lock_inode=prior_info.st_ino, + ) + state, journal = Path(intent.host_state_root), Path(intent.host_journal_root) + claim_fresh_initialization(state, intent) + + def transport(_socket, **request): + request.pop("trusted_uid", None) + return {**request, "challenge": "a" * 64} + + monkeypatch.setattr(fresh_initialization, "request_fresh_host_grant", transport) + # This is the production original-admission boundary, not an assertion that + # a real host grant or child was delivered in this component test. + with grant_boundary(intent): + fresh_initialization.request_and_claim_first_launch(state, tmp_path / "unused.sock", owner_incarnation=str(uuid.uuid4())) + owner_path = state / ".supervisor-owner.lock" + owner_path.touch(mode=0o600) + with owner_path.open("rb") as native_owner: + fcntl.flock(native_owner.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + previous_umask = os.umask(0o077) + try: + with closing(BootstrapStore(state, intent.new_instance)._connect()): + pass + with closing(HandoffStore(state, intent.new_instance)._connect()): + pass + with closing(ManagementOperationStore(journal)._connect()): + pass + RecoveryMailbox(state, intent.new_instance).initialize() + record_original_stores(state, intent) + finally: + os.umask(previous_umask) + for database in (*state.glob("*.sqlite3"), *journal.glob("*.sqlite3")): + with closing(sqlite3.connect(database)) as connection: + connection.execute("PRAGMA wal_checkpoint(TRUNCATE)") + with prior_owner_path.open("rb") as prior_owner, owner_path.open("rb") as owner: + for stream in (prior_owner, owner): + fcntl.flock(stream.fileno(), fcntl.LOCK_SH | fcntl.LOCK_NB) + fences = (prior_owner.fileno(), owner.fileno()) + with observe(anchor.binding) as history: + evidence = derive_lineage(anchor.binding, history, fences) + assert evidence.unavailable == () + assert [(item.instance_id, item.outcome) for item in evidence.generations] == [(anchor.instance_id, "unknown"), (intent.new_instance, "unknown")] + assert verify_lineage(evidence, anchor.binding, history, fences) == evidence + original = _fingerprint(tmp_path) + claim = state / "fresh-launch-claimed.json" + claim.rename(state / "removed-original-claim") + missing = derive_lineage(anchor.binding, history, fences) + assert missing.unavailable and any("fresh-launch-claimed.json" in reason for reason in missing.unavailable) + assert (state / "removed-original-claim").read_bytes() == original[str(claim.relative_to(tmp_path))][2] + with pytest.raises(OriginalEvidenceUnavailable): + verify_lineage(missing, anchor.binding, history, fences) diff --git a/tests/test_management_original_genesis_capture.py b/tests/test_management_original_genesis_capture.py new file mode 100644 index 000000000..e9b31adca --- /dev/null +++ b/tests/test_management_original_genesis_capture.py @@ -0,0 +1,67 @@ +"""Ordinary genesis predecessors preserve the existing strict Caddy capture.""" + +from __future__ import annotations + +import os +from pathlib import Path + +import pytest + +from auto_coder.management import fresh_host_observation as observation +from auto_coder.management.exclusion_host import inherited_support +from auto_coder.management.original_genesis import HISTORY_MOUNT +from tests.test_management_original_genesis import captured, root_authorization # noqa: F401 + + +def test_original_genesis_predecessor_retains_its_exact_readonly_history(captured) -> None: # noqa: F811 + anchor, _ = captured + mounts = [{"Type": "bind", "Source": item.host_path, "Destination": item.container_path, "RW": item.role in {"state", "journal", "oauth"}} for item in anchor.roots] + assert inherited_support(mounts) == ((), {str(HISTORY_MOUNT)}) + mounts[-1]["RW"] = True + with pytest.raises(ValueError): + inherited_support(mounts) + + +def test_original_genesis_predecessor_cannot_replace_only_history_mount(captured, tmp_path: Path) -> None: # noqa: F811 + anchor, _ = captured + alternate = tmp_path / "alternate" + alternate.mkdir(mode=0o700) + mounts = [{"Type": "bind", "Source": item.host_path if item.role != "history" else str(alternate), "Destination": item.container_path, "RW": item.role in {"state", "journal", "oauth"}} for item in anchor.roots] + with pytest.raises(ValueError, match="mounted root differs"): + inherited_support(mounts) + + +@pytest.fixture +def caddy_profile(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + config = tmp_path / "Caddyfile" + config.write_bytes(b"http://127.0.0.1:8080 {}") + paths = [config] + for name in ("data", "config"): + path = tmp_path / name + path.mkdir(mode=0o700) + paths.append(path) + mounts = [{"Type": "bind", "Source": str(path), "Destination": destination, "RW": index > 0} for index, (path, destination) in enumerate(zip(paths, ("/etc/caddy/Caddyfile", "/data", "/config")))] + identifier, image = "a" * 64, "sha256:" + "b" * 64 + values = [identifier, image, True, os.getpid(), "running", False, "", None, None, "container:" + "c" * 64, None, mounts, "unused", "started", 0, ["caddy"], ["run", "--config", "/etc/caddy/Caddyfile", "--adapter", "caddyfile"]] + marker = object() + monkeypatch.setattr(observation, "_engine_id", lambda: "component-engine") + monkeypatch.setattr(observation, "_inspect", lambda _: values) + monkeypatch.setattr(observation, "_capture_prior_kernel", lambda *_args, **_kwargs: marker) + return identifier, image, values, marker + + +def test_original_genesis_proxy_uses_the_existing_single_config_capture(caddy_profile) -> None: + identifier, image, values, marker = caddy_profile + values[11] = values[11][:1] + expected = tuple((item["Source"], item["Destination"], item["RW"]) for item in values[11]) + assert observation.capture_prior_container(identifier, image, expected_mounts=expected) is marker + + +@pytest.mark.parametrize("extra", ["data", "config", "both"]) +def test_ordinary_predecessor_capture_does_not_admit_stock_data_mounts(caddy_profile, extra: str) -> None: + identifier, image, values, _ = caddy_profile + if extra != "both": + values[11] = [item for item in values[11] if item["Destination"] in {"/etc/caddy/Caddyfile", "/" + extra}] + expected = tuple((item["Source"], item["Destination"], item["RW"]) for item in values[11]) + with pytest.raises(observation.HostObservationUnavailable): + observation.capture_prior_container(identifier, image, expected_mounts=expected) diff --git a/tests/test_management_original_genesis_cli.py b/tests/test_management_original_genesis_cli.py new file mode 100644 index 000000000..fbe5f04e1 --- /dev/null +++ b/tests/test_management_original_genesis_cli.py @@ -0,0 +1,36 @@ +"""Actual CLI refusal boundaries, separate from Docker/genesis acceptance.""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.usefixtures("_use_real_commands", "_use_real_sleep") + + +def test_selected_genesis_missing_state_root_is_not_recreated_by_diagnostic_logging(tmp_path: Path) -> None: + state = tmp_path / "missing-original-state" + configuration = tmp_path / "config.json" + configuration.write_text( + json.dumps( + { + "instance_id": "original-component", + "state_root": str(state), + "oauth_root": str(tmp_path / "oauth"), + "journal_root": str(tmp_path / "journal"), + "listener_url": "http://127.0.0.1:8081/mcp", + "initial_runtime": "must-not-launch", + "runtimes": [{"reference": "must-not-launch", "receipt": "sha256:" + "a" * 64, "executable": "/opt/auto-coder/.venv/bin/auto-coder", "arguments": ["management", "serve", "--config", "/bootstrap/management.yaml"]}], + "original_genesis": "/bootstrap/original-genesis.json", + } + ) + ) + configuration.chmod(0o600) + result = subprocess.run([sys.executable, "-B", "-m", "auto_coder.management.bootstrap_supervisor", "--config", str(configuration)], capture_output=True, text=True, timeout=15) + assert result.returncode != 0 + assert not state.exists(), "diagnostics recreated a missing selected original state root before refusal" + assert not (tmp_path / "journal").exists() and not (tmp_path / "oauth").exists() diff --git a/tests/test_management_recovery_containment.py b/tests/test_management_recovery_containment.py new file mode 100644 index 000000000..05cc0b4a9 --- /dev/null +++ b/tests/test_management_recovery_containment.py @@ -0,0 +1,436 @@ +"""Bounded component tests; native closed-host acceptance is a separate harness.""" + +from __future__ import annotations + +import hashlib +import io +import json +import os +import pwd +import stat +import subprocess +import tarfile +import tempfile +from dataclasses import asdict, replace +from pathlib import Path + +import pytest + +from auto_coder.management import original_attempts +from auto_coder.management import recovery_containment as observer +from auto_coder.management import recovery_source_profile as source +from auto_coder.management.containment_enrollment import OriginalNetworkPin +from auto_coder.management.containment_route_manifest import DeploymentRootBinding, RecoveryRouteScope, RecoveryRouteSelection, _route_container_ids +from auto_coder.management.recovery_containment_model import RECOVERY_PROFILE, NamedDescriptor, RecoveryContainmentUnavailable, RecoveryEnrollment, RecoveryFile, RecoveryLock, RecoveryOriginal, RecoveryResources, RecoveryTarget, RecoveryWatcher +from auto_coder.management.recovery_containment_profile import RecoveryManagerFact, _ordinary_mount_effects, assert_disjoint_target + +pytestmark = pytest.mark.usefixtures("_use_real_commands", "_use_real_sleep") +_BOOT = "00000000-0000-4000-8000-000000000001" + + +@pytest.fixture +def _use_real_commands(): + yield + + +@pytest.fixture +def _use_real_sleep(): + yield + + +def _file(path: str, inode: int) -> RecoveryFile: + return RecoveryFile(path, "d" * 64, 1, inode) + + +def _original(index: int = 0, *, started: bool = True) -> RecoveryOriginal: + manager, proxy = format(2 * index + 1, "064x"), format(2 * index + 2, "064x") + base = "/private/generation-" + str(index) + mounts = (DeploymentRootBinding(manager, "/management-child-state", base + "/state", True, 1, 100 + index, stat.S_IFDIR | 0o700), DeploymentRootBinding(proxy, "/etc/caddy/Caddyfile", base + "/Caddyfile", False, 1, 200 + index, stat.S_IFREG | 0o600)) + owner = RecoveryLock(base + "/state/.supervisor-owner.lock", 1, 1000 + index) if started else None + prior = _file(base + "/prior-evidence.json", 2000 + index) if started else None + pin = OriginalNetworkPin(42, "12345", _BOOT, 10 + index, "net:[" + str(3000 + index) + "]", 4, 3000 + index) if started else None + return RecoveryOriginal( + f"generation-{index:02d}", + f"instance-{index:02d}", + manager, + "sha256:" + "a" * 64, + proxy, + "sha256:" + "b" * 64, + "started" if started else "never-started", + _BOOT if started else "", + _file(base + "/bootstrap/config.json", 4000 + index), + _file(base + "/bootstrap/management.yml", 5000 + index), + _file(base + "/Caddyfile", 200 + index), + mounts, + owner, + prior, + pin, + ) + + +def _enrollment(*, generations: int = 1, watchers: int = 1, started: bool = True) -> RecoveryEnrollment: + target_id = "f" * 64 + target = RecoveryTarget( + "target-instance", + target_id, + "sha256:" + "a" * 64, + _file("/private/target/bootstrap/config.json", 9000), + _file("/private/target/bootstrap/management.yml", 9001), + (DeploymentRootBinding(target_id, "/management-child-state", "/private/target/state", True, 1, 9002, stat.S_IFDIR | 0o700),), + RecoveryLock("/private/target/state/.supervisor-owner.lock", 1, 9003), + 18081, + ) + controls = tuple( + RecoveryWatcher( + f"auto-coder-management-proxy-watch-{index:02d}.service", + _file(f"/private/watcher-{index}/config.json", 10000 + index), + _file("/private/package/management/proxy_watch_cli.py", 20000), + _file(f"/run/systemd/system/auto-coder-management-proxy-watch-{index:02d}.service", 40000 + index), + RecoveryLock(f"/private/watcher-{index}/state/proxy-watch.lock", 1, 30000 + index), + ) + for index in range(watchers) + ) + return RecoveryEnrollment("approved-management-recovery-containment-v1", "deployment", "enrollment", "authorization", RECOVERY_PROFILE, "local-engine", "sha256:" + "c" * 64, tuple(_original(index, started=started) for index in range(generations)), target, controls, True) + + +def _json(value: object) -> object: + return json.loads(json.dumps(asdict(value))) + + +def test_scope_round_trip_preserves_all_members_and_fixed_profile(): + expected = _enrollment(generations=3, watchers=3) + actual = RecoveryEnrollment.parse(_json(expected)) + assert actual == expected + assert actual.container_ids == tuple(sorted([expected.target.management_id, *(identifier for item in expected.originals for identifier in (item.management_id, item.proxy_id))])) + assert len(actual.container_ids) == 7 + assert actual.approved_observation_only is True + + +@pytest.mark.parametrize("field,value", [("schema", "approved-current-containment-enrollment-v3"), ("profile", "ordinary-v1"), ("approved_observation_only", False), ("source_identity", "same-as-before"), ("deployment_id", "../other"), ("engine_id", "")]) +def test_scope_refuses_foreign_legacy_or_authorizing_schema(field, value): + raw = _json(_enrollment()) + raw[field] = value + with pytest.raises(RecoveryContainmentUnavailable): + RecoveryEnrollment.parse(raw) + + +def test_scope_refuses_unknown_fields_and_container_aliases(): + raw = _json(_enrollment()) + raw["trusted_root"] = "/request/override" + with pytest.raises(RecoveryContainmentUnavailable, match="schema"): + RecoveryEnrollment.parse(raw) + raw.pop("trusted_root") + raw["target"]["management_id"] = raw["originals"][0]["management_id"] + raw["target"]["mounts"][0]["container_id"] = raw["target"]["management_id"] + with pytest.raises(RecoveryContainmentUnavailable, match="alias"): + RecoveryEnrollment.parse(raw) + + +def test_scope_refuses_shared_proxy_identity_between_generations(): + raw = _json(_enrollment(generations=2)) + old_proxy = raw["originals"][0]["proxy_id"] + raw["originals"][1]["proxy_id"] = old_proxy + raw["originals"][1]["mounts"][1]["container_id"] = old_proxy + raw["originals"][1]["mounts"].sort(key=lambda item: (item["container_id"], item["destination"])) + with pytest.raises(RecoveryContainmentUnavailable, match="alias"): + RecoveryEnrollment.parse(raw) + + +def test_never_started_scope_cannot_invent_owner_or_historical_pin(): + selected = _enrollment(started=False) + actual = RecoveryEnrollment.parse(_json(selected)) + assert actual.originals[0].owner is None + assert actual.originals[0].prior_evidence is None + assert actual.originals[0].namespace_pin is None + raw = _json(selected) + raw["originals"][0]["owner"] = asdict(RecoveryLock("/private/fabricated.lock", 1, 2)) + with pytest.raises(RecoveryContainmentUnavailable, match="invent"): + RecoveryEnrollment.parse(raw) + + +def test_scope_requires_original_boot_and_real_namespace_capture_fields(): + raw = _json(_enrollment()) + raw["originals"][0]["boot_id"] = "00000000-0000-4000-8000-000000000002" + with pytest.raises(RecoveryContainmentUnavailable, match="boot"): + RecoveryEnrollment.parse(raw) + raw = _json(_enrollment()) + raw["originals"][0]["namespace_pin"] = None + with pytest.raises(ValueError, match="namespace pin"): + RecoveryEnrollment.parse(raw) + + +@pytest.mark.parametrize("kind", ["owner-path", "owner-inode", "controller-owner", "unsorted"]) +def test_scope_rejects_ambiguous_resource_authority(kind): + raw = _json(_enrollment(generations=2, watchers=2)) + if kind == "owner-path": + raw["originals"][1]["owner"]["path"] = raw["originals"][0]["owner"]["path"] + elif kind == "owner-inode": + raw["originals"][1]["owner"]["inode"] = raw["originals"][0]["owner"]["inode"] + elif kind == "controller-owner": + raw["watchers"][0]["lock"]["inode"] = raw["target"]["owner"]["inode"] + else: + raw["originals"].reverse() + with pytest.raises(RecoveryContainmentUnavailable): + RecoveryEnrollment.parse(raw) + + +def test_complete_peer_resource_bound_is_checked_before_observation(): + boundary = _enrollment(generations=16, watchers=29) + assert RecoveryEnrollment.parse(_json(boundary)) == boundary + with pytest.raises(RecoveryContainmentUnavailable, match="complete peer resource bound"): + RecoveryEnrollment.parse(_json(_enrollment(generations=16, watchers=30))) + + +def test_named_descriptors_are_exact_sorted_and_have_no_aliases(): + expected = RecoveryResources(42, "12345", _BOOT, (NamedDescriptor("generation-00", 10),), (NamedDescriptor("watcher.service", 11),), (NamedDescriptor("generation-00", 12),)) + assert RecoveryResources.parse(_json(expected)) == expected + raw = _json(expected) + raw["namespace_fds"][0]["descriptor"] = 10 + with pytest.raises(RecoveryContainmentUnavailable, match="alias"): + RecoveryResources.parse(raw) + raw = _json(expected) + raw["controller_fds"][0]["descriptor"] = True + with pytest.raises(RecoveryContainmentUnavailable, match="descriptor"): + RecoveryResources.parse(raw) + + +def test_protected_fixture_load_rejects_unknown_fields_duplicates_and_permissions(): + with tempfile.TemporaryDirectory(prefix="recovery-enrollment-component-", dir=pwd.getpwuid(os.geteuid()).pw_dir) as directory: + root = Path(directory) + path = root / "enrollment.json" + selected = _enrollment() + path.write_text(json.dumps(asdict(selected))) + path.chmod(0o600) + actual = observer._load_enrollment("enrollment", root=root, expected_uid=os.geteuid()) + assert actual.enrollment == selected + assert actual.device == path.stat().st_dev and actual.inode == path.stat().st_ino + assert actual.sha256 == hashlib.sha256(path.read_bytes()).hexdigest() + path.chmod(0o644) + with pytest.raises(ValueError, match="protected"): + observer._load_enrollment("enrollment", root=root, expected_uid=os.geteuid()) + path.chmod(0o600) + content = json.dumps(asdict(selected)) + path.write_text(content[:-1] + ',"profile":"ordinary-recovery-v1"}') + with pytest.raises(ValueError, match="duplicate"): + observer._load_enrollment("enrollment", root=root, expected_uid=os.geteuid()) + + +def test_recovery_route_selection_uses_every_member_root_and_configuration(): + selected = _enrollment(generations=3, watchers=3) + approved = observer.RecoveryEnrollmentObservation(selected, "e" * 64, 1, 2, 3) + scope = observer.route_selection(approved) + assert _route_container_ids(scope) == selected.container_ids + assert {item.path for item in scope.enrollment.original_files} >= {item.bootstrap_config.path for item in selected.originals} | {selected.target.bootstrap_config.path} | {item.configuration.path for item in selected.watchers} + forged = RecoveryRouteSelection(RecoveryRouteScope("d", "e", "a", "ordinary-v1", selected.container_ids, ()), "e" * 64) + with pytest.raises(ValueError, match="selection"): + _route_container_ids(forged) + + +def _manager_fact(selected: RecoveryOriginal | RecoveryTarget, *, state: str, journal: str, oauth: str, port: int) -> RecoveryManagerFact: + return RecoveryManagerFact( + selected.instance_id, selected.instance_id, selected.management_id, selected.management_image_id, "component-only", "sha256:" + "c" * 64, "d" * 64, selected.bootstrap_config.sha256, selected.management_config.sha256, state, journal, oauth, selected.owner, port, selected.mounts + ) + + +def test_target_retains_original_oauth_and_separates_state_journal_and_listener(): + selected = _enrollment() + old = _manager_fact(selected.originals[0], state="/private/old/state", journal="/private/old/journal", oauth="/private/oauth", port=18080) + target = _manager_fact(selected.target, state="/private/target/state", journal="/private/target/journal", oauth="/private/oauth", port=18081) + assert_disjoint_target(selected, (old, target)) + with pytest.raises(RecoveryContainmentUnavailable, match="OAuth"): + assert_disjoint_target(selected, (old, replace(target, oauth_root="/private/copied-oauth"))) + with pytest.raises(RecoveryContainmentUnavailable, match="overlaps"): + assert_disjoint_target(selected, (old, replace(target, state_root="/private/old/state/child"))) + with pytest.raises(RecoveryContainmentUnavailable, match="listener"): + assert_disjoint_target(selected, (old, replace(target, published_port=18080))) + + +def test_mount_effects_reject_undeclared_writable_or_interpreter_inputs(): + config = {"state_root": "/management-child-state", "journal_root": "/management-journal", "oauth_root": "/management-oauth", "runtimes": []} + selected = _enrollment().target.mounts[0] + _ordinary_mount_effects(config, (selected,), target=True) + for destination, writable in (("/unselected-data", True), ("/etc/ld.so.preload", False), ("/management-recovery", True)): + with pytest.raises(RecoveryContainmentUnavailable, match="undeclared"): + _ordinary_mount_effects(config, (replace(selected, destination=destination, writable=writable),), target=True) + + +def _archive(entries: list[tuple[str, bytes, bytes | None]]) -> bytes: + output = io.BytesIO() + with tarfile.open(fileobj=output, mode="w") as archive: + for name, data, kind in entries: + info = tarfile.TarInfo(name) + info.size = len(data) + if kind is not None: + info.type = kind + info.linkname = "auto_coder/__init__.py" + archive.addfile(info, io.BytesIO(data)) + return output.getvalue() + + +def _sources() -> list[tuple[str, bytes, bytes | None]]: + return [ + ("auto_coder/__init__.py", b"", None), + ("auto_coder/management/bootstrap_supervisor.py", b"# supervisor\n", None), + ("auto_coder/management/original_attempts.py", b"# original\n", None), + ("auto_coder/management.py", b"# flat name\n", None), + ("auto_coder/management/a.py", b"# nested name\n", None), + ] + + +def test_complete_source_archive_matches_existing_source_identity_framing(tmp_path, monkeypatch): + package = tmp_path / "auto_coder" + for name, content, _ in _sources(): + path = tmp_path / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(content) + monkeypatch.setattr(original_attempts, "__file__", str(package / "management" / "original_attempts.py")) + assert source._source_archive_identity(_archive(list(reversed(_sources())))) == original_attempts.source_identity() + + +@pytest.mark.parametrize( + "name,kind", + [ + ("auto_coder/__pycache__/__init__.cpython-312.pyc", None), + ("auto_coder/original.pyo", None), + ("auto_coder/native.cpython-312-x86_64-linux-gnu.so", None), + ("auto_coder/native.pyd", None), + ("auto_coder/link.py", tarfile.SYMTYPE), + ("auto_coder/hard.py", tarfile.LNKTYPE), + ("auto_coder/../outside.py", None), + ("/auto_coder/absolute.py", None), + ("auto_coder//ambiguous.py", None), + ("auto_coder/__init__.py", None), + ], +) +def test_source_archive_rejects_alternate_executable_or_ambiguous_members(name, kind): + with pytest.raises(ValueError): + source._source_archive_identity(_archive(_sources() + [(name, b"unsafe", kind)])) + + +def test_source_archive_rejects_missing_history_and_bounded_content(monkeypatch): + with pytest.raises(ValueError, match="incomplete"): + source._source_archive_identity(_archive([("auto_coder/__init__.py", b"", None)])) + monkeypatch.setattr(source, "_MAX_SOURCE", 4) + with pytest.raises(ValueError, match="bound"): + source._source_archive_identity(_archive(_sources())) + + +def test_source_reader_uses_exact_local_object_without_execution(monkeypatch): + calls = [] + archive = _archive(_sources()) + + def command(arguments, **options): + calls.append((arguments, options)) + if arguments[1] == "context": + return subprocess.CompletedProcess(arguments, 0, '"unix:///var/run/docker.sock"', "") + return subprocess.CompletedProcess(arguments, 0, archive, b"") + + monkeypatch.delenv("DOCKER_HOST", raising=False) + monkeypatch.delenv("DOCKER_CONTEXT", raising=False) + monkeypatch.setattr(source.subprocess, "run", command) + assert source.read_image_source_identity("a" * 64) == source._source_archive_identity(archive) + assert calls[1][0] == ("docker", "cp", "a" * 64 + ":/opt/auto-coder/.venv/lib/python3.12/site-packages/auto_coder", "-") + assert not any("exec" in arguments for arguments, _ in calls) + monkeypatch.setenv("DOCKER_HOST", "unix:///unselected.sock") + with pytest.raises(ValueError, match="override"): + source.read_image_source_identity("a" * 64) + assert len(calls) == 2 + + +@pytest.mark.parametrize("mutation", ["grant", "partial", "hash", "unknown"]) +def test_saved_report_cannot_supply_missing_or_authorizing_evidence(monkeypatch, mutation): + raw = { + "schema": "current-recovery-containment-v1", + "status": "observed-current-recovery-containment", + "observation_id": _BOOT, + "observer_pid": 42, + "observer_start_token": "12345", + "boot_id": _BOOT, + "started_monotonic_ns": 1, + "finished_monotonic_ns": 2, + "facts": {}, + "evidence_sha256": observer.digest({}), + "grant_issued": False, + "history_asserted": False, + "admission_created": False, + } + if mutation == "grant": + raw["grant_issued"] = True + elif mutation == "partial": + raw.pop("facts") + elif mutation == "hash": + raw["facts"] = {"forged": True} + else: + raw["repair_history"] = True + monkeypatch.setattr(observer, "observe", lambda *_: pytest.fail("malformed prior report reached current observation")) + with pytest.raises(RecoveryContainmentUnavailable): + observer.verify("enrollment", raw, RecoveryResources(42, "12345", _BOOT, (), (), ())) + + +def _consumer_scope() -> tuple[RecoveryEnrollment, bytes]: + selected = _enrollment() + original, target = selected.originals[0], selected.target + assert original.owner is not None and original.namespace_pin is not None + mounts = ( + *target.mounts, + DeploymentRootBinding(target.management_id, "/management-recovery", "/var/lib/auto-coder/recovery-epoch-peer", False, 1, 50000, stat.S_IFDIR | 0o700), + DeploymentRootBinding(target.management_id, "/management-recovery-history/history.lock", "/private/history/history.lock", False, 1, 50001, stat.S_IFREG | 0o600), + DeploymentRootBinding(target.management_id, "/management-recovery-originals/0/owner.lock", original.owner.path, False, original.owner.device, original.owner.inode, stat.S_IFREG | 0o600), + ) + selected = replace(selected, target=replace(target, mounts=tuple(sorted(mounts, key=lambda item: item.destination)))) + payload = { + "schema_version": 1, + "request_id": "request", + "authorization_id": selected.authorization_id, + "target_id": "target", + "issuer_uid": 0, + "socket_directory_device": 1, + "socket_directory_inode": 50000, + "original_fences": [{"name": "history", "path": "/management-recovery-history/history.lock", "device": 1, "inode": 50001}, {"name": "owner-0", "path": "/management-recovery-originals/0/owner.lock", "device": original.owner.device, "inode": original.owner.inode}], + "target_owner": {"name": "target-owner", "device": target.owner.device, "inode": target.owner.inode}, + "transferred_resources": [ + {"name": "epoch", "kind": "transferred-ex", "device": 1, "inode": 50002}, + {"name": "controller-0", "kind": "transferred-ex", "device": selected.watchers[0].lock.device, "inode": selected.watchers[0].lock.inode}, + {"name": "namespace-0", "kind": "network-namespace", "device": original.namespace_pin.device, "inode": original.namespace_pin.inode}, + ], + } + return selected, json.dumps(payload).encode() + + +def test_target_consumer_inputs_bind_the_exact_socket_owner_history_and_resources(): + selected, encoded = _consumer_scope() + observer._consumer_custody_profile(selected, encoded) + + +@pytest.mark.parametrize("mutation", ["socket-source", "socket-writable", "socket-inode", "owner-source", "owner-inode", "extra-owner", "history-writable", "controller-inode", "namespace-inode", "omitted-epoch"]) +def test_target_consumer_custody_inputs_refuse_substitution_or_omission(mutation): + selected, encoded = _consumer_scope() + payload = json.loads(encoded) + mounts = list(selected.target.mounts) + positions = {item.destination: index for index, item in enumerate(mounts)} + if mutation == "socket-source": + index = positions["/management-recovery"] + mounts[index] = replace(mounts[index], source="/private/other-issuer") + elif mutation == "socket-writable": + index = positions["/management-recovery"] + mounts[index] = replace(mounts[index], writable=True) + elif mutation == "socket-inode": + payload["socket_directory_inode"] += 1 + elif mutation == "owner-source": + index = positions["/management-recovery-originals/0/owner.lock"] + mounts[index] = replace(mounts[index], source="/private/unrelated.lock") + elif mutation == "owner-inode": + payload["original_fences"][1]["inode"] += 1 + elif mutation == "extra-owner": + mounts.append(replace(mounts[positions["/management-recovery-originals/0/owner.lock"]], destination="/management-recovery-originals/1/owner.lock")) + elif mutation == "history-writable": + index = positions["/management-recovery-history/history.lock"] + mounts[index] = replace(mounts[index], writable=True) + elif mutation == "controller-inode": + payload["transferred_resources"][1]["inode"] += 1 + elif mutation == "namespace-inode": + payload["transferred_resources"][2]["inode"] += 1 + else: + payload["transferred_resources"].pop(0) + with pytest.raises(ValueError): + observer._consumer_custody_profile(replace(selected, target=replace(selected.target, mounts=tuple(mounts))), json.dumps(payload).encode()) diff --git a/tests/test_management_recovery_containment_resources.py b/tests/test_management_recovery_containment_resources.py new file mode 100644 index 000000000..dbbaa7657 --- /dev/null +++ b/tests/test_management_recovery_containment_resources.py @@ -0,0 +1,362 @@ +"""Real kernel resource observations, not a complete Docker/host acceptance. + +The typed enrollment is a component selector. Its files and actual holders are +real; no kernel, process, ancestor-protection or flock observer is replaced. +Full production enrollment/profile/controller closure belongs to other tests. +""" + +from __future__ import annotations + +import fcntl +import hashlib +import inspect +import json +import os +import pwd +import select +import subprocess +import sys +import tempfile +from contextlib import contextmanager +from dataclasses import dataclass, replace +from functools import wraps +from pathlib import Path +from typing import Callable, Iterator, ParamSpec + +import pytest + +from auto_coder.management.containment_enrollment import OriginalNetworkPin +from auto_coder.management.recovery_containment_model import NamedDescriptor, RecoveryContainmentUnavailable, RecoveryEnrollment, RecoveryFile, RecoveryLock, RecoveryOriginal, RecoveryResources, RecoveryTarget, RecoveryWatcher +from auto_coder.management.recovery_containment_resources import hold_observed_resources + +pytestmark = pytest.mark.usefixtures("_use_real_commands", "_use_real_sleep") + +_PARAMETERS = ParamSpec("_PARAMETERS") +_CASE_PARAMETER_SETS: dict[str, tuple[tuple[tuple[str, str], ...], ...]] = { + "test_real_independent_resource_holder_is_observed_without_observer_flock_authority": ((),), + "test_inherited_open_file_description_is_not_independent_issuer_acquisition": tuple((("mode", value),) for value in ("inherited-original", "inherited-controller")), + "test_real_resource_change_at_observed_barrier_refuses_and_closes_observer_references": tuple( + (("change", value),) for value in ("unlock-original", "unlock-controller", "close-original", "close-controller", "close-namespace", "replace-original", "replace-controller", "replace-namespace", "holder-death", "original-pin-closed", "owner-path-replaced") + ), + "test_required_resource_group_cannot_be_omitted": tuple((("field", value),) for value in ("original_owner_fds", "controller_fds", "namespace_fds")), + "test_holder_lifetime_binding_cannot_name_another_process_instance": ((("field", "holder_start_token"), ("value", "1")), (("field", "holder_boot_id"), ("value", "00000000-0000-0000-0000-000000000000"))), +} +_RESOURCE_ASSERTIONS: dict[str, Callable[..., None]] = {} +_CASE_RECEIPT = "RECOVERY_RESOURCE_CASE_COMPLETED " + + +def _selected_resource_case(raw: object) -> tuple[str, dict[str, str]]: + if not isinstance(raw, dict) or set(raw) != {"case", "parameters"}: + raise ValueError("resource component invocation is malformed") + name, parameters = raw["case"], raw["parameters"] + if not isinstance(name, str) or name not in _CASE_PARAMETER_SETS or not isinstance(parameters, dict) or any(not isinstance(key, str) or not isinstance(value, str) for key, value in parameters.items()): + raise ValueError("resource component invocation is not selected") + if tuple(sorted(parameters.items())) not in _CASE_PARAMETER_SETS[name]: + raise ValueError("resource component parameters are not selected") + return name, parameters + + +def _read_host_namespaces() -> None: + for namespace in ("pid", "user", "mnt", "net"): + os.readlink("/proc/1/ns/" + namespace) + + +def _native_resource_case(function: Callable[_PARAMETERS, None]) -> Callable[_PARAMETERS, None]: + """Run the identical real assertions with CI's explicitly available sudo. + + The parent pytest node remains responsible for its result and source receipt. + Local test execution never escalates automatically, and an unavailable host + component is a failure rather than evidence of native VM acceptance. + """ + _RESOURCE_ASSERTIONS[function.__name__] = function + + @wraps(function) + def run(*args: _PARAMETERS.args, **kwargs: _PARAMETERS.kwargs) -> None: + try: + _read_host_namespaces() + except OSError as exc: + if os.environ.get("GITHUB_ACTIONS") != "true": + pytest.fail("Actual host namespace access is unavailable for this component test; run it explicitly with the required host permissions. This is not native VM acceptance: " + str(exc)) + parameters = inspect.signature(function).bind(*args, **kwargs).arguments + name, selected = _selected_resource_case({"case": function.__name__, "parameters": parameters}) + invocation = json.dumps({"case": name, "parameters": selected}, sort_keys=True) + root = Path(__file__).resolve().parents[1] + command = ["sudo", "-n", sys.executable, "-I", "-B", str(Path(__file__).resolve()), "--resource-case", invocation] + try: + completed = subprocess.run(command, cwd=root, capture_output=True, text=True, timeout=60, check=False) + except (OSError, subprocess.TimeoutExpired) as failure: + pytest.fail("Genuine privileged resource component execution is unavailable: " + str(failure)) + assert completed.returncode == 0, "Genuine privileged resource assertions did not complete:\n" + completed.stdout + completed.stderr + receipts = [json.loads(line.removeprefix(_CASE_RECEIPT)) for line in completed.stdout.splitlines() if line.startswith(_CASE_RECEIPT)] + assert len(receipts) == 1, "The selected resource assertions did not produce their completion receipt" + receipt = receipts[0] + assert receipt == {"case": name, "parameters": selected, "uid": 0, "source": str(Path(__file__).resolve())} + return + function(*args, **kwargs) + + return run + + +def _run_resource_case(arguments: list[str]) -> None: + if len(arguments) != 2 or arguments[0] != "--resource-case": + raise ValueError("only an explicitly selected resource component case can run") + name, parameters = _selected_resource_case(json.loads(arguments[1])) + if os.geteuid() != 0: + raise RuntimeError("the privileged resource component child did not obtain root") + expected_source = Path(__file__).resolve().parents[1] / "src/auto_coder/management/recovery_containment_resources.py" + if Path(inspect.getsourcefile(inspect.unwrap(hold_observed_resources)) or "").resolve() != expected_source: + raise RuntimeError("the resource component child imported another checkout") + _read_host_namespaces() + _RESOURCE_ASSERTIONS[name](**parameters) + print(_CASE_RECEIPT + json.dumps({"case": name, "parameters": parameters, "uid": os.geteuid(), "source": str(Path(__file__).resolve())}, sort_keys=True), flush=True) + + +_HOLDER = r""" +import fcntl,json,os,sys +from pathlib import Path +paths, inherited = map(json.loads,sys.argv[1:]) +held={} +for name,operation in (('original',fcntl.LOCK_SH),('controller',fcntl.LOCK_EX)): + if name in inherited: + held[name]=inherited[name] + else: + held[name]=os.open(paths[name],os.O_RDONLY|os.O_NOFOLLOW|os.O_CLOEXEC) + fcntl.flock(held[name],operation|fcntl.LOCK_NB) +held['namespace']=os.open('/proc/self/ns/net',os.O_RDONLY|os.O_CLOEXEC) +print(json.dumps({'pid':os.getpid(),'start':Path('/proc/self/stat').read_text().rsplit(')',1)[1].split()[19],'boot':Path('/proc/sys/kernel/random/boot_id').read_text().strip(),'descriptors':held}),flush=True) +for command in sys.stdin: + command=command.strip() + if command=='exit': break + if command.startswith('unlock-'): + fcntl.flock(held[command.removeprefix('unlock-')],fcntl.LOCK_UN) + elif command.startswith('close-'): + os.close(held[command.removeprefix('close-')]) + elif command.startswith('replace-'): + name=command.removeprefix('replace-') + previous=held[name] + os.close(previous) + replacement=os.open(paths['other'],os.O_RDONLY|os.O_NOFOLLOW) + if replacement!=previous: + os.dup2(replacement,previous) + os.close(replacement) + else: raise RuntimeError('unsupported holder barrier') + print(command,flush=True) +""" + + +@dataclass +class ResourceCase: + root: Path + process: subprocess.Popen[str] + enrollment: RecoveryEnrollment + resources: RecoveryResources + original_namespace: int + paths: dict[str, Path] + parent_descriptors: list[int] + + def command(self, value: str) -> None: + assert self.process.stdin is not None + self.process.stdin.write(value + "\n") + self.process.stdin.flush() + assert line(self.process) == value + + +def line(process: subprocess.Popen[str]) -> str: + assert process.stdout is not None + assert select.select([process.stdout], [], [], 10)[0], "real resource holder did not reach its barrier" + result = process.stdout.readline().strip() + assert result, (process.poll(), process.stderr.read() if process.poll() is not None and process.stderr is not None else "") + return result + + +def bound(path: Path) -> RecoveryFile: + info = path.stat() + return RecoveryFile(str(path), hashlib.sha256(path.read_bytes()).hexdigest(), info.st_dev, info.st_ino) + + +def selected_lock(path: Path) -> RecoveryLock: + info = path.stat() + return RecoveryLock(str(path), info.st_dev, info.st_ino) + + +@contextmanager +def resource_case(mode: str = "independent") -> Iterator[ResourceCase]: + # Existing containment tests use the real account home to keep every actual + # ancestor protected; pytest's temporary root may live below writable /tmp. + with tempfile.TemporaryDirectory(prefix="ac-recovery-resources-", dir=pwd.getpwuid(os.geteuid()).pw_dir) as directory: + root = Path(directory) + paths = {name: root / name for name in ("original", "controller", "target", "other", "configuration")} + parent_fds: list[int] = [] + process = None + try: + for path in paths.values(): + descriptor = os.open(path, os.O_RDWR | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600) + os.close(descriptor) + inherited = {} + if mode.startswith("inherited-"): + name = mode.removeprefix("inherited-") + descriptor = os.open(paths[name], os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC) + parent_fds.append(descriptor) + fcntl.flock(descriptor, (fcntl.LOCK_SH if name == "original" else fcntl.LOCK_EX) | fcntl.LOCK_NB) + inherited[name] = descriptor + namespace = os.open("/proc/self/ns/net", os.O_RDONLY | os.O_CLOEXEC) + parent_fds.append(namespace) + process = subprocess.Popen([sys.executable, "-I", "-B", "-c", _HOLDER, json.dumps({name: str(path) for name, path in paths.items()}), json.dumps(inherited)], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, pass_fds=tuple(inherited.values())) + observed = json.loads(line(process)) + descriptors = observed["descriptors"] + resources = RecoveryResources(observed["pid"], observed["start"], observed["boot"], (NamedDescriptor("generation-one", descriptors["original"]),), (NamedDescriptor("recovery-watch.service", descriptors["controller"]),), (NamedDescriptor("generation-one", descriptors["namespace"]),)) + namespace_info = os.fstat(namespace) + parent_start = Path("/proc/self/stat").read_text().rsplit(")", 1)[1].split()[19] + pin = OriginalNetworkPin(holder_pid=os.getpid(), holder_start_token=parent_start, holder_boot_id=observed["boot"], holder_fd=namespace, namespace=os.readlink(f"/proc/self/fd/{namespace}"), device=namespace_info.st_dev, inode=namespace_info.st_ino) + configuration = bound(paths["configuration"]) + original = RecoveryOriginal("generation-one", "original-one", "a" * 64, "sha256:" + "b" * 64, "c" * 64, "sha256:" + "d" * 64, "started", observed["boot"], configuration, configuration, configuration, (), selected_lock(paths["original"]), configuration, pin) + target = RecoveryTarget("target-one", "e" * 64, "sha256:" + "f" * 64, configuration, configuration, (), selected_lock(paths["target"]), 18080) + unit = root / "recovery-watch.service" + unit.write_bytes(b"# Resource-observation fixture; no executable controller.\n") + unit.chmod(0o600) + watcher = RecoveryWatcher("recovery-watch.service", configuration, configuration, bound(unit), selected_lock(paths["controller"])) + enrollment = RecoveryEnrollment("approved-management-recovery-containment-v1", "deployment-one", "enrollment-one", "authorization-one", "ordinary-recovery-v1", "engine-one", "sha256:" + "0" * 64, (original,), target, (watcher,), True) + yield ResourceCase(root, process, enrollment, resources, namespace, paths, parent_fds) + finally: + if process is not None: + if process.poll() is None: + try: + assert process.stdin is not None + process.stdin.write("exit\n") + process.stdin.flush() + except BrokenPipeError: + pass + try: + process.communicate(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.communicate(timeout=5) + for descriptor in parent_fds: + try: + os.close(descriptor) + except OSError: + pass + + +@_native_resource_case +def test_real_independent_resource_holder_is_observed_without_observer_flock_authority() -> None: + with resource_case() as case: + before = set(os.listdir("/proc/self/fd")) + with hold_observed_resources(case.enrollment, case.resources, expected_uid=os.geteuid()) as observation: + assert observation.holder_pid == case.process.pid + assert [(item.name, item.kind) for item in observation.resources] == [("generation-one", "original-sh"), ("recovery-watch.service", "controller-ex"), ("generation-one", "network-namespace")] + locks = [item.kernel_lock for item in observation.resources if item.kernel_lock is not None] + assert [(lock.pid, lock.mode) for lock in locks] == [(case.process.pid, "READ"), (case.process.pid, "WRITE")] + contenders = [os.open(case.paths[name], os.O_RDONLY | os.O_NOFOLLOW) for name in ("original", "controller")] + try: + for descriptor in contenders: + with pytest.raises(BlockingIOError): + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + finally: + for descriptor in contenders: + os.close(descriptor) + assert set(os.listdir("/proc/self/fd")) == before + # The observer did not acquire an independent lock or release the owner. + case.command("unlock-original") + case.command("unlock-controller") + for name in ("original", "controller"): + descriptor = os.open(case.paths[name], os.O_RDONLY | os.O_NOFOLLOW) + try: + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + finally: + os.close(descriptor) + + +@pytest.mark.parametrize("mode", ["inherited-original", "inherited-controller"]) +@_native_resource_case +def test_inherited_open_file_description_is_not_independent_issuer_acquisition(mode: str) -> None: + with resource_case(mode) as case: + before = set(os.listdir("/proc/self/fd")) + with pytest.raises(RecoveryContainmentUnavailable, match="independently acquired"): + with hold_observed_resources(case.enrollment, case.resources, expected_uid=os.geteuid()): + pytest.fail("an inherited lock was accepted as independent custody") + assert set(os.listdir("/proc/self/fd")) == before + + +@pytest.mark.parametrize("change", ["unlock-original", "unlock-controller", "close-original", "close-controller", "close-namespace", "replace-original", "replace-controller", "replace-namespace", "holder-death", "original-pin-closed", "owner-path-replaced"]) +@_native_resource_case +def test_real_resource_change_at_observed_barrier_refuses_and_closes_observer_references(change: str) -> None: + with resource_case() as case: + before = len(os.listdir("/proc/self/fd")) + entered = injected = False + with pytest.raises((ValueError, OSError)): + with hold_observed_resources(case.enrollment, case.resources, expected_uid=os.geteuid()): + entered = True + if change == "holder-death": + case.process.kill() + case.process.wait(timeout=5) + elif change == "original-pin-closed": + os.close(case.original_namespace) + case.parent_descriptors.remove(case.original_namespace) + elif change == "owner-path-replaced": + case.paths["original"].rename(case.root / "retained-owner") + descriptor = os.open(case.paths["original"], os.O_RDWR | os.O_CREAT | os.O_EXCL, 0o600) + os.close(descriptor) + else: + case.command(change) + injected = True + assert entered, "the resource observation failed before the real mutation barrier" + assert injected, "the intended real resource mutation did not complete" + expected = before - int(change == "original-pin-closed") + assert len(os.listdir("/proc/self/fd")) == expected + + +@pytest.mark.parametrize("field", ["original_owner_fds", "controller_fds", "namespace_fds"]) +@_native_resource_case +def test_required_resource_group_cannot_be_omitted(field: str) -> None: + with resource_case() as case: + if field == "original_owner_fds": + changed = replace(case.resources, original_owner_fds=()) + elif field == "controller_fds": + changed = replace(case.resources, controller_fds=()) + else: + assert field == "namespace_fds" + changed = replace(case.resources, namespace_fds=()) + with pytest.raises(RecoveryContainmentUnavailable, match="membership"): + with hold_observed_resources(case.enrollment, changed, expected_uid=os.geteuid()): + pytest.fail("missing required resource group was accepted") + + +@pytest.mark.parametrize("field,value", [("holder_start_token", "1"), ("holder_boot_id", "00000000-0000-0000-0000-000000000000")]) +@_native_resource_case +def test_holder_lifetime_binding_cannot_name_another_process_instance(field: str, value: str) -> None: + with resource_case() as case: + if field == "holder_start_token": + changed = replace(case.resources, holder_start_token=value) + else: + assert field == "holder_boot_id" + changed = replace(case.resources, holder_boot_id=value) + with pytest.raises(RecoveryContainmentUnavailable, match="changed or exited"): + with hold_observed_resources(case.enrollment, changed, expected_uid=os.geteuid()): + pytest.fail("another lifetime was accepted") + + +@pytest.mark.parametrize( + "invocation", + [ + {}, + {"case": "arbitrary", "parameters": {}}, + {"case": "test_real_independent_resource_holder_is_observed_without_observer_flock_authority", "parameters": {"unexpected": "value"}}, + {"case": "test_required_resource_group_cannot_be_omitted", "parameters": {"field": "arbitrary"}}, + {"case": "test_holder_lifetime_binding_cannot_name_another_process_instance", "parameters": {"field": "holder_start_token", "value": "00000000-0000-0000-0000-000000000000"}}, + ], +) +def test_privileged_component_dispatch_rejects_unselected_case_or_parameters(invocation: object) -> None: + with pytest.raises(ValueError, match="resource component"): + _selected_resource_case(invocation) + + +def test_privileged_component_dispatch_covers_every_real_assertion() -> None: + assert set(_RESOURCE_ASSERTIONS) == set(_CASE_PARAMETER_SETS) + for name, selections in _CASE_PARAMETER_SETS.items(): + for parameters in selections: + assert _selected_resource_case({"case": name, "parameters": dict(parameters)}) == (name, dict(parameters)) + + +if __name__ == "__main__": + _run_resource_case(sys.argv[1:]) diff --git a/tests/test_management_recovery_epoch_authorization.py b/tests/test_management_recovery_epoch_authorization.py new file mode 100644 index 000000000..50f2fd7b9 --- /dev/null +++ b/tests/test_management_recovery_epoch_authorization.py @@ -0,0 +1,265 @@ +"""Fixed authorization/persistence component tests, not host recovery acceptance.""" + +import hashlib +import inspect +import json +import os +import subprocess +import sys +from dataclasses import asdict, replace +from pathlib import Path + +import pytest + +from auto_coder.management.original_attempts import HistoryBinding, canonical +from auto_coder.management.recovery_containment_model import RecoveryFile +from auto_coder.management.recovery_epoch_authorization import AUTHORIZATION_SCHEMA, EpochAuthorization, _load_authorization, load_authorization, read_bound_file, validate_uv_version +from auto_coder.management.recovery_epoch_state import EpochBinding, EpochInventory, EpochUnavailable, _hold_recovery_epoch, issuer_lifetime + +pytestmark = pytest.mark.usefixtures("_use_real_commands", "_use_real_sleep") + + +@pytest.fixture +def _use_real_commands(): + pass + + +@pytest.fixture +def _use_real_sleep(): + pass + + +def _write(path, encoded): + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, "wb") as stream: + stream.write(encoded) + + +def _selected(path): + info = path.stat() + return RecoveryFile(str(path), hashlib.sha256(path.read_bytes()).hexdigest(), info.st_dev, info.st_ino) + + +def _fixture(tmp_path): + authority_root = tmp_path / "authority" + authority_root.mkdir(mode=0o700) + for name in ("consumer.json", "READY.json"): + _write(authority_root / name, b"{}") + authorization = EpochAuthorization( + AUTHORIZATION_SCHEMA, + "deployment-A", + "review-A", + "request-A", + "preparation-A", + "a" * 64, + "enrollment-A", + HistoryBinding(str(tmp_path / "history"), "deployment-A", "00000000-0000-4000-8000-000000000001", "sha256:" + "b" * 64), + _selected(authority_root / "consumer.json"), + _selected(authority_root / "READY.json"), + str(tmp_path / "source"), + str(authority_root / "source-token"), + "/usr/bin/python3", + "/usr/bin/uv", + "0.7.22", + "c" * 40, + authority_root.stat().st_dev, + authority_root.stat().st_ino, + ) + path = authority_root / "authorization.json" + encoded = canonical(asdict(authorization)) + _write(path, encoded) + root = tmp_path / "epoch" + root.mkdir(mode=0o700) + _write(root / "epoch.lock", b"") + lock = (root / "epoch.lock").stat() + inventory = EpochInventory( + 2, + str(root), + root.stat().st_dev, + root.stat().st_ino, + lock.st_dev, + lock.st_ino, + authorization.deployment_id, + authorization.authorization_id, + (authorization.enrollment_id,), + authorization.request_id, + authorization.preparation_id, + authorization.target_id, + "b" * 64, + "d" * 64, + hashlib.sha256(encoded).hexdigest(), + ) + _write(root / "inventory.json", canonical(asdict(inventory))) + return root, path, authorization, inventory + + +def _binding(inventory, *, evidence=None): + evidence = canonical({"originals": [{"journal": "retained", "sha256": "e" * 64}], "outcome": "unknown"}) if evidence is None else evidence + boot, pid, start = issuer_lifetime() + return EpochBinding( + inventory.deployment_id, + inventory.authorization_id, + inventory.enrollment_ids, + inventory.request_id, + inventory.preparation_id, + inventory.target_id, + inventory.source_sha256, + inventory.config_sha256, + hashlib.sha256(evidence).hexdigest(), + boot, + pid, + start, + inventory.authorization_sha256, + evidence.decode(), + ) + + +def test_complete_authorization_is_bound_to_fixed_inventory(tmp_path): + _, path, authorization, inventory = _fixture(tmp_path) + result = _load_authorization(path, inventory) + assert result.authorization == authorization + assert result.original == _selected(path) + assert tuple(inspect.signature(load_authorization).parameters) == ("inventory",) + + +@pytest.mark.parametrize("version", ["0.7.22", "0.13.6 (329bf94 2026-10-26)", "0.13.6 (x86_64-unknown-linux-gnu)", "0.13.6 (x)", "0.13.6 (" + "x" * 119 + ")"]) +def test_authorization_preserves_the_complete_installer_pin(tmp_path, version): + _, path, authorization, inventory = _fixture(tmp_path) + selected = replace(authorization, uv_version=version) + encoded = canonical(asdict(selected)) + path.write_bytes(encoded) + observed = _load_authorization(path, replace(inventory, authorization_sha256=hashlib.sha256(encoded).hexdigest())) + assert validate_uv_version(version) == version + assert observed.authorization == selected + assert observed.authorization.uv_version == version + assert observed.original.sha256 != inventory.authorization_sha256 or version == authorization.uv_version + + +@pytest.mark.parametrize( + "version", + [ + None, + True, + 13, + "", + "0.13", + "uv 0.13.6", + "0.13.6\n", + "0.13.6\r", + "0.13.6 ", + "0.13.6 (x) extra", + "0.13.6 (x) (y)", + "0.13.6 ()", + "0.13.6 ( )", + "0.13.6 ( x)", + "0.13.6 (x )", + "0.13.6 (x\ny)", + "0.13.6 (x\ty)", + "0.13.6 (nested(x))", + "0.13.6 ($HOME)", + "0.13.6 (`id`)", + "0.13.6 (x;y)", + "0.13.6 (x/y)", + "0.13.6 (é)", + "０.１３.６", + "0.13.6 (" + "x" * 120 + ")", + ], +) +def test_authorization_rejects_ambiguous_or_unbounded_installer_pin(tmp_path, version): + _, _, authorization, _ = _fixture(tmp_path) + raw = asdict(authorization) + raw["uv_version"] = version + with pytest.raises(EpochUnavailable, match="exact configured installer version"): + EpochAuthorization.parse(raw) + with pytest.raises(EpochUnavailable, match="exact configured installer version"): + validate_uv_version(version) + + +@pytest.mark.parametrize("field,value", [("authorization_id", "other-review"), ("request_id", "other-request"), ("preparation_id", "other-preparation"), ("target_id", "f" * 64), ("deployment_id", "other-deployment"), ("enrollment_ids", ("another-scope",)), ("authorization_sha256", "f" * 64)]) +def test_authorization_cannot_retarget_an_inventory(tmp_path, field, value): + root, path, _, inventory = _fixture(tmp_path) + before = sorted(item.name for item in root.iterdir()) + with pytest.raises(EpochUnavailable): + _load_authorization(path, replace(inventory, **{field: value})) + assert sorted(item.name for item in root.iterdir()) == before + + +def test_legacy_persistence_inventory_does_not_create_issuer_authority(tmp_path): + _, path, _, inventory = _fixture(tmp_path) + with pytest.raises(EpochUnavailable, match="version-two"): + _load_authorization(path, replace(inventory, schema_version=1)) + + +@pytest.mark.parametrize("change", ["partial", "duplicate", "noncanonical", "different-history", "unknown-field", "wrong-target-type"]) +def test_authorization_schema_refuses_ambiguous_selection(tmp_path, change): + _, path, authorization, inventory = _fixture(tmp_path) + raw = asdict(authorization) + if change == "partial": + del raw["ready_receipt"] + elif change == "different-history": + raw["history"]["deployment"] = "elsewhere" + elif change == "unknown-field": + raw["inventory_root"] = "/tmp/another-epoch" + elif change == "wrong-target-type": + raw["target_id"] = True + encoded = canonical(raw) + if change == "duplicate": + encoded = encoded[:-1] + b',"request_id":"request-A"}' + elif change == "noncanonical": + encoded = json.dumps(raw, indent=2).encode() + path.unlink() + _write(path, encoded) + selected = replace(inventory, authorization_sha256=hashlib.sha256(encoded).hexdigest()) + with pytest.raises(ValueError): + _load_authorization(path, selected) + + +def test_version_two_retains_full_canonical_evidence_before_consumption(tmp_path): + root, _, _, inventory = _fixture(tmp_path) + binding = _binding(inventory) + with _hold_recovery_epoch(root, expected_uid=os.geteuid()) as state: + state.reserve(binding) + state.record_intent(binding) + receipt = state.consume(binding) + state.check() + assert state.descriptor >= 0 + for phase in ("reserved", "intent", "consumed", "receipt"): + saved = json.loads((root / (phase + ".json")).read_bytes())["binding"] + assert saved["evidence_json"] == binding.evidence_json + assert hashlib.sha256(saved["evidence_json"].encode()).hexdigest() == saved["evidence_sha256"] + assert saved["authorization_sha256"] == inventory.authorization_sha256 + assert receipt.grant_delivered is False + with pytest.raises(EpochUnavailable): + with _hold_recovery_epoch(root, expected_uid=os.geteuid()): + pytest.fail("a consumed version-two epoch was replayed") + + +@pytest.mark.parametrize("evidence", [b"", b"[]", b'{"x": 1}', b'{"x":1,"x":2}', b'{"x":NaN}', b'{"x":"' + b"a" * (512 * 1024) + b'"}']) +def test_version_two_rejects_incomplete_or_ambiguous_evidence_before_reservation(tmp_path, evidence): + root, _, _, inventory = _fixture(tmp_path) + before = {item.name: item.read_bytes() for item in root.iterdir()} + with _hold_recovery_epoch(root, expected_uid=os.geteuid()) as state: + with pytest.raises(ValueError): + state.reserve(_binding(inventory, evidence=evidence)) + assert {item.name: item.read_bytes() for item in root.iterdir()} == before + + +def test_retained_evidence_digest_mismatch_cannot_consume(tmp_path): + root, _, _, inventory = _fixture(tmp_path) + binding = _binding(inventory) + with pytest.raises(EpochUnavailable, match="closed, failed or inherited"): + with _hold_recovery_epoch(root, expected_uid=os.geteuid()) as state: + state.reserve(binding) + with pytest.raises(EpochUnavailable, match="digest"): + state.record_intent(replace(binding, evidence_sha256="f" * 64)) + with pytest.raises(EpochUnavailable, match="closed, failed or inherited"): + state.record_intent(binding) + assert not (root / "consumed.json").exists() + + +@pytest.mark.parametrize("argument", ["--root", "--inventory", "--authorization", "--request"]) +def test_host_cli_rejects_request_selected_authority_before_execution(tmp_path, argument): + result = subprocess.run([sys.executable, "-m", "auto_coder.management.recovery_epoch", argument, str(tmp_path)], capture_output=True, text=True, timeout=20) + assert result.returncode == 2 + assert "unrecognized arguments" in result.stderr + assert list(tmp_path.iterdir()) == [] diff --git a/tests/test_management_recovery_epoch_consumer.py b/tests/test_management_recovery_epoch_consumer.py new file mode 100644 index 000000000..ff94b4136 --- /dev/null +++ b/tests/test_management_recovery_epoch_consumer.py @@ -0,0 +1,390 @@ +"""Real consumer custody with a wire issuer, not complete recovery authority. + +The independent issuer owns actual flocks and creates the socket pair, so the +consumer's SO_PEERCRED/SO_PEERPIDFD name that real issuer process. The fixture +supplies protocol grant bytes; it does not claim Docker/lineage/epoch acceptance. +The separate peer tests exercise the issuer's authenticated consumer checks. +""" + +from __future__ import annotations + +import array +import copy +import fcntl +import json +import os +import select +import signal +import socket +import subprocess +import sys +from contextlib import contextmanager +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Iterator + +import pytest + +from auto_coder.management.recovery_epoch_consumer import ConsumerConfiguration, OriginalFence, TargetOwner, TransferredResource, connect_recovery_epoch, parse_consumer_configuration +from auto_coder.management.recovery_epoch_peer import PeerUnavailable + +pytestmark = pytest.mark.usefixtures("_use_real_commands", "_use_real_sleep") + +_ISSUER = r""" +import array,fcntl,json,os,secrets,socket,sys +from pathlib import Path +from auto_coder.management.recovery_epoch_peer import OFFER_SCHEMA, RECEIPT_SCHEMA, encode_peer_message, receive_peer_message +from auto_coder.management.recovery_epoch_consumer import GRANT_SCHEMA +mailbox_fd, paths_json, config_json, mode = sys.argv[1:] +paths, configuration = json.loads(paths_json), json.loads(config_json) +held = {} +for name, operation in (('original', fcntl.LOCK_SH), ('controller', fcntl.LOCK_EX)): + held[name] = os.open(paths[name], os.O_RDONLY|os.O_NOFOLLOW) + fcntl.flock(held[name], operation|fcntl.LOCK_NB) +if any(item['kind']=='network-namespace' for item in configuration['transferred_resources']): + held['namespace'] = os.open('/proc/self/ns/net',os.O_RDONLY|os.O_CLOEXEC) +connection, consumer_endpoint = socket.socketpair(socket.AF_UNIX,socket.SOCK_STREAM) +mailbox=socket.socket(fileno=int(mailbox_fd)) +mailbox.sendmsg([b'c'],[(socket.SOL_SOCKET,socket.SCM_RIGHTS,array.array('i',[consumer_endpoint.fileno()]))]) +consumer_endpoint.close();mailbox.close() +print('issuer-ready',flush=True) +request,_=receive_peer_message(connection,timeout=10) +print('challenge-read',flush=True) +if mode=='cancel-before-offer': + try: + receive_peer_message(connection,timeout=10) + except Exception: + print('consumer-closed-before-offer',flush=True) + sys.stdin.readline() + raise SystemExit(0) + raise RuntimeError('unexpected pre-offer consumer message') +if sys.stdin.readline().strip()!='offer': raise SystemExit(0) +nonce=secrets.token_hex(32) +resources=[] +for item in configuration['original_fences']: + resources.append({'name':item['name'],'kind':'original-sh','device':item['device'],'inode':item['inode'],'transfer_index':None}) +owner=configuration['target_owner'] +resources.append({'name':owner['name'],'kind':'target-ex','device':owner['device'],'inode':owner['inode'],'transfer_index':None}) +transferred=[] +for item in configuration['transferred_resources']: + resources.append({**item,'transfer_index':len(transferred)}) + transferred.append(held[item['name']]) +offer={key:request[key] for key in ('request_id','authorization_id','target_id','challenge')} +offer.update(schema=OFFER_SCHEMA,receipt_nonce=nonce,resources=resources) +if mode=='offer-wrong-nonce': offer['challenge']='0'*64 +if mode=='offer-omitted': resources.pop() +if mode=='offer-extra': resources.append({'name':'undeclared','kind':'transferred-ex','device':1,'inode':2,'transfer_index':0}) +if mode=='offer-wrong-inode': resources[-1]['inode']+=1 +if mode=='offer-original-transfer': resources[0]['transfer_index']=0 +if mode=='offer-extra-fd': transferred.append(held['original']) +if mode=='offer-unheld-ex': fcntl.flock(held['controller'],fcntl.LOCK_UN) +connection.sendmsg([encode_peer_message(offer)],[(socket.SOL_SOCKET,socket.SCM_RIGHTS,array.array('i',transferred))]) +try: + receipt,_=receive_peer_message(connection,timeout=10) +except Exception: + print('receipt-unavailable',flush=True) + sys.stdin.readline() + raise SystemExit(0) +assert receipt['schema']==RECEIPT_SCHEMA and receipt['receipt_nonce']==nonce +assert {item['name'] for item in receipt['descriptors']}=={item['name'] for item in resources} +print('receipt-read',flush=True) +if sys.stdin.readline().strip()!='grant': raise SystemExit(0) +grant={key:request[key] for key in ('request_id','authorization_id','target_id','challenge')} +grant.update(schema=GRANT_SCHEMA,granted='recovery-epoch-consumed',receipt_nonce=nonce,inventory_sha256='1'*64,binding_sha256='2'*64,consumed_sha256='3'*64,receipt_sha256='4'*64,host_boot_id=Path('/proc/sys/kernel/random/boot_id').read_text().strip(),host_pid=os.getpid(),host_start_token=Path('/proc/self/stat').read_text().rsplit(')',1)[1].split()[19]) +if mode=='grant-wrong-target': grant['target_id']='another-target' +if mode=='grant-stale-nonce': grant['receipt_nonce']='0'*64 +if mode=='grant-missing-consumption': grant['consumed_sha256']='' +if mode=='grant-wrong-host': grant['host_pid']+=1 +if mode=='grant-extra-field': grant['another_authority']=True +connection.sendall(encode_peer_message(grant)) +print('grant-sent',flush=True) +for command in sys.stdin: + command=command.strip() + if command=='exit': break + if command=='close-channel': connection.close() + elif command=='unlock-controller': fcntl.flock(held['controller'],fcntl.LOCK_UN) + else: raise RuntimeError('unsupported issuer command') + print(command,flush=True) +""" + +_CONSUMER = r""" +import fcntl,json,os,socket,sys +from dataclasses import asdict +from pathlib import Path +from auto_coder.management.recovery_epoch_consumer import RecoveryEpochConsumer,parse_consumer_configuration +fd,config_path,target_path=sys.argv[1:] +connection=socket.socket(fileno=int(fd)) +configuration=parse_consumer_configuration(Path(config_path).read_bytes()) +target=os.open(target_path,os.O_RDONLY|os.O_NOFOLLOW) +fcntl.flock(target,fcntl.LOCK_EX|fcntl.LOCK_NB) +try: + handle=RecoveryEpochConsumer(connection,configuration,target,timeout=10,configuration_file=Path(config_path)) +except Exception as error: + print(json.dumps({'status':'refused','error':type(error).__name__,'detail':str(error)}),flush=True) + sys.stdin.readline() + raise SystemExit(0) +print(json.dumps({'status':'received','grant':asdict(handle.grant),'retained':len(handle.retained_descriptors)}),flush=True) +for command in sys.stdin: + command=command.strip() + if command=='exit': handle.close();break + if command=='close': + handle.close();print(json.dumps({'status':'closed'}),flush=True);continue + if command!='check': raise RuntimeError('unsupported consumer command') + try: + handle.check_current() + result={'status':'current'} + except Exception as error: + result={'status':'unavailable','error':type(error).__name__,'retained':len(handle.retained_descriptors),'all_open':all(os.fstat(fd).st_ino>0 for fd in handle.retained_descriptors)} + print(json.dumps(result),flush=True) +""" + + +_BOOTSTRAP = r""" +import json,os,socket,sys +from pathlib import Path +from auto_coder.management import recovery_epoch_bootstrap as bootstrap +from auto_coder.management.bootstrap_supervisor import BootstrapConfig,RuntimeSpec +from auto_coder.management.recovery_epoch_consumer import CONSUMER_CONFIG,RecoveryEpochConsumer,parse_consumer_configuration +fd,config_path,target_path=sys.argv[1:] +connection=socket.socket(fileno=int(fd)) +configuration=parse_consumer_configuration(Path(config_path).read_bytes()) +state=Path(target_path).parent +root=state.parent +for name in ('journal','oauth'): + (root/name).mkdir(mode=0o700) +config=BootstrapConfig('target-one',state,root/'oauth',root/'journal','http://127.0.0.1:8080','never-launched',(RuntimeSpec('never-launched','sha256:'+'1'*64,Path(sys.executable),('-c','raise RuntimeError("must not launch")')),),verification_timeout=10,recovery_epoch_config=CONSUMER_CONFIG) +# Component seam: use the real connected wire fixture, preserving the actual +# bootstrap ownership acquisition, signal handlers, consumer, and cleanup. +def connected_fixture(config_path,owner,*,timeout): + assert config_path==CONSUMER_CONFIG + return RecoveryEpochConsumer(connection,configuration,owner,timeout=timeout) +bootstrap.connect_recovery_epoch=connected_fixture +status=bootstrap.run_recovery_epoch_bootstrap(config) +print(json.dumps({'bootstrap_status':status}),flush=True) +raise SystemExit(status) +""" + + +@dataclass +class Exchange: + issuer: subprocess.Popen[str] + consumer: subprocess.Popen[str] + configuration: ConsumerConfiguration + configuration_path: Path + paths: dict[str, str] + + +def line(process: subprocess.Popen[str]) -> str: + assert process.stdout is not None + assert select.select([process.stdout], [], [], 15)[0], "independent process did not reach the required barrier" + value = process.stdout.readline().strip() + assert value, (process.poll(), process.stderr.read() if process.poll() is not None and process.stderr is not None else "") + return value + + +def command(process: subprocess.Popen[str], value: str) -> None: + assert process.stdin is not None + process.stdin.write(value + "\n") + process.stdin.flush() + + +def configuration(root: Path, *, namespace: bool = False, bootstrap: bool = False) -> tuple[ConsumerConfiguration, dict[str, str]]: + paths = {name: str(root / name) for name in ("original", "target", "controller")} + if bootstrap: + state = root / "state" + state.mkdir(mode=0o700) + paths["target"] = str(state / ".supervisor-owner.lock") + for path in paths.values(): + descriptor = os.open(path, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0o600) + os.close(descriptor) + original_info, target_info, controller_info = (Path(paths[name]).stat() for name in ("original", "target", "controller")) + directory = root.stat() + transferred = [TransferredResource("controller", "transferred-ex", controller_info.st_dev, controller_info.st_ino)] + if namespace: + info = Path("/proc/self/ns/net").stat() + transferred.append(TransferredResource("namespace", "network-namespace", info.st_dev, info.st_ino)) + return ( + ConsumerConfiguration( + 1, "request-one", "authorization-one", "target-one", os.geteuid(), directory.st_dev, directory.st_ino, (OriginalFence("original", paths["original"], original_info.st_dev, original_info.st_ino),), TargetOwner("target", target_info.st_dev, target_info.st_ino), tuple(transferred) + ), + paths, + ) + + +@contextmanager +def exchange(tmp_path: Path, mode: str = "valid", *, namespace: bool = False, bootstrap: bool = False) -> Iterator[Exchange]: + selected, paths = configuration(tmp_path, namespace=namespace, bootstrap=bootstrap) + config_path = tmp_path / "configuration.json" + config_path.write_text(json.dumps(asdict(selected))) + config_path.chmod(0o600) + mailbox, issuer_mailbox = socket.socketpair() + mailbox.settimeout(10) + issuer = consumer = None + endpoint = None + try: + issuer = subprocess.Popen([sys.executable, "-B", "-c", _ISSUER, str(issuer_mailbox.fileno()), json.dumps(paths), json.dumps(asdict(selected)), mode], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, pass_fds=(issuer_mailbox.fileno(),)) + issuer_mailbox.close() + payload, ancillary, flags, _ = mailbox.recvmsg(1, socket.CMSG_SPACE(array.array("i").itemsize), socket.MSG_CMSG_CLOEXEC) + assert payload == b"c" and not flags & (socket.MSG_TRUNC | socket.MSG_CTRUNC) and len(ancillary) == 1 + descriptors = array.array("i") + descriptors.frombytes(ancillary[0][2]) + assert len(descriptors) == 1 + endpoint = descriptors[0] + assert line(issuer) == "issuer-ready" + consumer = subprocess.Popen([sys.executable, "-B", "-c", _BOOTSTRAP if bootstrap else _CONSUMER, str(endpoint), str(config_path), paths["target"]], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, pass_fds=(endpoint,)) + os.close(endpoint) + endpoint = None + assert line(issuer) == "challenge-read" + yield Exchange(issuer, consumer, selected, config_path, paths) + finally: + if endpoint is not None: + os.close(endpoint) + mailbox.close() + issuer_mailbox.close() + for process in (consumer, issuer): + if process is None: + continue + if process.poll() is None: + try: + command(process, "exit") + except BrokenPipeError: + pass + try: + process.communicate(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.communicate(timeout=5) + + +def grant(case: Exchange) -> dict[str, object]: + command(case.issuer, "offer") + assert line(case.issuer) == "receipt-read" + assert case.consumer.poll() is None + command(case.issuer, "grant") + assert line(case.issuer) == "grant-sent" + return json.loads(line(case.consumer)) + + +@pytest.mark.parametrize("namespace", [False, True]) +def test_real_consumer_retains_original_target_controller_and_namespace_custody(tmp_path: Path, namespace: bool) -> None: + with exchange(tmp_path, namespace=namespace) as case: + result = grant(case) + assert result["status"] == "received" and result["retained"] == 2 + int(namespace) + assert result["grant"]["host_pid"] == case.issuer.pid + command(case.consumer, "check") + assert json.loads(line(case.consumer)) == {"status": "current"} + assert sorted(path.name for path in tmp_path.iterdir()) == ["configuration.json", "controller", "original", "target"] + + +@pytest.mark.parametrize("mode", ["offer-wrong-nonce", "offer-omitted", "offer-extra", "offer-wrong-inode", "offer-original-transfer", "offer-extra-fd", "offer-unheld-ex"]) +def test_consumer_refuses_incomplete_or_wrong_physical_offer(tmp_path: Path, mode: str) -> None: + with exchange(tmp_path, mode) as case: + command(case.issuer, "offer") + result = json.loads(line(case.consumer)) + assert result["status"] == "refused" and result["error"] == "PeerUnavailable" + assert line(case.issuer) == "receipt-unavailable" + + +@pytest.mark.parametrize("mode", ["grant-wrong-target", "grant-stale-nonce", "grant-missing-consumption", "grant-wrong-host", "grant-extra-field"]) +def test_consumer_refuses_wrong_or_unbound_grant(tmp_path: Path, mode: str) -> None: + with exchange(tmp_path, mode) as case: + result = grant(case) + assert result["status"] == "refused" and result["error"] == "PeerUnavailable" + + +@pytest.mark.parametrize("failure", ["host-death", "channel-closed", "controller-unlocked", "config-changed", "config-replaced"]) +def test_current_failure_preserves_received_custody_until_explicit_close(tmp_path: Path, failure: str) -> None: + with exchange(tmp_path, namespace=True) as case: + assert grant(case)["status"] == "received" + if failure == "host-death": + case.issuer.kill() + case.issuer.wait(timeout=5) + elif failure == "channel-closed": + command(case.issuer, "close-channel") + assert line(case.issuer) == "close-channel" + elif failure == "controller-unlocked": + command(case.issuer, "unlock-controller") + assert line(case.issuer) == "unlock-controller" + elif failure == "config-changed": + with case.configuration_path.open("ab") as stream: + stream.write(b"\n") + else: + encoded = case.configuration_path.read_bytes() + case.configuration_path.rename(tmp_path / "preserved-configuration.json") + case.configuration_path.write_bytes(encoded) + case.configuration_path.chmod(0o600) + command(case.consumer, "check") + result = json.loads(line(case.consumer)) + assert result == {"status": "unavailable", "error": "PeerUnavailable", "retained": 3, "all_open": True} + command(case.consumer, "close") + assert json.loads(line(case.consumer)) == {"status": "closed"} + + +def test_issuer_death_keeps_original_and_transferred_fences_exclusive_until_consumer_closes(tmp_path: Path) -> None: + with exchange(tmp_path) as case: + assert grant(case)["status"] == "received" + case.issuer.kill() + case.issuer.wait(timeout=5) + command(case.consumer, "check") + assert json.loads(line(case.consumer))["status"] == "unavailable" + contenders = [os.open(case.paths[name], os.O_RDONLY | os.O_NOFOLLOW) for name in ("original", "controller")] + try: + for descriptor in contenders: + with pytest.raises(BlockingIOError): + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + command(case.consumer, "close") + assert json.loads(line(case.consumer)) == {"status": "closed"} + for descriptor in contenders: + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + finally: + for descriptor in contenders: + os.close(descriptor) + + +@pytest.mark.parametrize("signum", [signal.SIGTERM, signal.SIGINT]) +def test_bootstrap_cancellation_before_offer_closes_connection_without_waiting_for_grant(tmp_path: Path, signum: int) -> None: + with exchange(tmp_path, "cancel-before-offer", bootstrap=True) as case: + # The observed challenge proves that the actual bootstrap signal handler + # is installed and its consumer is blocked before any physical offer. + assert case.consumer.poll() is None + case.consumer.send_signal(signum) + assert case.consumer.wait(timeout=5) == 1 + assert line(case.issuer) == "consumer-closed-before-offer" + stdout, stderr = case.consumer.communicate(timeout=5) + assert '{"bootstrap_status": 1}' in stdout, (stdout, stderr) + assert "Recovery custody acknowledged" not in stdout + assert not any((tmp_path / "journal").iterdir()) + assert sorted(path.name for path in (tmp_path / "state").iterdir()) == [".supervisor-owner.lock", "logs"] + descriptor = os.open(case.paths["target"], os.O_RDONLY | os.O_NOFOLLOW) + try: + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + finally: + os.close(descriptor) + + +@pytest.mark.parametrize("change", ["duplicate-name", "duplicate-object", "duplicate-path", "missing-controller", "unknown-kind", "boolean-identity", "extra-field"]) +def test_consumer_configuration_is_complete_and_unambiguous(tmp_path: Path, change: str) -> None: + selected, _ = configuration(tmp_path) + raw = copy.deepcopy(asdict(selected)) + assert parse_consumer_configuration(json.dumps(raw).encode()) == selected + if change == "duplicate-name": + raw["target_owner"]["name"] = "original" + elif change == "duplicate-object": + raw["target_owner"].update(device=raw["original_fences"][0]["device"], inode=raw["original_fences"][0]["inode"]) + elif change == "duplicate-path": + raw["original_fences"] = (*raw["original_fences"], {**raw["original_fences"][0], "name": "another", "inode": 1234}) + elif change == "missing-controller": + raw["transferred_resources"] = [] + elif change == "unknown-kind": + raw["transferred_resources"][0]["kind"] = "unverified-ex" + elif change == "boolean-identity": + raw["target_owner"]["inode"] = True + else: + raw["root_override"] = str(tmp_path) + with pytest.raises(PeerUnavailable): + parse_consumer_configuration(json.dumps(raw).encode()) + + +def test_production_factory_rejects_config_path_override_before_connecting(tmp_path: Path) -> None: + with pytest.raises(PeerUnavailable, match="fixed bootstrap path"): + connect_recovery_epoch(tmp_path / "configuration.json", -1) diff --git a/tests/test_management_recovery_epoch_diagnostics.py b/tests/test_management_recovery_epoch_diagnostics.py new file mode 100644 index 000000000..f999e41f4 --- /dev/null +++ b/tests/test_management_recovery_epoch_diagnostics.py @@ -0,0 +1,242 @@ +"""Bounded issuer-refusal diagnostics; no native acceptance authority.""" + +from __future__ import annotations + +import errno +import json +import os +import re +import subprocess +import sys +from dataclasses import asdict +from pathlib import Path + +import pytest + +from auto_coder.logger_config import setup_logger +from auto_coder.management import recovery_containment as containment +from auto_coder.management import recovery_diagnostics as diagnostics +from auto_coder.management import recovery_epoch as issuer +from auto_coder.management.recovery_containment_model import RecoveryContainmentUnavailable, RecoveryResources + + +def _failure() -> ValueError: + # The code location is safe metadata; its directory, exception arguments, + # attached filename and frame locals are deliberately private test data. + source = "def fail():\n local_credential = 'private-local-credential'\n raise FileNotFoundError(2, 'private-error-message', '/private/source-token.txt')\n" + namespace = {} + exec(compile(source, "/private/code-directory/observed_component.py", "exec"), namespace) + try: + try: + namespace["fail"]() + except OSError as cause: + raise ValueError("fixed issuer refusal") from cause + except ValueError as error: + return error + + +def test_refusal_causes_retain_only_code_identity_and_integer_errno() -> None: + assert issuer._refusal_causes is diagnostics.refusal_causes + error = _failure() + rows = issuer._refusal_causes(error) + assert rows[0] == {"type": "ValueError", "file": Path(__file__).name, "function": "_failure", "line": error.__traceback__.tb_lineno} + assert rows[1] == {"type": "FileNotFoundError", "errno": errno.ENOENT, "file": "observed_component.py", "function": "fail", "line": 3} + assert "private" not in json.dumps(rows) and "fixed issuer refusal" not in json.dumps(rows) + assert all(set(row) <= {"type", "errno", "file", "function", "line"} for row in rows) + + +def test_refusal_cause_chain_is_limited_to_eight_entries() -> None: + error = ValueError("private-deep-message") + for _ in range(40): + outer = RuntimeError("private-outer-message") + outer.__cause__ = error + error = outer + assert issuer._refusal_causes(error) == [{"type": "RuntimeError"}] * 8 + + +@pytest.mark.parametrize("self_cycle", [False, True]) +def test_refusal_cause_cycles_are_finite(self_cycle: bool) -> None: + first, second = ValueError("private-first"), OSError(errno.EIO, "private-second") + first.__cause__ = first if self_cycle else second + second.__cause__ = first + assert issuer._refusal_causes(first) == ([{"type": "ValueError"}] if self_cycle else [{"type": "ValueError"}, {"type": "OSError", "errno": errno.EIO}]) + + +@pytest.mark.parametrize("number", ["private-errno", True, None]) +def test_refusal_causes_ignore_noninteger_errno(number: object) -> None: + error = OSError("private-argument") + error.errno = number + assert issuer._refusal_causes(error) == [{"type": "OSError"}] + + +def test_refusal_causes_never_format_exception_objects_or_unsafe_code_metadata() -> None: + class SensitiveFailure(ValueError): + def __str__(self) -> str: + pytest.fail("causal metadata must not format the exception") + + try: + exec(compile("raise error", "/private/path/invalid filename.py", "exec"), {"error": SensitiveFailure("private-argument")}) + except SensitiveFailure as error: + assert issuer._refusal_causes(error) == [{"type": "SensitiveFailure"}] + + +def test_cli_refusal_preserves_exit_and_emits_safe_causes_to_stdout_and_file(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: + log = tmp_path / "issuer.log" + + def configure(**kwargs: object) -> None: + kwargs["log_file"] = str(log) + setup_logger(**kwargs) + + def refuse() -> None: + raise _failure() + + monkeypatch.setattr(issuer, "setup_logger", configure) + monkeypatch.setattr(issuer, "run_issuer", refuse) + assert issuer.main([]) == 2 + issuer.logger.complete() + standard = re.sub(r"\x1b\[[0-9;]*m", "", capsys.readouterr().out) + persisted = log.read_text() + for output in (standard, persisted): + assert "Recovery epoch unavailable; preserved evidence is not repaired: fixed issuer refusal" in output + messages = [line.partition("Recovery epoch refusal cause ")[2] for line in output.splitlines() if "Recovery epoch refusal cause " in line] + assert len(messages) == 2 + assert [line.partition(": ")[0] for line in messages] == ["1", "2"] + rows = [json.loads(line.partition(": ")[2]) for line in messages] + assert rows[1] == {"type": "FileNotFoundError", "errno": errno.ENOENT, "file": "observed_component.py", "function": "fail", "line": 3} + assert "private" not in output + issuer.logger.remove() + + +@pytest.mark.parametrize("action", ["observe", "verify"]) +def test_containment_cli_preserves_masked_refusal_and_exports_safe_nested_causes(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], action: str) -> None: + log = tmp_path / "containment.log" + resources = tmp_path / "resources.json" + resources.write_text(json.dumps(asdict(RecoveryResources(42, "12345", "00000000-0000-4000-8000-000000000001", (), (), ())))) + arguments = ["recovery_containment", action, "--enrollment", "component-observation", "--resources", str(resources)] + if action == "verify": + previous = tmp_path / "previous.json" + previous.write_text("{}") + arguments.extend(("--previous", str(previous))) + + def configure(**kwargs: object) -> None: + kwargs["log_file"] = str(log) + setup_logger(**kwargs) + + def masked(*_args: object) -> None: + try: + raise _failure() + except ValueError as cause: + raise RecoveryContainmentUnavailable("the original namespace pin cannot be retained") from cause + + monkeypatch.setattr(sys, "argv", arguments) + monkeypatch.setattr(containment, "setup_logger", configure) + monkeypatch.setattr(containment, action, masked) + assert containment.main() == 2 + containment.logger.complete() + standard = re.sub(r"\x1b\[[0-9;]*m", "", capsys.readouterr().out) + result = json.loads(standard.splitlines()[-1]) + assert result["schema"] == "current-recovery-containment-v1" and result["status"] == "unavailable" + assert result["reason"] == "the original namespace pin cannot be retained" + assert all(result[name] is False for name in ("grant_issued", "history_asserted", "admission_created")) + assert len(result["causes"]) == 3 + assert result["causes"][-1] == {"type": "FileNotFoundError", "errno": errno.ENOENT, "file": "observed_component.py", "function": "fail", "line": 3} + assert diagnostics.safe_refusal_causes(result["causes"]) == result["causes"] + for output in (standard, log.read_text()): + assert "Current recovery containment unavailable: the original namespace pin cannot be retained" in output + messages = [line.partition("Current recovery containment refusal cause ")[2] for line in output.splitlines() if "Current recovery containment refusal cause " in line] + assert [json.loads(line.partition(": ")[2]) for line in messages] == result["causes"] + assert "private" not in output and "fixed issuer refusal" not in output + containment.logger.remove() + + +def test_safe_subprocess_causes_are_a_detached_copy_of_the_exact_metadata() -> None: + rows = diagnostics.refusal_causes(_failure()) + parsed = diagnostics.safe_refusal_causes(json.loads(json.dumps(rows))) + assert parsed == rows + parsed[-1]["line"] = 100 + assert rows[-1]["line"] == 3 + + +@pytest.mark.parametrize( + "value", + [ + None, + {}, + "private-data", + [{"type": "ValueError"}] * 9, + ["private-data"], + [{"type": "ValueError", "message": "private-data"}], + [{"type": "invalid type"}], + [{"type": "ValueError", "errno": True}], + [{"type": "ValueError", "errno": "private-data"}], + [{"type": "ValueError", "file": "/private/observed.py", "function": "fail", "line": 3}], + [{"type": "ValueError", "file": "observed.py", "function": "private argument", "line": 3}], + [{"type": "ValueError", "file": "observed.py", "function": "fail", "line": True}], + [{"type": "ValueError", "file": "observed.py", "function": "fail", "line": 0}], + [{"type": "ValueError", "file": "observed.py"}], + [{"type": "V" * 97}], + ], +) +def test_subprocess_causes_reject_oversized_or_nonmetadata_fields(value: object) -> None: + assert diagnostics.safe_refusal_causes(value) == [] + + +@pytest.mark.usefixtures("_use_real_commands") +@pytest.mark.parametrize("unavailable", [False, True]) +def test_containment_cli_keeps_json_last_with_real_asynchronous_logging(tmp_path: Path, unavailable: bool) -> None: + resources = tmp_path / "resources.json" + resources.write_text(json.dumps(asdict(RecoveryResources(42, "12345", "00000000-0000-4000-8000-000000000001", (), (), ())))) + log = tmp_path / "queued.log" + program = r""" +import sys +import time +from dataclasses import dataclass +from auto_coder.logger_config import setup_logger +from auto_coder.management import recovery_containment as cli + +resources, logfile, unavailable = sys.argv[1:] + +class DelayedStandardOutput: + def write(self, text): + time.sleep(0.002) + return sys.stdout.write(text) + def flush(self): + return sys.stdout.flush() + +def configure(**kwargs): + setup_logger(stream=DelayedStandardOutput(), log_file=logfile) + assert all(handler._enqueue for handler in cli.logger._core.handlers.values()) + +@dataclass +class ComponentReport: + status: str = "observed-current-recovery-containment" + grant_issued: bool = False + history_asserted: bool = False + admission_created: bool = False + +def observe(*args): + for index in range(100): + cli.logger.warning("Queued component observation {}", index) + if unavailable == "yes": + try: + raise FileNotFoundError(2, "private-inner-message", "/private/source-token") + except OSError as cause: + raise cli.RecoveryContainmentUnavailable("fixed masked refusal") from cause + return ComponentReport() + +cli.setup_logger = configure +cli.observe = observe +sys.argv = ["recovery_containment", "observe", "--enrollment", "component", "--resources", resources] +raise SystemExit(cli.main()) +""" + environment = {name: value for name, value in os.environ.items() if name not in {"PYTEST_CURRENT_TEST", "PYTEST_RUNNER", "_PYTEST_CURRENT_TEST", "LLM_LOGGING_DISABLED"}} + environment["AUTO_CODER_LOG_FILE"] = str(log) + completed = subprocess.run([sys.executable, "-c", program, str(resources), str(log), "yes" if unavailable else "no"], capture_output=True, text=True, timeout=30, env=environment) + assert completed.returncode == (2 if unavailable else 0), completed.stderr + result = json.loads(completed.stdout.splitlines()[-1]) + assert result["status"] == ("unavailable" if unavailable else "observed-current-recovery-containment") + assert all(result[name] is False for name in ("grant_issued", "history_asserted", "admission_created")) + assert "Queued component observation 99" in completed.stdout and "Queued component observation 99" in log.read_text() + assert "private" not in completed.stdout and "private" not in log.read_text() + if unavailable: + assert result["causes"][-1] == {"type": "FileNotFoundError", "errno": 2} diff --git a/tests/test_management_recovery_epoch_fault_tracer.py b/tests/test_management_recovery_epoch_fault_tracer.py new file mode 100644 index 000000000..8d51f5e32 --- /dev/null +++ b/tests/test_management_recovery_epoch_fault_tracer.py @@ -0,0 +1,135 @@ +"""Native syscall barrier checks, separate from full VM issuer acceptance. + +These tests trace the real durable-state component in independent processes. +The VM workflow separately traces the actual zero-argument host issuer with +real Docker/history/containment/READY and authenticated consumer custody. +""" + +import errno +import os +import signal +import subprocess +import sys +from pathlib import Path + +import pytest +from scripts.recovery_epoch_fault_tracer import BOUNDARY_SCENARIOS, PUBLICATIONS, KernelPublicationBarrier, parse_boundary + +from auto_coder.management.recovery_epoch_state import EpochUnavailable, _hold_recovery_epoch +from tests.test_management_recovery_epoch_state import _inventory, _snapshot + +pytestmark = pytest.mark.usefixtures("_use_real_commands", "_use_real_sleep") + + +@pytest.fixture +def _use_real_commands(): + pass + + +@pytest.fixture +def _use_real_sleep(): + pass + + +_CHILD = """ +import os +import sys +from pathlib import Path +from auto_coder.management.recovery_epoch_state import EpochBinding, _hold_recovery_epoch, issuer_lifetime +assert os.read(int(sys.argv[2]), 1) == b'1' +os.close(int(sys.argv[2])) +with _hold_recovery_epoch(Path(sys.argv[1]), expected_uid=os.geteuid()) as state: + inventory = state.inventory + boot, pid, start = issuer_lifetime() + binding = EpochBinding(inventory.deployment_id, inventory.authorization_id, inventory.enrollment_ids, inventory.request_id, inventory.preparation_id, inventory.target_id, inventory.source_sha256, inventory.config_sha256, 'd' * 64, boot, pid, start) + state.reserve(binding) + state.record_intent(binding) + state.consume(binding) +print('all-publications-completed', flush=True) +""" + + +def _start(process: subprocess.Popen, root: Path, name: str, when: str) -> KernelPublicationBarrier: + barrier = KernelPublicationBarrier(process, root=root, name=name, when=when) + try: + barrier.start() + except OSError as error: + barrier.close() + if error.errno in (errno.EPERM, errno.EACCES): + pytest.skip("this environment prohibits actual ptrace; full native VM acceptance remains required") + raise + return barrier + + +@pytest.mark.parametrize("scenario", BOUNDARY_SCENARIOS) +def test_real_syscall_stop_preserves_exact_publication_prefix(tmp_path: Path, scenario: str) -> None: + root, _ = _inventory(tmp_path) + name, when = parse_boundary(scenario) + reader, writer = os.pipe() + process = subprocess.Popen([sys.executable, "-B", "-c", _CHILD, str(root), str(reader)], pass_fds=(reader,), stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + os.close(reader) + barrier = None + try: + barrier = _start(process, root, name, when) + os.write(writer, b"1") + barrier.resume_initial() + observation = barrier.wait(timeout=20) + assert observation.process_id == process.pid and observation.name == name and observation.when == when + count = PUBLICATIONS.index(name) + (when == "after") + assert {path.name for path in root.iterdir()} == {"epoch.lock", "inventory.json", *PUBLICATIONS[:count]} + assert observation.file_fsync_observed == (when == "after") + assert observation.directory_fsync_observed == (when == "after") + assert (observation.file_sha256 is not None) == (when == "after") + retained = _snapshot(root) + barrier.kill() + assert process.returncode == -signal.SIGKILL + assert _snapshot(root) == retained + if count: + with pytest.raises(EpochUnavailable, match="existing, partial or conflicting"): + with _hold_recovery_epoch(root, expected_uid=os.geteuid()): + pytest.fail("a published original record was reused") + assert _snapshot(root) == retained + finally: + os.close(writer) + if barrier is not None: + barrier.close() + if process.poll() is None: + process.kill() + process.communicate(timeout=15) + + +def test_releasing_actual_syscall_barrier_continues_original_production_code(tmp_path: Path) -> None: + root, _ = _inventory(tmp_path) + reader, writer = os.pipe() + process = subprocess.Popen([sys.executable, "-B", "-c", _CHILD, str(root), str(reader)], pass_fds=(reader,), stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + os.close(reader) + barrier = None + try: + barrier = _start(process, root, "intent-head.json", "after") + os.write(writer, b"1") + barrier.resume_initial() + barrier.wait(timeout=20) + assert not (root / "consumed.json").exists() + barrier.release() + stdout, stderr = process.communicate(timeout=20) + assert process.returncode == 0, stderr + assert "all-publications-completed" in stdout + assert {path.name for path in root.iterdir()} == {"epoch.lock", "inventory.json", *PUBLICATIONS} + finally: + os.close(writer) + if barrier is not None: + barrier.close() + if process.poll() is None: + process.kill() + process.communicate(timeout=15) + + +@pytest.mark.parametrize("scenario", ["", "before", "after-inventory", "before-../reserved", "before-reserved.json", "before-reserved-head-extra", "after-consumed-root"]) +def test_boundary_selector_cannot_choose_unreviewed_paths(scenario: str) -> None: + with pytest.raises(ValueError, match="reviewed publication"): + parse_boundary(scenario) + + +def test_all_reviewed_record_and_head_boundaries_are_explicit() -> None: + assert len(BOUNDARY_SCENARIOS) == 16 and len(set(BOUNDARY_SCENARIOS)) == 16 + assert {parse_boundary(item) for item in BOUNDARY_SCENARIOS} == {(name, when) for name in PUBLICATIONS for when in ("before", "after")} diff --git a/tests/test_management_recovery_epoch_host.py b/tests/test_management_recovery_epoch_host.py new file mode 100644 index 000000000..410626426 --- /dev/null +++ b/tests/test_management_recovery_epoch_host.py @@ -0,0 +1,799 @@ +"""Authorization gates and acceptance oracles for the real recovery VM driver. + +These are component tests of the driver, never substitutes for its native VM +run. The separate workflow runs the driver against genuine Docker/systemd, +original stores, qualified source publication, READY and peer custody. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest +import yaml +from scripts import recovery_epoch_native_negatives as negatives +from scripts import test_management_recovery_epoch_host as fixture + +from auto_coder.management.original_attempts import canonical +from tests.test_management_archived_cutover_container import Generation + + +def _inputs(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> dict[str, object]: + source = tmp_path / "source" + source.mkdir() + token = tmp_path / "token" + token.write_text("component-token-placeholder") + for name in ("AUTOCODER_FRESH_TEST_BASE_IMAGE", "AUTOCODER_RECOVERY_CADDY_IMAGE"): + monkeypatch.setenv(name, "sha256:" + "a" * 64) + return {"authorization_id": "test-authorization", "source_repository": source, "source_token_file": token, "source_sha": "b" * 40, "uv": Path(sys.executable).resolve(), "uv_version": "0.13.6", "output": tmp_path / "result.json"} + + +def test_invalid_source_stops_before_host_authorization(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + inputs = _inputs(tmp_path, monkeypatch) + inputs["source_sha"] = "main" + monkeypatch.setattr(fixture, "require_disposable_host", lambda _: pytest.fail("host authorization must not be reached")) + with pytest.raises(ValueError, match="exact qualified source"): + fixture.run_fixture(**inputs) + assert not inputs["output"].exists() + + +def test_unmatched_marker_stops_before_fixture_provisioning(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + inputs = _inputs(tmp_path, monkeypatch) + created = [] + + def refusal(_): + raise PermissionError("the actual original marker does not select this host") + + monkeypatch.setattr(fixture, "require_disposable_host", refusal) + monkeypatch.setattr(fixture, "_private_directory", lambda path: created.append(path)) + monkeypatch.setattr(fixture.subprocess, "run", lambda *_args, **_kwargs: pytest.fail("unauthorized driver attempted a host effect")) + with pytest.raises(PermissionError, match="original marker"): + fixture.run_fixture(**inputs) + assert created == [] and not inputs["output"].exists() + + +def test_unique_fixture_leaf_fits_the_actual_original_socket_paths() -> None: + first, second = fixture._fixture_directory(), fixture._fixture_directory() + assert first.parent == second.parent == fixture.FIXTURE_ROOT and first != second + assert len(first.name) == 16 and int(first.name, 16) >= 0 + for name in ("A", "B", "C", "D"): + assert len(os.fsencode(first / "deployment" / name / "guard" / "fresh-guard.sock")) <= 107 + assert len(os.fsencode(fixture.PEER_SOCKET)) <= 107 + + +def test_oversized_socket_path_stops_before_any_original_provisioning(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + inputs = _inputs(tmp_path, monkeypatch) + monkeypatch.setattr(fixture, "require_disposable_host", lambda _: {}) + monkeypatch.setattr(fixture, "FIXTURE_ROOT", tmp_path / ("long-original-root-" * 8)) + monkeypatch.setattr(fixture, "_private_directory", lambda _: pytest.fail("oversized socket path reached original provisioning")) + monkeypatch.setattr(fixture.subprocess, "run", lambda *_args, **_kwargs: pytest.fail("oversized socket path reached a native effect")) + with pytest.raises(ValueError, match="Unix socket pathname exceeds"): + fixture.run_fixture(**inputs) + assert not inputs["output"].exists() + + +def test_mutable_image_tag_stops_before_host_authorization(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + inputs = _inputs(tmp_path, monkeypatch) + monkeypatch.setenv("AUTOCODER_FRESH_TEST_BASE_IMAGE", "auto-coder:latest") + monkeypatch.setattr(fixture, "require_disposable_host", lambda _: pytest.fail("host authorization must not be reached")) + with pytest.raises(ValueError, match="exact already imported"): + fixture.run_fixture(**inputs) + + +def test_linked_credential_stops_before_host_authorization(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + inputs = _inputs(tmp_path, monkeypatch) + linked = tmp_path / "linked-token" + linked.symlink_to(inputs["source_token_file"]) + inputs["source_token_file"] = linked + monkeypatch.setattr(fixture, "require_disposable_host", lambda _: pytest.fail("host authorization must not be reached")) + with pytest.raises(ValueError, match="canonical absolute"): + fixture.run_fixture(**inputs) + + +def test_command_failure_exports_bounded_diagnostics_without_inputs_or_credentials() -> None: + error = subprocess.CalledProcessError( + 1, + ["docker", "compose", "--private-command-value"], + output="Network selected Created\nError response from daemon: AppArmor profile could not be loaded\nAuthorization: Bearer secret-value\n", + stderr="stderr boundary\n" + "safe line\n" * 100, + ) + result = fixture._command_failure(error) + assert result["type"] == "CalledProcessError" and result["returncode"] == 1 + assert len(result["diagnostics"]) <= 60 + assert "private-command-value" not in json.dumps(result) and "secret-value" not in json.dumps(result) + short = fixture._command_failure(subprocess.CalledProcessError(1, ["docker", "compose"], output=b"Error response from daemon: AppArmor profile could not be loaded\n")) + assert "AppArmor profile could not be loaded" in "".join(short["diagnostics"]) + assert fixture._command_failure(ValueError("unrelated error")) is None + + +def test_cli_failure_causes_survive_the_actual_final_result_log_truncation(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + inputs = _inputs(tmp_path, monkeypatch) + root = tmp_path / "fixtures" / "original" + for name, path in (("FIXTURE_ROOT", root.parent), ("EPOCH_ROOT", tmp_path / "epoch"), ("PEER_ROOT", tmp_path / "peer"), ("AUTHORIZATION_ROOT", tmp_path / "authorization")): + monkeypatch.setattr(fixture, name, path) + monkeypatch.setattr(fixture, "_fixture_directory", lambda: root) + monkeypatch.setattr(fixture, "require_disposable_host", lambda _: {}) + + def private_directory(path: Path) -> None: + if path not in (Path("/var/lib/auto-coder"), Path("/etc/auto-coder")): + path.mkdir(mode=0o700) + + causes = [{"type": "OSError", "errno": 5, "file": "observed_component.py", "function": "read_current", "line": 42}] + document = {"schema": "current-recovery-containment-v1", "status": "unavailable", "grant_issued": False, "history_asserted": False, "admission_created": False, "reason": "fixed masked refusal", "causes": causes, "unrelated": "private-subprocess-data"} + monkeypatch.setattr(fixture, "_private_directory", private_directory) + monkeypatch.setattr(negatives.subprocess, "run", lambda *_args, **_kwargs: subprocess.CompletedProcess([], 2, json.dumps(document) + "\n", "")) + + def refuse() -> None: + negatives._containment_cli(root, "original-observer", "component-observation", tmp_path / "private-resources.json") + + def many_later_diagnostics(_result: object, _chain: object) -> None: + (root / "issuer.log").write_text("RuntimeError: observed_component.py read_current line42\n" + "".join("RuntimeError: Later issuer diagnostic " + str(index) + "\n" for index in range(150))) + for index in range(150): + fixture.logger.error("Later bounded native diagnostic {}", index) + + monkeypatch.setattr(fixture.InhibitedStockRoutes, "install", refuse) + monkeypatch.setattr(fixture, "_record_failure_sidecars", many_later_diagnostics) + previous_umask = os.umask(0o077) + try: + with pytest.raises(negatives.NativeContainmentFailure, match="fixed masked refusal"): + fixture.run_fixture(**inputs) + fixture.logger.complete() + result = json.loads(inputs["output"].read_bytes()) + finally: + os.umask(previous_umask) + fixture.logger.remove() + assert result["status"] == "failed" and result["failure_type"] == "NativeContainmentFailure" + assert result["containment_failure"] == {"schema": "native-recovery-containment-cli-failure-v1", "action": "observe", "returncode": 2, "causes": causes} + assert 1 <= len(result["diagnostics"]) <= 120 and any("Later issuer diagnostic 149" in line for line in result["diagnostics"]) + assert "observed_component.py" not in "\n".join(result["diagnostics"]) + assert all(result[name] is False for name in ("no_child_started", "preserved_originals", "preserved_workers")) + assert "private-subprocess-data" not in json.dumps(result) and "private-resources.json" not in json.dumps(result) + + +def _diagnostic_generation(root: Path, name: str = "A") -> Generation: + generation = Generation(name, root / name, "component-container", []) + generation.root.mkdir(parents=True) + for directory in ("state", "journal"): + generation.path(directory).mkdir() + return generation + + +def test_failure_sidecar_metadata_identifies_nonempty_shm_without_reading_stores(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + generation = _diagnostic_generation(tmp_path) + selected = generation.path("journal") / "management-operations.sqlite3-shm" + selected.write_bytes(b"private SQL and OAuth data must never be exported") + (generation.path("journal") / "management-operations.sqlite3-wal").touch() + (generation.path("state") / "oauth.sqlite3-shm").write_bytes(b"unrelated database") + (generation.path("state") / "management-child.sqlite3").write_bytes(b"private original database") + before = selected.stat() + with monkeypatch.context() as bounded: + bounded.setattr(Path, "read_bytes", lambda _: pytest.fail("sidecar diagnostics must not read file contents")) + bounded.setattr(fixture.sqlite3, "connect", lambda *_args, **_kwargs: pytest.fail("sidecar diagnostics must not connect to SQLite")) + rows = fixture._original_sidecar_diagnostics(tmp_path, (generation,)) + assert len(rows) == 15 + shm = next(row for row in rows if row["database"] == "management-operations.sqlite3" and row["suffix"] == "-shm") + assert shm == {"generation": "A", "database": "management-operations.sqlite3", "suffix": "-shm", "status": "observed", "type": "regular", "size": before.st_size, "device": before.st_dev, "inode": before.st_ino} + wal = next(row for row in rows if row["database"] == "management-operations.sqlite3" and row["suffix"] == "-wal") + assert wal["status"] == "observed" and wal["size"] == 0 + assert sum(row["status"] == "absent" for row in rows) == 13 + assert selected.stat() == before + assert "private" not in json.dumps(rows) and "oauth" not in json.dumps(rows) and str(tmp_path) not in json.dumps(rows) + + +def test_failure_sidecar_diagnostics_do_not_follow_links_or_export_other_scopes(tmp_path: Path) -> None: + generation = _diagnostic_generation(tmp_path) + private = tmp_path / "private-target" + private.write_bytes(b"not inspected") + sidecar = generation.path("state") / "management-child.sqlite3-shm" + sidecar.symlink_to(private) + (generation.path("journal") / "management-operations.sqlite3-journal").mkdir() + unrelated = _diagnostic_generation(tmp_path, "recovery-target") + rows = fixture._original_sidecar_diagnostics(tmp_path, (generation, unrelated)) + assert len(rows) == 15 and {row["generation"] for row in rows} == {"A"} + linked = next(row for row in rows if row.get("type") == "symlink") + assert linked["inode"] == sidecar.lstat().st_ino and linked["inode"] != private.stat().st_ino + assert any(row.get("type") == "directory" for row in rows) + assert "private-target" not in json.dumps(rows) + + +def test_failure_sidecar_diagnostics_are_finite_and_reject_ambiguous_original_roots(tmp_path: Path) -> None: + generations = tuple(_diagnostic_generation(tmp_path, name) for name in ("A", "B", "C", "D")) + assert len(fixture._original_sidecar_diagnostics(tmp_path, generations)) == 54 + ambiguous = fixture._original_sidecar_diagnostics(tmp_path, generations + (generations[0],) * 20) + assert len(ambiguous) == 54 + assert all(row["status"] == "unavailable" and row["error_type"] == "ValueError" for row in ambiguous if row["generation"] == "A") + foreign = Generation("A", tmp_path / "not-original-A", "component-container", []) + assert all(row["status"] == "unavailable" for row in fixture._original_sidecar_diagnostics(tmp_path, (foreign,))) + + +def test_failure_namespace_diagnostics_name_only_bounded_unexpected_sqlite_objects(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + generation = _diagnostic_generation(tmp_path) + state = generation.path("state") + (state / "management-readiness.sqlite3").write_bytes(b"observation must remain private") + (state / "unexpected.sqlite3").write_bytes(b"private unknown data") + (state / "directory.sqlite3").mkdir() + (state / "linked.sqlite3").symlink_to(tmp_path / "private-missing-target") + (state / "unrelated-config.json").write_bytes(b"private configuration") + with monkeypatch.context() as bounded: + bounded.setattr(Path, "read_bytes", lambda _: pytest.fail("namespace diagnostics must not read file contents")) + bounded.setattr(fixture.sqlite3, "connect", lambda *_args, **_kwargs: pytest.fail("namespace diagnostics must not connect to SQLite")) + rows = fixture._original_namespace_diagnostics(tmp_path, (generation,)) + assert len(rows) == 2 + row = next(row for row in rows if row["directory"] == "state") + assert row["status"] == "observed" and row["type"] == "directory" + assert {item["basename"]: item["type"] for item in row["unexpected"]} == {"unexpected.sqlite3": "regular", "directory.sqlite3": "directory", "linked.sqlite3": "symlink"} + assert "private" not in json.dumps(rows) and str(tmp_path) not in json.dumps(rows) + for index in range(40): + (state / f"extra-{index}.sqlite3").touch() + row = next(row for row in fixture._original_namespace_diagnostics(tmp_path, (generation,)) if row["directory"] == "state") + assert row["truncated"] is True and len(row["unexpected"]) == 16 + + +def test_failure_namespace_diagnostics_preserve_c_readiness_absence_and_exact_roots(tmp_path: Path) -> None: + generation = _diagnostic_generation(tmp_path, "C") + (generation.path("state") / "management-readiness.sqlite3").touch() + rows = fixture._original_namespace_diagnostics(tmp_path, (generation,)) + assert rows[0]["unexpected"] == [{"basename": "management-readiness.sqlite3", "type": "regular"}] + assert all(row["status"] == "unavailable" for row in fixture._original_namespace_diagnostics(tmp_path, (generation, generation))) + generation.path("journal").rmdir() + generation.path("journal").symlink_to(generation.path("state")) + row = fixture._original_namespace_diagnostics(tmp_path, (generation,))[1] + assert row["status"] == "unavailable" and row["type"] == "symlink" and "unexpected" not in row + + +def test_unavailable_sidecar_diagnostics_preserve_the_original_failure(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + chain = object.__new__(fixture.Chain) + chain.root = tmp_path + chain.generations = [_diagnostic_generation(tmp_path)] + result = {"status": "failed", "failure_type": "AssertionError"} + + def unavailable(*_args): + raise PermissionError("private diagnostic input must not be exported") + + monkeypatch.setattr(fixture, "_original_sidecar_diagnostics", unavailable) + original = AssertionError("the original native oracle failed") + with pytest.raises(AssertionError) as caught: + try: + raise original + except AssertionError: + fixture._record_failure_sidecars(result, chain) + raise + assert caught.value is original + assert result["original_database_sidecars"] == [{"status": "unavailable", "error_type": "PermissionError"}] + assert "private" not in json.dumps(result) + + +def test_failure_catalog_delta_names_exact_units_without_exporting_manifest_contents(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(fixture, "RECOVERY_ENROLLMENT_ROOT", tmp_path) + path = tmp_path / "component.routes.json" + path.write_bytes(canonical({"enrollment_id": "component", "systemd_routes": [{"unit": name, "sha256": "a" * 64, "role": "unrelated"} for name in ("kept.service", "removed.mount")], "private_other_data": "private-source-credential"})) + path.chmod(0o600) + before = path.read_bytes(), path.stat() + monkeypatch.setattr(fixture, "_systemd_catalog", lambda: ("added.mount", "kept.service")) + result = {"status": "failed", "failure_type": "AssertionError"} + fixture._record_failure_catalog(result, "component") + assert result["systemd_catalog_after_failure"] == {"status": "observed-after-failure", "approved_count": 2, "observed_count": 2, "added_count": 1, "removed_count": 1, "added": ["added.mount"], "removed": ["removed.mount"], "truncated": False} + assert "private" not in json.dumps(result) and str(tmp_path) not in json.dumps(result) + after = path.stat() + assert path.read_bytes() == before[0] and all(getattr(after, field) == getattr(before[1], field) for field in ("st_dev", "st_ino", "st_ctime_ns", "st_mtime_ns", "st_size", "st_mode")) and result["status"] == "failed" + monkeypatch.setattr(fixture, "_systemd_catalog", lambda: tuple(f"new-{index:02d}.mount" for index in range(50))) + fixture._record_failure_catalog(result, "component") + observed = result["systemd_catalog_after_failure"] + assert observed["added_count"] == 50 and len(observed["added"]) == 32 and observed["truncated"] is True + + +def test_failure_catalog_diagnostic_errors_never_replace_original_failure(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(fixture, "RECOVERY_ENROLLMENT_ROOT", tmp_path) + result = {"status": "failed", "failure_type": "AssertionError"} + original = AssertionError("original native refusal") + with pytest.raises(AssertionError) as caught: + try: + raise original + except AssertionError: + fixture._record_failure_catalog(result, "../private") + raise + assert caught.value is original + assert result["systemd_catalog_after_failure"] == {"status": "unavailable", "error_type": "ValueError"} + assert "private" not in json.dumps(result) + fixture._record_failure_catalog(result, "missing") + assert result["systemd_catalog_after_failure"] == {"status": "unavailable", "error_type": "FileNotFoundError"} + + +def test_no_child_or_admission_oracle_rejects_any_native_store(tmp_path: Path) -> None: + state, journal = tmp_path / "state", tmp_path / "journal" + state.mkdir() + journal.mkdir() + (state / ".supervisor-owner.lock").write_bytes(b"") + (state / "logs").mkdir() + generation = Generation("target", tmp_path, "a" * 64, []) + fixture._no_target_child(generation, running=False) + (state / "management-child.sqlite3").write_bytes(b"not allowed") + with pytest.raises(AssertionError, match="native consumer state"): + fixture._no_target_child(generation, running=False) + (state / "management-child.sqlite3").unlink() + (journal / "management-operations.sqlite3").write_bytes(b"not allowed") + with pytest.raises(AssertionError, match="admitted an operation"): + fixture._no_target_child(generation, running=False) + + +@pytest.mark.parametrize("inventory", ["a" * 64, "truncated-id", "\n".join(["b" * 64, "b" * 64])]) +def test_removed_target_oracle_rejects_presence_or_incomplete_inventory(monkeypatch: pytest.MonkeyPatch, inventory: str) -> None: + monkeypatch.setattr(fixture.Chain, "docker", lambda *_args: inventory) + with pytest.raises(AssertionError): + fixture._require_target_absent("a" * 64) + + +def test_removed_target_oracle_compares_exact_full_ids(monkeypatch: pytest.MonkeyPatch) -> None: + calls = [] + + def docker(*arguments): + calls.append(arguments) + return "a" * 63 + "b" + + monkeypatch.setattr(fixture.Chain, "docker", docker) + fixture._require_target_absent("a" * 64) + assert calls == [("ps", "--all", "--quiet", "--no-trunc")] + + +def test_original_fingerprint_binds_inodes_and_bytes(tmp_path: Path) -> None: + path = tmp_path / "native-original" + path.write_bytes(b"original") + before = fixture._snapshot((tmp_path,)) + assert fixture._snapshot((tmp_path,)) == before + other = tmp_path / "replacement" + other.write_bytes(path.read_bytes()) + os.replace(other, path) + assert fixture._snapshot((tmp_path,)) != before + + +def _workers(root: Path) -> None: + for channel in ("beta", "release", "agent-table"): + parent = root / channel / "supervisor" + parent.mkdir(parents=True) + (parent / "state.json").write_bytes( + canonical({"schema_version": 1, "operations": {}, "selected_runtime_id": "bootstrap", "worker": {"pid": 2, "start_token": "1000", "process_run_id": channel + "-run", "deployment_generation": channel + "-deployment", "runtime_id": "bootstrap", "binding_observed_at": 1.0}}) + ) + for name in ("compose.yml", "beta-secret", "release-secret", "agent-table-secret"): + (root / name).write_bytes(b"original-component-input") + + +def test_worker_claim_oracle_preserves_effects_and_excludes_only_observation_refresh(tmp_path: Path) -> None: + _workers(tmp_path) + baseline = fixture._worker_claims(tmp_path) + path = tmp_path / "beta" / "supervisor" / "state.json" + state = json.loads(path.read_bytes()) + state["worker"]["binding_observed_at"] = 2.0 + replacement = path.with_name("replacement") + replacement.write_bytes(canonical(state)) + os.replace(replacement, path) + assert fixture._worker_claims(tmp_path) == baseline + state["operations"]["new-operation"] = {"status": "running"} + path.write_bytes(canonical(state)) + assert fixture._worker_claims(tmp_path) != baseline + + +def test_worker_claim_oracle_requires_real_nonempty_ownership(tmp_path: Path) -> None: + _workers(tmp_path) + path = tmp_path / "beta" / "supervisor" / "state.json" + state = json.loads(path.read_bytes()) + state["worker"]["process_run_id"] = None + path.write_bytes(canonical(state)) + with pytest.raises(AssertionError, match="ownership evidence"): + fixture._worker_claims(tmp_path) + + +def _durable(root: Path) -> tuple[str, str]: + # Deliberately synthetic pure oracle input; the VM driver obtains these + # records only from the actual production issuer and genuine peer logs. + inventory = canonical({"component": "inventory"}) + (root / "inventory.json").write_bytes(inventory) + inventory_sha = hashlib.sha256(inventory).hexdigest() + evidence = canonical({"lineage": {"unavailable": [], "genesis": {"component": "origin"}}, "inspection": {"unresolved_targets": 1, "historical_keys": 3, "launch_authority": False}, "target": {"receipt": {"source_sha": "a" * 40}}}).decode() + binding = {"evidence_json": evidence, "evidence_sha256": hashlib.sha256(evidence.encode()).hexdigest()} + previous = "" + for sequence, phase in enumerate(("reserved", "intent", "consumed", "receipt")): + record = canonical({"schema_version": 1, "phase": phase, "sequence": sequence, "previous": previous, "inventory_sha256": inventory_sha, "binding": binding}) + (root / (phase + ".json")).write_bytes(record) + head = canonical({"schema_version": 1, "phase": phase, "record_sha256": hashlib.sha256(record).hexdigest(), "previous": previous, "inventory_sha256": inventory_sha}) + (root / (phase + "-head.json")).write_bytes(head) + previous = hashlib.sha256(head).hexdigest() + return "One consumed recovery epoch grant send completed", "Recovery custody acknowledged" + + +def test_durable_oracle_requires_all_heads_and_actual_consumer_acknowledgment(tmp_path: Path) -> None: + issuer, consumer = _durable(tmp_path) + result = fixture.inspect_durable_result(tmp_path, issuer_log=issuer, consumer_log=consumer) + assert result["unresolved_targets"] == 1 and result["historical_keys"] == 3 + with pytest.raises(AssertionError): + fixture.inspect_durable_result(tmp_path, issuer_log=issuer, consumer_log="consumer waiting") + (tmp_path / "receipt-head.json").unlink() + with pytest.raises(FileNotFoundError): + fixture.inspect_durable_result(tmp_path, issuer_log=issuer, consumer_log=consumer) + + +def test_durable_oracle_rejects_a_changed_original_record(tmp_path: Path) -> None: + issuer, consumer = _durable(tmp_path) + path = tmp_path / "consumed.json" + value = json.loads(path.read_bytes()) + value["binding"]["evidence_sha256"] = "b" * 64 + path.write_bytes(canonical(value)) + with pytest.raises(AssertionError): + fixture.inspect_durable_result(tmp_path, issuer_log=issuer, consumer_log=consumer) + + +def _result() -> dict[str, object]: + return { + "schema": fixture.RESULT_SCHEMA, + "status": "passed", + "cases": sorted(fixture.required_cases("grant")), + "no_child_started": True, + "preserved_originals": True, + "preserved_workers": True, + "original_outcomes": {"C": "launched", "D": "never-created"}, + "durable": {"record_sha256": {name: "a" * 64 for name in ("reserved", "intent", "consumed", "receipt")}}, + "peer_observations": { + "schema": "actual-recovery-native-peer-faults-v1", + "cases": [ + { + "case": case, + "issuer_returncode": 2, + "epoch_unreserved": True, + "issuer_endpoint_removed_by_original": True, + "consumer_grant_accepted": False, + "no_child_or_admission": True, + "issuer_syscall": {"bytes_observed": 10}, + "receiver_alive_until_issuer_refusal": True, + "issuer_refused_before_receiver_termination": True, + } + for case in sorted(fixture.REQUIRED_PEER_CASES) + ], + }, + "factory_override_observations": { + "schema": "actual-recovery-factory-overrides-v1", + "cases": [{"case": case, "returncode": 2, "refusal": "component refusal", "alternate_inputs_sha256": "d" * 64, "epoch_sha256": "e" * 64} for case in sorted(fixture.FACTORY_OBSERVATION_CASES)], + }, + "task_observations": { + "schema": "actual-recovery-task-observations-v1", + "cases": [ + {"case": "escaped-old-network-nonleader-refused"}, + {"case": "escaped-nonleader-current-containment-refused", "status": "unavailable"}, + { + "case": "measured-exited-nonleader-current-containment-reverified", + "status": "observed-current-recovery-containment", + "after_exit": {"pidfd_readable": True, "nonleader_absent": True, "leader": {"state": "Z", "network_namespace": None}}, + "after_verification": {"pidfd_readable": True, "nonleader_absent": True, "leader": {"state": "Z", "network_namespace": None}}, + }, + ], + }, + } + + +@pytest.mark.parametrize("case", sorted(fixture.required_cases("grant"))) +def test_native_result_cannot_omit_any_required_case(case: str) -> None: + result = _result() + fixture.validate_result(result) + result["cases"].remove(case) + with pytest.raises(AssertionError): + fixture.validate_result(result) + + +@pytest.mark.parametrize("field", ["no_child_started", "preserved_originals", "preserved_workers"]) +def test_native_result_cannot_claim_success_without_preservation(field: str) -> None: + result = _result() + result[field] = False + with pytest.raises(AssertionError): + fixture.validate_result(result) + + +def _lost_reply_durable() -> dict[str, object]: + lifetimes = {"target_pid": 321, "target_start_token": "12345", "issuer_pid": 654, "issuer_start_token": "54321"} + grant = {"schema": fixture.GRANT_SCHEMA, "target_id": "a" * 64, "host_pid": lifetimes["issuer_pid"], "host_start_token": lifetimes["issuer_start_token"]} + encoded = canonical(grant) + b"\n" + return { + "phase": "receipt", + "consumer_grant_acknowledged": False, + "grant_send_completed": True, + "receiver_paused_before_send": True, + "issuer_died_before_receiver_resumed": True, + "native_observation": {**lifetimes, "issuer_death_verified_before_receiver_resumed": True}, + "queued_grant": { + **lifetimes, + "schema": "actual-recovery-queued-grant-v1", + "grant_schema": fixture.GRANT_SCHEMA, + "target_id": grant["target_id"], + "grant": grant, + "message_sha256": hashlib.sha256(encoded).hexdigest(), + "bytes_observed": len(encoded), + "socket_inode": 789, + "socket_flags_before": 2048, + "socket_flags_after": 2048, + "peer_credentials": {"pid": lifetimes["issuer_pid"], "uid": 0, "gid": 0}, + "peek_only": True, + "socket_flags_unchanged": True, + "bytes_still_queued": True, + "receiver_stopped": True, + "issuer_alive": True, + }, + } + + +@pytest.mark.parametrize("scenario,phase", [("kill-after-intent", "intent"), ("kill-after-consume", "consumed"), ("lost-reply", "receipt")]) +def test_interruption_result_requires_the_actual_boundary_and_no_grant(scenario: str, phase: str) -> None: + result = _result() + durable = _lost_reply_durable() if scenario == "lost-reply" else {"phase": phase, "consumer_grant_acknowledged": False, "grant_send_completed": False} + result.update(scenario=scenario, cases=sorted(fixture.required_cases(scenario)), durable=durable) + fixture.validate_result(result) + result["durable"]["consumer_grant_acknowledged"] = True + with pytest.raises(AssertionError): + fixture.validate_result(result) + result["durable"] = {"phase": "reserved", "consumer_grant_acknowledged": False} + with pytest.raises(AssertionError): + fixture.validate_result(result) + + +def test_unknown_native_scenario_is_not_a_success_alias() -> None: + with pytest.raises(ValueError, match="unsupported"): + fixture.required_cases("unreviewed-boundary") + + +def test_canonical_acceptance_contains_all_distinct_publication_and_late_faults() -> None: + assert len(set(fixture.ACCEPTANCE_SCENARIOS)) == len(fixture.ACCEPTANCE_SCENARIOS) == 31 + assert set(fixture.ACCEPTANCE_SCENARIOS) == {"grant", "lost-reply", "retarget-after-reservation", *fixture.BOUNDARY_SCENARIOS, *fixture.LATE_MUTATION_SCENARIOS, *fixture.POST_RECEIPT_MUTATION_SCENARIOS} + assert fixture.ACCEPTANCE_SCENARIOS[: 1 + len(fixture.POST_RECEIPT_MUTATION_SCENARIOS)] == ("grant", *fixture.POST_RECEIPT_MUTATION_SCENARIOS) + assert fixture.CONTROL not in fixture.ACCEPTANCE_SCENARIOS + jobs = yaml.safe_load((Path(__file__).resolve().parents[1] / ".github/workflows/management-recovery-epoch.yml").read_text())["jobs"] + assert jobs["recovery-epoch"]["strategy"]["matrix"]["scenario"] == "${{ fromJSON(needs.acceptance-plan.outputs.scenarios) }}" + assert jobs["verifier-omission-control"]["env"]["RECOVERY_SCENARIO"] in fixture.POST_RECEIPT_MUTATION_SCENARIOS + assert jobs["recovery-epoch"]["env"]["RECOVERY_FIXTURE"] != jobs["verifier-omission-control"]["env"]["RECOVERY_FIXTURE"] + assert set(jobs["acceptance"]["needs"]) == {"acceptance-plan", "recovery-epoch", "verifier-omission-control"} + assert set(fixture.SCENARIOS) - set(fixture.ACCEPTANCE_SCENARIOS) == {"kill-after-intent", "kill-after-consume"} + assert fixture._selected_boundary("post-receipt-ready-mutation") == ("receipt-head.json", "after") + assert all(fixture._selected_boundary(name) == ("intent-head.json", "after") for name in fixture.LATE_MUTATION_SCENARIOS) + assert fixture._selected_boundary("kill-after-intent") == fixture._selected_boundary("after-intent-head") == ("intent-head.json", "after") + assert fixture._selected_boundary("kill-after-consume") == fixture._selected_boundary("after-consumed-head") == ("consumed-head.json", "after") + assert fixture._selected_boundary("lost-reply") == ("receipt-head.json", "after") + + +@pytest.mark.parametrize("field", ["grant_send_completed", "receiver_paused_before_send", "issuer_died_before_receiver_resumed"]) +def test_lost_reply_requires_actual_send_before_issuer_death(field: str) -> None: + result = _result() + result.update(scenario="lost-reply", cases=sorted(fixture.required_cases("lost-reply")), durable=_lost_reply_durable()) + fixture.validate_result(result) + result["durable"][field] = False + with pytest.raises(AssertionError): + fixture.validate_result(result) + + +@pytest.mark.parametrize("field", ["peek_only", "socket_flags_unchanged", "bytes_still_queued", "receiver_stopped", "issuer_alive"]) +def test_lost_reply_requires_nonconsuming_current_kernel_observation(field: str) -> None: + result = _result() + result.update(scenario="lost-reply", cases=sorted(fixture.required_cases("lost-reply")), durable=_lost_reply_durable()) + fixture.validate_result(result) + result["durable"]["queued_grant"][field] = False + with pytest.raises(AssertionError): + fixture.validate_result(result) + + +@pytest.mark.parametrize("mutation", ["missing", "foreign-peer", "foreign-lifetime", "changed-flags", "empty-queue", "issuer-still-alive", "changed-packet", "foreign-grant"]) +def test_lost_reply_log_cannot_replace_exact_queued_bytes_and_lifetimes(mutation: str) -> None: + result = _result() + result.update(scenario="lost-reply", cases=sorted(fixture.required_cases("lost-reply")), durable=_lost_reply_durable()) + if mutation == "missing": + result["durable"].pop("queued_grant") + elif mutation == "foreign-peer": + result["durable"]["queued_grant"]["peer_credentials"]["pid"] += 1 + elif mutation == "foreign-lifetime": + result["durable"]["queued_grant"]["target_start_token"] = "99999" + elif mutation == "changed-flags": + result["durable"]["queued_grant"]["socket_flags_after"] += 1 + elif mutation == "empty-queue": + result["durable"]["queued_grant"]["bytes_observed"] = 0 + elif mutation in {"changed-packet", "foreign-grant"}: + queued = result["durable"]["queued_grant"] + queued["grant"]["target_id"] = "b" * 64 + if mutation == "foreign-grant": + encoded = canonical(queued["grant"]) + b"\n" + queued.update(message_sha256=hashlib.sha256(encoded).hexdigest(), bytes_observed=len(encoded)) + else: + result["durable"]["native_observation"]["issuer_death_verified_before_receiver_resumed"] = False + with pytest.raises((AssertionError, KeyError)): + fixture.validate_result(result) + + +@pytest.mark.parametrize("case", sorted(fixture.REQUIRED_PEER_CASES)) +def test_grant_result_requires_the_actual_peer_evidence_inventory(case: str) -> None: + result = _result() + fixture.validate_result(result) + result["peer_observations"]["cases"] = [item for item in result["peer_observations"]["cases"] if item["case"] != case] + with pytest.raises(AssertionError): + fixture.validate_result(result) + + +@pytest.mark.parametrize("field", ["receiver_alive_until_issuer_refusal", "issuer_refused_before_receiver_termination"]) +@pytest.mark.parametrize("case", ["queued-rights-without-receipt-refused", "received-rights-without-receipt-refused"]) +def test_missing_peer_receipt_cannot_be_masked_by_an_already_dead_receiver(case: str, field: str) -> None: + result = _result() + fixture.validate_result(result) + selected = next(item for item in result["peer_observations"]["cases"] if item["case"] == case) + selected[field] = False + with pytest.raises(AssertionError): + fixture.validate_result(result) + + +@pytest.mark.parametrize("name", ["factory_override_observations", "task_observations"]) +def test_native_factory_and_task_measurements_are_required(name: str) -> None: + result = _result() + fixture.validate_result(result) + result.pop(name) + with pytest.raises(AssertionError): + fixture.validate_result(result) + + +def test_exited_task_success_requires_the_same_observed_zombie() -> None: + result = _result() + fixture.validate_result(result) + selected = next(item for item in result["task_observations"]["cases"] if item["case"] == "measured-exited-nonleader-current-containment-reverified") + selected["after_verification"]["leader"]["state"] = "R" + with pytest.raises(AssertionError): + fixture.validate_result(result) + + +@pytest.mark.parametrize("scenario", fixture.LATE_MUTATION_SCENARIOS) +def test_late_mutation_result_names_its_deliberate_preservation_boundary(scenario: str) -> None: + result = _result() + result.update( + scenario=scenario, + cases=sorted(fixture.required_cases(scenario)), + preservation_baseline="after-deliberate-fixture-mutation", + preserved_originals_sha256="c" * 64, + durable={"phase": "intent", "consumer_grant_acknowledged": False, "grant_send_completed": False, "mutation": {"scenario": scenario, "boundary": "after-intent-head", "preservation_baseline": "after-deliberate-fixture-mutation"}, "originals_after_injection_sha256": "c" * 64}, + ) + if scenario == "late-target-loss": + result["durable"]["mutation"]["target_fault"] = {"pidfd_death_observed": True, "exact_docker_id_absent": True, "docker_rm_returncode": 0, "other_docker_ids_preserved": True} + elif scenario == "late-original-attempt": + result["durable"]["mutation"].update( + original_attempt={"instance_id": "D", "outcome": "never-created", "terminal_kind": "never-created", "independent_lineage_verified": True, "retained_intent_lineage_matched": True, "original_file": {"path": "/component/history/00000007.json", "device": 1, "inode": 2, "sha256": "a" * 64}}, + file_fault={"path": "/component/history/00000007.json", "before": [1, 2, "a" * 64]}, + ) + fixture.validate_result(result) + result["preservation_baseline"] = "before-deliberate-fixture-mutation" + with pytest.raises(AssertionError): + fixture.validate_result(result) + result["preservation_baseline"] = "after-deliberate-fixture-mutation" + result["durable"]["originals_after_injection_sha256"] = "d" * 64 + with pytest.raises(AssertionError): + fixture.validate_result(result) + + +def _post_receipt_result() -> dict[str, object]: + result = _result() + scenario = "post-receipt-ready-mutation" + result.update( + scenario=scenario, + cases=sorted(fixture.required_cases(scenario)), + preservation_baseline="after-deliberate-fixture-mutation", + preserved_originals_sha256="c" * 64, + durable={ + "phase": "receipt", + "consumer_grant_acknowledged": False, + "grant_send_completed": False, + "mutation": {"scenario": scenario, "boundary": "after-receipt-head", "preservation_baseline": "after-deliberate-fixture-mutation", "issuer_pid": 123, "issuer_start_token": "456", "file_fault": {"before": [1, 2, "a" * 64], "after": [1, 2, "b" * 64]}}, + "originals_after_injection_sha256": "c" * 64, + "dispatch": { + "schema": "actual-recovery-post-receipt-dispatch-v1", + "issuer_pid": 123, + "issuer_start_token": "456", + "receiver_stopped": True, + "queue_empty_before_resume": True, + "socket_flags_unchanged": True, + "queue_bytes_observed": 0, + "queued_grant": None, + "issuer_returncode": 2, + "issuer_death_before_receiver_resumed": True, + }, + "replay_returncode": 2, + "retained_publications": list(fixture.PUBLICATIONS), + "epoch_sha256_before_resume": "d" * 64, + "epoch_sha256_after_refusal": "d" * 64, + "epoch_sha256_after_replay": "d" * 64, + "publication_boundary": {"name": "receipt-head.json", "when": "after", "file_fsync_observed": True, "directory_fsync_observed": True}, + }, + ) + return result + + +def test_post_receipt_mutation_result_requires_current_refusal_and_consumed_history() -> None: + fixture.validate_result(_post_receipt_result()) + + +@pytest.mark.parametrize("mutation", ["intent-boundary", "no-fsync", "lost-receipt", "repaired-epoch", "replayed-epoch", "replay-accepted", "issuer-killed", "queued-data", "receiver-running", "unchanged-ready", "foreign-issuer"]) +def test_post_receipt_result_cannot_substitute_kill_only_or_earlier_boundary(mutation: str) -> None: + result = _post_receipt_result() + durable = result["durable"] + if mutation == "intent-boundary": + durable["publication_boundary"]["name"] = "intent-head.json" + elif mutation == "no-fsync": + durable["publication_boundary"]["directory_fsync_observed"] = False + elif mutation == "lost-receipt": + durable["retained_publications"].pop() + elif mutation == "repaired-epoch": + durable["epoch_sha256_after_refusal"] = "e" * 64 + elif mutation == "replayed-epoch": + durable["epoch_sha256_after_replay"] = "e" * 64 + elif mutation == "replay-accepted": + durable["replay_returncode"] = 0 + elif mutation == "issuer-killed": + durable["dispatch"]["issuer_returncode"] = -9 + elif mutation == "queued-data": + durable["dispatch"]["queue_bytes_observed"] = 1 + elif mutation == "receiver-running": + durable["dispatch"]["receiver_stopped"] = False + elif mutation == "unchanged-ready": + durable["mutation"]["file_fault"]["after"] = durable["mutation"]["file_fault"]["before"] + else: + durable["dispatch"]["issuer_start_token"] = "999" + with pytest.raises(AssertionError): + fixture.validate_result(result) + + +@pytest.mark.parametrize("field", ["pidfd_death_observed", "exact_docker_id_absent", "other_docker_ids_preserved"]) +def test_late_target_loss_requires_death_absence_and_preserved_other_objects(field: str) -> None: + scenario = "late-target-loss" + result = _result() + result.update( + scenario=scenario, + cases=sorted(fixture.required_cases(scenario)), + preservation_baseline="after-deliberate-fixture-mutation", + preserved_originals_sha256="c" * 64, + durable={ + "phase": "intent", + "consumer_grant_acknowledged": False, + "grant_send_completed": False, + "mutation": {"scenario": scenario, "boundary": "after-intent-head", "preservation_baseline": "after-deliberate-fixture-mutation", "target_fault": {"pidfd_death_observed": True, "exact_docker_id_absent": True, "docker_rm_returncode": 0, "other_docker_ids_preserved": True}}, + "originals_after_injection_sha256": "c" * 64, + }, + ) + fixture.validate_result(result) + result["durable"]["mutation"]["target_fault"][field] = False + with pytest.raises(AssertionError): + fixture.validate_result(result) + + +@pytest.mark.parametrize("scenario", fixture.BOUNDARY_SCENARIOS) +def test_publication_result_requires_exact_kernel_observation(scenario: str) -> None: + publication, when = fixture.parse_boundary(scenario) + result = _result() + result.update(scenario=scenario, cases=sorted(fixture.required_cases(scenario)), durable={"publication": publication, "when": when, "consumer_grant_acknowledged": False, "file_fsync_observed": when == "after", "directory_fsync_observed": when == "after"}) + fixture.validate_result(result) + result["durable"]["directory_fsync_observed"] = when != "after" + with pytest.raises(AssertionError): + fixture.validate_result(result) + + +def test_retarget_result_cannot_hide_unchanged_inventory() -> None: + result = _result() + result.update(scenario="retarget-after-reservation", cases=sorted(fixture.required_cases("retarget-after-reservation")), durable={"phase": "reserved", "consumer_grant_acknowledged": False, "inventory_sha256": "a" * 64, "retargeted_inventory_sha256": "a" * 64}) + with pytest.raises(AssertionError): + fixture.validate_result(result) + result["durable"]["retargeted_inventory_sha256"] = "b" * 64 + fixture.validate_result(result) + + +def test_semantic_preservation_excludes_only_deliberate_rename_metadata(tmp_path: Path) -> None: + original = tmp_path / "original" + original.mkdir() + database = original / "database" + database.write_bytes(b"retained bytes") + baseline = fixture._snapshot((original,), semantic=True) + held = tmp_path / "held-original" + database.rename(held) + assert fixture._snapshot((original,), semantic=True) != baseline + held.rename(database) + assert fixture._snapshot((original,), semantic=True) == baseline + database.write_bytes(b"changed bytes") + assert fixture._snapshot((original,), semantic=True) != baseline diff --git a/tests/test_management_recovery_epoch_join.py b/tests/test_management_recovery_epoch_join.py new file mode 100644 index 000000000..fd2b9cf02 --- /dev/null +++ b/tests/test_management_recovery_epoch_join.py @@ -0,0 +1,426 @@ +"""Authority-join component contracts, not original or live host authority. + +The lineage and containment dataclasses below are explicit component values. +Their agreement tests the issuer's cross-producer joins, not their provenance. +Consumer-scope tests additionally use real protected files and held kernel FDs; +no fixture result substitutes for production Docker or current T observations. +""" + +from __future__ import annotations + +import fcntl +import hashlib +import os +import pwd +import stat +import tempfile +from contextlib import ExitStack, contextmanager +from dataclasses import asdict, dataclass, replace +from pathlib import Path +from typing import Iterator + +import pytest + +from auto_coder.management.containment_enrollment import OriginalNetworkPin +from auto_coder.management.containment_observation import DockerObservation +from auto_coder.management.containment_route_manifest import ApprovedRouteManifest, DeploymentRootBinding, RouteManifestObservation +from auto_coder.management.original_attempt_evidence import OriginalGeneration, OriginalLineageEvidence +from auto_coder.management.original_attempts import HistoryBinding, canonical, source_identity +from auto_coder.management.original_genesis import GenesisAnchor, GenesisRoot +from auto_coder.management.recovery_containment import RecoveryContainmentFacts, RecoveryContainmentReport, RecoveryTargetFact +from auto_coder.management.recovery_containment_model import NamedDescriptor, RecoveryEnrollment, RecoveryEnrollmentObservation, RecoveryFile, RecoveryLock, RecoveryOriginal, RecoveryResources, RecoveryTarget +from auto_coder.management.recovery_containment_profile import RecoveryManagerFact +from auto_coder.management.recovery_containment_resources import RecoveryResourceObservation +from auto_coder.management.recovery_epoch import _check_consumer_scope, join_lineage +from auto_coder.management.recovery_epoch_authorization import PEER_ROOT, EpochAuthorization +from auto_coder.management.recovery_epoch_consumer import ConsumerConfiguration, OriginalFence, TargetOwner, TransferredResource +from auto_coder.management.recovery_epoch_peer import HeldFence +from auto_coder.management.recovery_epoch_resources import EpochResources +from auto_coder.management.recovery_epoch_state import EpochInventory, EpochUnavailable, issuer_lifetime + +pytestmark = pytest.mark.usefixtures("_use_real_commands", "_use_real_sleep") +_BOOT = "00000000-0000-4000-8000-000000000001" +_IMAGE = "sha256:" + "a" * 64 +_OAUTH = "/component/oauth" + + +def component_file(path: str, inode: int) -> RecoveryFile: + return RecoveryFile(path, format(inode, "064x"), 1, inode) + + +def component_generation(index: int, source: str, *, never: bool = False) -> tuple[OriginalGeneration, RecoveryOriginal, RecoveryManagerFact]: + manager, proxy = format(2 * index + 1, "064x"), format(2 * index + 2, "064x") + base = "/component/generation-" + str(index) + state, journal = base + "/state", base + "/journal" + bootstrap, management = component_file(base + "/config.json", 100 + index), component_file(base + "/management.yaml", 200 + index) + owner = None if never else RecoveryLock(state + "/.supervisor-owner.lock", 1, 300 + index) + mounts = ( + DeploymentRootBinding(manager, "/management-child-state", state, True, 1, 400 + index, stat.S_IFDIR | 0o700), + DeploymentRootBinding(manager, "/management-journal", journal, True, 1, 500 + index, stat.S_IFDIR | 0o700), + DeploymentRootBinding(manager, "/management-oauth", _OAUTH, True, 1, 600, stat.S_IFDIR | 0o700), + ) + generation = OriginalGeneration( + f"00000000-0000-4000-8000-{index + 10:012d}", + f"instance-{index}", + manager, + _IMAGE, + bootstrap.sha256, + management.sha256, + state, + 1, + 400 + index, + journal, + 1, + 500 + index, + _OAUTH, + 1, + 600, + owner.path if owner else "", + owner.device if owner else 0, + owner.inode if owner else 0, + "never-created" if never else "unknown", + (), + proxy if index == 0 else "", + _IMAGE if index == 0 else "", + ) + original = RecoveryOriginal( + f"generation-{index}", + generation.instance_id, + manager, + _IMAGE, + proxy, + _IMAGE, + "never-started" if never else "started", + "" if never else _BOOT, + bootstrap, + management, + component_file(base + "/Caddyfile", 700 + index), + mounts, + owner, + None if never else component_file(base + "/prior.json", 800 + index), + None if never else OriginalNetworkPin(42, "12345", _BOOT, 10 + index, f"net:[{900 + index}]", 4, 900 + index), + ) + fact = RecoveryManagerFact(original.generation_id, generation.instance_id, manager, _IMAGE, original.state, source, "b" * 64, bootstrap.sha256, management.sha256, state, journal, _OAUTH, owner, 18080 + index, mounts) + return generation, original, fact + + +def component_docker(identifier: str) -> DockerObservation: + return DockerObservation("engine-one", identifier, _IMAGE, "c" * 64, "no", 0, False, "exited", 0, "old-start", "old-stop", 0, "d" * 64) + + +@dataclass(frozen=True) +class JoinCase: + authorization: EpochAuthorization + inventory: EpochInventory + enrollment: RecoveryEnrollmentObservation + lineage: OriginalLineageEvidence + containment: RecoveryContainmentReport + + def check(self) -> None: + join_lineage(self.authorization, self.inventory, self.enrollment, self.lineage, self.containment) + + def scope(self, selected: RecoveryEnrollment) -> JoinCase: + enrollment = replace(self.enrollment, enrollment=selected) + return replace(self, enrollment=enrollment, containment=replace(self.containment, facts=replace(self.containment.facts, enrollment=enrollment))) + + def managers(self, values: tuple[RecoveryManagerFact, ...]) -> JoinCase: + return replace(self, containment=replace(self.containment, facts=replace(self.containment.facts, managers=values))) + + +def component_join() -> JoinCase: + source = source_identity() + history = HistoryBinding("/component/history", "deployment-one", "history-one", source) + first, second, absent = (component_generation(index, source, never=index == 2) for index in range(3)) + target_id = "f" * 64 + target_roots = ( + DeploymentRootBinding(target_id, "/management-child-state", "/component/target/state", True, 1, 1001, stat.S_IFDIR | 0o700), + DeploymentRootBinding(target_id, "/management-journal", "/component/target/journal", True, 1, 1002, stat.S_IFDIR | 0o700), + DeploymentRootBinding(target_id, "/management-oauth", _OAUTH, True, 1, 600, stat.S_IFDIR | 0o700), + ) + target = RecoveryTarget("target-one", target_id, _IMAGE, component_file("/component/target/config.json", 1101), component_file("/component/target/management.yaml", 1102), target_roots, RecoveryLock("/component/target/state/.supervisor-owner.lock", 1, 1103), 18090) + selected = RecoveryEnrollment("approved-management-recovery-containment-v1", history.deployment, "enrollment-one", "authorization-one", "ordinary-recovery-v1", "engine-one", source, (first[1], second[1]), target, (), True) + enrollment = RecoveryEnrollmentObservation(selected, "1" * 64, 1, 1200, 100) + genesis = GenesisAnchor("ordinary-original-genesis-v1", history, "old-authorization", first[0].instance_id, "engine-one", first[0].management_id, _IMAGE, "2" * 64, first[1].proxy_id, _IMAGE, "3" * 64, (), (GenesisRoot("oauth", _OAUTH, "/management-oauth", 1, 600),), "{}") + lineage = OriginalLineageEvidence(history, genesis=genesis, frontier="4" * 64, generations=(first[0], second[0], absent[0])) + target_fact = RecoveryManagerFact( + "target", target.instance_id, target_id, _IMAGE, "prelaunch-target", source, "5" * 64, target.bootstrap_config.sha256, target.management_config.sha256, "/component/target/state", "/component/target/journal", _OAUTH, target.owner, target.published_port, target_roots + ) + docker = tuple(component_docker(identifier) for identifier in selected.container_ids) + manifest = ApprovedRouteManifest("approved-management-recovery-routes-v1", selected.deployment_id, selected.enrollment_id, selected.authorization_id, selected.profile, enrollment.sha256, True, True, (), selected.container_ids, (), (), (), ()) + routes = RouteManifestObservation(manifest, "6" * 64, 1, 1300, 100, (), (), (), selected.container_ids) + facts = RecoveryContainmentFacts( + enrollment, + source, + docker, + (first[2], second[2], target_fact), + (), + (), + RecoveryTargetFact(target_id, 101, "12345", _BOOT, "net:[1400]", "pid:[1401]", "/component/target-cgroup", 1, 1402, 1, target.owner.inode, target.published_port), + RecoveryResourceObservation(102, "12346", _BOOT, ()), + routes, + (), + (), + ) + containment = RecoveryContainmentReport("current-recovery-containment-v1", "observed-current-recovery-containment", "00000000-0000-4000-8000-000000000050", 102, "12346", _BOOT, 100, 200, facts, "7" * 64) + authorization = EpochAuthorization( + "management-recovery-authorization-v1", + selected.deployment_id, + selected.authorization_id, + "new-request", + "new-preparation", + target_id, + selected.enrollment_id, + history, + component_file("/component/target/consumer.json", 1500), + component_file("/component/target/READY.json", 1501), + "/component/source", + "/component/source-token", + "/usr/bin/python3", + "/usr/bin/uv", + "0.13.6", + "a" * 40, + 1, + 1600, + ) + inventory = EpochInventory( + 2, "/var/lib/auto-coder/recovery-epoch", 1, 1700, 1, 1701, selected.deployment_id, selected.authorization_id, (selected.enrollment_id,), authorization.request_id, authorization.preparation_id, target_id, source.removeprefix("sha256:"), target.bootstrap_config.sha256, "8" * 64 + ) + return JoinCase(authorization, inventory, enrollment, lineage, containment) + + +def test_agreeing_joins_preserve_unknowns_and_proven_absent_never_created_branch() -> None: + case = component_join() + before = asdict(case.lineage) + case.check() + assert asdict(case.lineage) == before + assert [item.outcome for item in case.lineage.generations] == ["unknown", "unknown", "never-created"] + assert case.lineage.generations[-1].management_id not in case.containment.docker_ids + + +@pytest.mark.parametrize("change", ["missing-genesis", "missing-generations", "unavailable", "foreign-history", "reused-request", "reused-authorization"]) +def test_original_authority_must_be_complete_and_new_recovery_identity_distinct(change: str) -> None: + case = component_join() + if change == "missing-genesis": + case = replace(case, lineage=replace(case.lineage, genesis=None)) + elif change == "missing-generations": + case = replace(case, lineage=replace(case.lineage, generations=())) + elif change == "unavailable": + case = replace(case, lineage=replace(case.lineage, unavailable=("original stores absent",))) + elif change == "foreign-history": + case = replace(case, lineage=replace(case.lineage, binding=replace(case.lineage.binding, history="foreign"))) + elif change == "reused-request": + case = replace(case, authorization=replace(case.authorization, request_id=case.lineage.generations[0].request_id)) + else: + assert case.lineage.genesis is not None + case = replace(case, authorization=replace(case.authorization, authorization_id=case.lineage.genesis.authorization_id)) + with pytest.raises(EpochUnavailable): + case.check() + + +@pytest.mark.parametrize("change", ["unavailable-status", "claims-grant", "claims-history", "claims-admission", "stale-enrollment"]) +def test_containment_is_the_exact_current_nonauthorizing_scope(change: str) -> None: + case = component_join() + if change == "stale-enrollment": + stale = replace(case.enrollment, inode=case.enrollment.inode + 1) + case = replace(case, containment=replace(case.containment, facts=replace(case.containment.facts, enrollment=stale))) + else: + field, value = {"unavailable-status": ("status", "unavailable"), "claims-grant": ("grant_issued", True), "claims-history": ("history_asserted", True), "claims-admission": ("admission_created", True)}[change] + case = replace(case, containment=replace(case.containment, **{field: value})) + with pytest.raises(EpochUnavailable, match="non-authorizing"): + case.check() + + +@pytest.mark.parametrize("change", ["deployment", "source", "inventory-source", "target", "target-config", "engine"]) +def test_current_source_target_and_engine_must_join_original_and_fixed_inventory(change: str) -> None: + case = component_join() + if change in ("deployment", "source", "engine"): + field, value = {"deployment": ("deployment_id", "foreign"), "source": ("source_identity", "sha256:" + "0" * 64), "engine": ("engine_id", "another-engine")}[change] + case = case.scope(replace(case.enrollment.enrollment, **{field: value})) + else: + field = {"inventory-source": "source_sha256", "target": "target_id", "target-config": "config_sha256"}[change] + case = replace(case, inventory=replace(case.inventory, **{field: "0" * 64})) + with pytest.raises(EpochUnavailable): + case.check() + + +@pytest.mark.parametrize("change", ["omitted-lineage", "omitted-observation", "omitted-started-scope", "duplicate-lineage", "duplicate-observation"]) +def test_no_original_generation_or_current_manager_can_disappear_or_alias(change: str) -> None: + case = component_join() + if change == "omitted-lineage": + case = replace(case, lineage=replace(case.lineage, generations=(case.lineage.generations[0], case.lineage.generations[2]))) + elif change == "duplicate-lineage": + case = replace(case, lineage=replace(case.lineage, generations=(*case.lineage.generations, case.lineage.generations[0]))) + elif change == "duplicate-observation": + case = case.managers((*case.containment.managers, case.containment.managers[0])) + else: + case = case.managers((case.containment.managers[0], case.containment.managers[-1])) + if change == "omitted-started-scope": + case = case.scope(replace(case.enrollment.enrollment, originals=case.enrollment.enrollment.originals[:1])) + with pytest.raises(EpochUnavailable): + case.check() + + +@pytest.mark.parametrize("change", ["bootstrap", "proxy", "owner", "root-path", "root-inode", "missing-root"]) +def test_original_native_configuration_owner_and_actual_root_objects_must_match(change: str) -> None: + case = component_join() + original, fact = case.enrollment.enrollment.originals[0], case.containment.managers[0] + if change == "bootstrap": + original = replace(original, bootstrap_config=replace(original.bootstrap_config, sha256="0" * 64)) + elif change == "proxy": + original = replace(original, proxy_id="0" * 64) + elif change == "owner": + assert original.owner is not None + original = replace(original, owner=replace(original.owner, inode=original.owner.inode + 1)) + elif change == "root-path": + fact = replace(fact, state_root="/component/different-root") + elif change == "root-inode": + fact = replace(fact, mounts=(replace(fact.mounts[0], inode=fact.mounts[0].inode + 1), *fact.mounts[1:])) + else: + fact = replace(fact, mounts=fact.mounts[1:]) + case = case.scope(replace(case.enrollment.enrollment, originals=(original, *case.enrollment.enrollment.originals[1:]))).managers((fact, *case.containment.managers[1:])) + with pytest.raises(EpochUnavailable): + case.check() + + +@pytest.mark.parametrize("change", ["docker-object-exists", "owner-exists", "outcome-unknown"]) +def test_absent_branch_requires_original_never_created_boundary_without_current_object(change: str) -> None: + case = component_join() + absent = case.lineage.generations[-1] + if change == "docker-object-exists": + case = replace(case, containment=replace(case.containment, facts=replace(case.containment.facts, docker=(*case.containment.facts.docker, component_docker(absent.management_id))))) + else: + absent = replace(absent, **({"owner_inode": 9999} if change == "owner-exists" else {"outcome": "unknown"})) + case = replace(case, lineage=replace(case.lineage, generations=(*case.lineage.generations[:-1], absent))) + with pytest.raises(EpochUnavailable, match="absent"): + case.check() + + +@pytest.mark.parametrize("change", ["oauth-path", "oauth-inode", "state-reused", "journal-nested", "state-parent"]) +def test_target_preserves_original_oauth_and_keeps_new_roots_disjoint(change: str) -> None: + case = component_join() + target = case.containment.managers[-1] + if change == "oauth-path": + target = replace(target, oauth_root="/component/other-oauth") + elif change == "oauth-inode": + target = replace(target, mounts=(*target.mounts[:-1], replace(target.mounts[-1], inode=601))) + elif change == "state-reused": + target = replace(target, state_root=case.lineage.generations[0].state_root) + elif change == "journal-nested": + target = replace(target, journal_root=case.lineage.generations[-1].journal_root + "/nested") + else: + target = replace(target, state_root="/component") + with pytest.raises(EpochUnavailable): + case.managers((*case.containment.managers[:-1], target)).check() + + +@dataclass +class ScopeCase: + authorization: EpochAuthorization + enrollment: RecoveryEnrollment + resources: EpochResources + configuration: ConsumerConfiguration + path: Path + + def write_configuration(self, configuration: ConsumerConfiguration) -> None: + self.configuration = configuration + self.path.write_bytes(canonical(asdict(configuration))) + info = self.path.stat() + self.authorization = replace(self.authorization, consumer_config=RecoveryFile(str(self.path), hashlib.sha256(self.path.read_bytes()).hexdigest(), info.st_dev, info.st_ino)) + + def check(self) -> None: + _check_consumer_scope(self.authorization, self.enrollment, self.resources) + + +@contextmanager +def component_scope() -> Iterator[ScopeCase]: + with tempfile.TemporaryDirectory(prefix="ac-recovery-join-", dir=pwd.getpwuid(os.geteuid()).pw_dir) as directory, ExitStack() as held: + root = Path(directory) + history_root = root / "history" + history_root.mkdir(mode=0o700) + paths = {"history": history_root / "history.lock", "owner-0": root / "owner.lock", "controller-0": root / "controller.lock", "epoch": root / "epoch.lock", "target-owner": root / "target.lock"} + descriptors = {} + for name, path in paths.items(): + descriptor = os.open(path, os.O_RDWR | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600) + held.callback(os.close, descriptor) + descriptors[name] = descriptor + if name != "target-owner": + fcntl.flock(descriptor, (fcntl.LOCK_SH if name in ("history", "owner-0") else fcntl.LOCK_EX) | fcntl.LOCK_NB) + identities = {name: os.fstat(descriptor) for name, descriptor in descriptors.items()} + fixture = component_join() + original = replace(fixture.enrollment.enrollment.originals[0], owner=RecoveryLock(str(paths["owner-0"]), identities["owner-0"].st_dev, identities["owner-0"].st_ino)) + target = fixture.enrollment.enrollment.target + mounts = [DeploymentRootBinding(target.management_id, "/management-recovery", str(PEER_ROOT), False, 1, 1600, stat.S_IFDIR | 0o700)] + original_paths = {"history": "/management-recovery-history/history.lock", "owner-0": "/management-recovery-originals/0/owner.lock"} + mounts.extend(DeploymentRootBinding(target.management_id, destination, str(paths[name]), False, identities[name].st_dev, identities[name].st_ino, stat.S_IFREG | 0o600) for name, destination in original_paths.items()) + target = replace(target, mounts=tuple(mounts), owner=RecoveryLock(str(paths["target-owner"]), identities["target-owner"].st_dev, identities["target-owner"].st_ino)) + enrollment = replace(fixture.enrollment.enrollment, originals=(original,), target=target) + fences = tuple(HeldFence(name, descriptors[name], "original-sh" if name in original_paths else "target-ex" if name == "target-owner" else "transferred-ex") for name in paths) + boot, pid, start = issuer_lifetime() + resources = EpochResources(descriptors["history"], RecoveryResources(pid, start, boot, (NamedDescriptor(original.generation_id, descriptors["owner-0"]),), (), ()), fences, ()) + configuration = ConsumerConfiguration( + 1, + fixture.authorization.request_id, + fixture.authorization.authorization_id, + target.management_id, + 0, + 1, + 1600, + tuple(OriginalFence(name, original_paths[name], identities[name].st_dev, identities[name].st_ino) for name in original_paths), + TargetOwner("target-owner", identities["target-owner"].st_dev, identities["target-owner"].st_ino), + tuple(TransferredResource(name, "transferred-ex", identities[name].st_dev, identities[name].st_ino) for name in ("controller-0", "epoch")), + ) + path = root / "consumer.json" + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + os.close(descriptor) + case = ScopeCase(replace(fixture.authorization, history=replace(fixture.authorization.history, root=str(history_root))), enrollment, resources, configuration, path) + case.write_configuration(configuration) + yield case + + +def test_consumer_scope_joins_exact_real_fds_and_pinned_read_only_mounts() -> None: + with component_scope() as case: + case.check() + + +@pytest.mark.parametrize("change", ["missing-peer", "writable-peer", "foreign-peer-source", "replaced-peer-inode", "duplicate-peer", "writable-original", "foreign-original-source", "replaced-original-inode"]) +def test_consumer_scope_requires_exact_read_only_endpoint_and_original_mounts(change: str) -> None: + with component_scope() as case: + mounts = list(case.enrollment.target.mounts) + if change == "missing-peer": + mounts.pop(0) + elif change == "duplicate-peer": + mounts.append(mounts[0]) + else: + index = 0 if "peer" in change else 1 + if change.startswith("writable"): + mounts[index] = replace(mounts[index], writable=True) + elif change.startswith("foreign"): + mounts[index] = replace(mounts[index], source="/component/foreign") + else: + mounts[index] = replace(mounts[index], inode=mounts[index].inode + 1) + case.enrollment = replace(case.enrollment, target=replace(case.enrollment.target, mounts=tuple(mounts))) + with pytest.raises(EpochUnavailable, match="read-only"): + case.check() + + +@pytest.mark.parametrize("change", ["foreign-target", "omitted-original", "wrong-original-path", "target-owner-inode", "omitted-controller", "transfer-inode", "transfer-kind"]) +def test_consumer_scope_cannot_omit_or_retarget_acknowledged_authority(change: str) -> None: + with component_scope() as case: + configuration = case.configuration + if change == "foreign-target": + configuration = replace(configuration, target_id="e" * 64) + elif change == "omitted-original": + configuration = replace(configuration, original_fences=configuration.original_fences[:1]) + elif change == "wrong-original-path": + configuration = replace(configuration, original_fences=(replace(configuration.original_fences[0], path="/component/foreign"), *configuration.original_fences[1:])) + elif change == "target-owner-inode": + configuration = replace(configuration, target_owner=replace(configuration.target_owner, inode=configuration.target_owner.inode + 999)) + elif change == "omitted-controller": + configuration = replace(configuration, transferred_resources=configuration.transferred_resources[1:]) + elif change == "transfer-inode": + configuration = replace(configuration, transferred_resources=(replace(configuration.transferred_resources[0], inode=configuration.transferred_resources[0].inode + 999), *configuration.transferred_resources[1:])) + else: + configuration = replace(configuration, transferred_resources=(replace(configuration.transferred_resources[0], kind="network-namespace"), *configuration.transferred_resources[1:])) + case.write_configuration(configuration) + with pytest.raises(EpochUnavailable): + case.check() diff --git a/tests/test_management_recovery_epoch_mutation_control.py b/tests/test_management_recovery_epoch_mutation_control.py new file mode 100644 index 000000000..635857229 --- /dev/null +++ b/tests/test_management_recovery_epoch_mutation_control.py @@ -0,0 +1,186 @@ +"""Bounded guards for the separate real-VM second-verifier mutation control. + +These checks validate transformation and result parsing, not native acceptance. +The dedicated disposable VM must execute the same no-grant oracle with genuine +sources, original records, Docker processes and actual queued receiver bytes. +""" + +from __future__ import annotations + +import ast +import hashlib +import json +from pathlib import Path + +import pytest +from scripts import recovery_epoch_mutation_control as control + +from auto_coder.management import recovery_epoch + + +def _original() -> bytes: + return Path(recovery_epoch.__file__).read_bytes() + + +def test_real_issuer_mutation_removes_only_the_second_post_consumption_verifier() -> None: + source = _original() + mutant, line = control.omit_post_consumption_verifier(source) + assert source.splitlines()[line - 1].strip() == b"final_verification()" + assert mutant == b"".join(source.splitlines(keepends=True)[: line - 1] + source.splitlines(keepends=True)[line:]) + function = next(node for node in ast.parse(mutant).body if isinstance(node, ast.FunctionDef) and node.name == "issue_epoch_once") + calls = [node for node in ast.walk(function) if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == "final_verification"] + assert len(calls) == 1 + consume = next(node for node in ast.walk(function) if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) and node.func.attr == "consume") + assert calls[0].lineno < consume.lineno + assert b"session.check(custody)" in mutant and b"session.connection.sendall(packet)" in mutant + + +@pytest.mark.parametrize("change", ["already-removed", "extra-call", "wrong-consume", "argument", "wrong-function"]) +def test_transform_refuses_other_issuer_shapes(change: str) -> None: + source = _original() + if change == "already-removed": + source = control.omit_post_consumption_verifier(source)[0] + elif change == "extra-call": + source = source.replace(b" final_verification()", b" final_verification()\n final_verification()", 1) + elif change == "wrong-consume": + source = source.replace(b"receipt = state.consume(binding)", b"receipt = state.reserve(binding)") + elif change == "argument": + source = source.replace(b" final_verification()", b" final_verification(binding)", 1) + else: + source = source.replace(b"def issue_epoch_once(", b"def another_issuer(") + with pytest.raises(ValueError): + control.omit_post_consumption_verifier(source) + + +def _copy(tmp_path: Path) -> tuple[Path, Path, Path]: + source, destination, evidence = tmp_path / "original", tmp_path / "control", tmp_path / "mutation-source.json" + selected = source / control.ISSUER_SOURCE + selected.parent.mkdir(parents=True) + selected.write_bytes(_original()) + (source / "unchanged.txt").write_bytes(b"original project bytes") + (source / ".venv").mkdir() + (source / ".venv" / "not-copied").write_bytes(b"environment") + return source, destination, evidence + + +def test_private_copy_retains_original_and_all_other_project_files(tmp_path: Path) -> None: + source, destination, evidence = _copy(tmp_path) + original = (source / control.ISSUER_SOURCE).read_bytes() + value = control.prepare(source, destination, evidence) + assert (source / control.ISSUER_SOURCE).read_bytes() == original + assert (destination / "unchanged.txt").read_bytes() == b"original project bytes" + assert not (destination / ".venv").exists() + observed, digest = control.verify_source_evidence(evidence) + assert observed == value and digest == hashlib.sha256(evidence.read_bytes()).hexdigest() + assert value["original_source_sha256"] != value["mutant_source_sha256"] + assert value["original_project_sha256"] != value["mutant_project_sha256"] + assert (evidence.stat().st_mode & 0o777) == 0o600 + + +@pytest.mark.parametrize("target", ["source", "copy", "evidence"]) +def test_source_evidence_refuses_changed_original_mutant_or_claim(tmp_path: Path, target: str) -> None: + source, destination, evidence = _copy(tmp_path) + control.prepare(source, destination, evidence) + if target == "evidence": + value = json.loads(evidence.read_bytes()) + value["first_verifier_retained"] = False + evidence.write_bytes(control._canonical(value)) + else: + path = (source if target == "source" else destination) / control.ISSUER_SOURCE + path.write_bytes(path.read_bytes() + b"\n# changed original source\n") + with pytest.raises(AssertionError): + control.verify_source_evidence(evidence) + + +def test_copy_refuses_existing_destination_without_changing_it(tmp_path: Path) -> None: + source, destination, evidence = _copy(tmp_path) + destination.mkdir() + retained = destination / "retained" + retained.write_bytes(b"existing") + with pytest.raises(ValueError, match="new separate"): + control.prepare(source, destination, evidence) + assert retained.read_bytes() == b"existing" and not evidence.exists() + + +def _empty_dispatch() -> dict[str, object]: + return {"schema": "actual-recovery-post-receipt-dispatch-v1", "receiver_stopped": True, "queue_empty_before_resume": True, "socket_flags_unchanged": True, "queue_bytes_observed": 0, "queued_grant": None, "issuer_returncode": 2} + + +def test_correct_post_receipt_refusal_passes_the_shared_oracle() -> None: + control.assert_post_receipt_refusal(_empty_dispatch()) + + +@pytest.mark.parametrize("field,value", [("issuer_returncode", None), ("issuer_returncode", 0), ("issuer_returncode", -9), ("queue_bytes_observed", 1), ("receiver_stopped", False), ("queue_empty_before_resume", False), ("socket_flags_unchanged", False)]) +def test_generic_failure_kill_timeout_or_unsupported_queue_is_not_mutation_detection(field: str, value: object) -> None: + observation = _empty_dispatch() + observation[field] = value + with pytest.raises(AssertionError) as caught: + control.assert_post_receipt_refusal(observation) + assert type(caught.value) is not control.PostReceiptGrantDispatched + + +def _queued_dispatch() -> dict[str, object]: + issuer = {"issuer_pid": 123, "issuer_start_token": "345", "target_pid": 678, "target_start_token": "901", "peer_credentials": {"pid": 123, "uid": 0, "gid": 0}} + grant = { + "schema": "management-recovery-epoch-grant-v1", + "granted": "recovery-epoch-consumed", + "request_id": "fixture-request", + "authorization_id": "fixture-authorization", + "target_id": "a" * 64, + "challenge": "b" * 64, + "receipt_nonce": "c" * 64, + "inventory_sha256": "d" * 64, + "binding_sha256": "e" * 64, + "consumed_sha256": "f" * 64, + "receipt_sha256": "0" * 64, + "host_boot_id": "actual-component-boot", + "host_pid": issuer["issuer_pid"], + "host_start_token": issuer["issuer_start_token"], + } + encoded = control._canonical(grant) + b"\n" + queued = { + **issuer, + "schema": "actual-recovery-queued-grant-v1", + "grant_schema": grant["schema"], + "target_id": grant["target_id"], + "grant": grant, + "bytes_observed": len(encoded), + "message_sha256": hashlib.sha256(encoded).hexdigest(), + "socket_flags_before": 2, + "socket_flags_after": 2, + "socket_inode": 77, + "peek_only": True, + "socket_flags_unchanged": True, + "bytes_still_queued": True, + "receiver_stopped": True, + "issuer_alive": True, + } + return {**_empty_dispatch(), **issuer, "queued_grant": queued, "queue_bytes_observed": len(encoded), "issuer_returncode": None} + + +def test_only_exact_original_queued_grant_triggers_named_native_failure() -> None: + with pytest.raises(control.PostReceiptGrantDispatched, match=control.ORACLE): + control.assert_post_receipt_refusal(_queued_dispatch()) + + +@pytest.mark.parametrize("mutation", ["digest", "length", "foreign-issuer", "dead-issuer", "consumed-bytes", "wrong-lifetime", "modified-flags"]) +def test_nonmatching_or_incomplete_dispatch_evidence_cannot_count_as_mutation_detection(mutation: str) -> None: + observation = _queued_dispatch() + queued = observation["queued_grant"] + if mutation == "digest": + queued["message_sha256"] = "f" * 64 + elif mutation == "length": + queued["bytes_observed"] += 1 + elif mutation == "foreign-issuer": + queued["peer_credentials"] = {"pid": 999, "uid": 0, "gid": 0} + elif mutation == "dead-issuer": + queued["issuer_alive"] = False + elif mutation == "consumed-bytes": + queued["bytes_still_queued"] = False + elif mutation == "wrong-lifetime": + queued["issuer_start_token"] = "987" + else: + queued["socket_flags_after"] += 1 + with pytest.raises(AssertionError) as caught: + control.assert_post_receipt_refusal(observation) + assert type(caught.value) is not control.PostReceiptGrantDispatched diff --git a/tests/test_management_recovery_epoch_native_negatives.py b/tests/test_management_recovery_epoch_native_negatives.py new file mode 100644 index 000000000..20e8d8dae --- /dev/null +++ b/tests/test_management_recovery_epoch_native_negatives.py @@ -0,0 +1,397 @@ +"""Real-file/process safety guards for the native negative fixture helper. + +These component checks do not claim Docker/systemd or complete issuer acceptance. +The separately authorized VM runs every production-issuer and report-CLI case. +""" + +from __future__ import annotations + +import fcntl +import json +import os +import select +import signal +import stat +import subprocess +import sys +from contextlib import nullcontext +from dataclasses import replace +from pathlib import Path + +import pytest +from scripts import recovery_epoch_native_negatives as fixture +from scripts.test_management_containment_enrollment_host import _write_private + +from auto_coder.management.containment_enrollment import OriginalNetworkPin +from auto_coder.management.containment_kernel_subobservation import _holder_start_token +from auto_coder.management.recovery_containment_model import RecoveryLock +from tests.test_management_recovery_containment import _enrollment + +pytestmark = pytest.mark.usefixtures("_use_real_commands", "_use_real_sleep") + + +@pytest.fixture +def _use_real_commands(): + yield + + +@pytest.fixture +def _use_real_sleep(): + yield + + +def _context(tmp_path: Path) -> fixture.NativeNegativeContext: + epoch = tmp_path / "epoch" + epoch.mkdir(mode=0o700) + _write_private(epoch / "epoch.lock", b"") + _write_private(epoch / "inventory.json", b'{"component":"no-authority"}') + return fixture.NativeNegativeContext(_enrollment(), tmp_path, epoch) + + +@pytest.mark.parametrize("previous,refusal", [(False, None), (True, None), (True, "expected distinct refusal")]) +def test_containment_cli_propagates_only_safe_failure_metadata(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, previous: bool, refusal: str | None) -> None: + causes = [{"type": "OSError", "errno": 5, "file": "observed_component.py", "function": "read_current", "line": 42}] + document = {"schema": "current-recovery-containment-v1", "status": "unavailable", "grant_issued": False, "history_asserted": False, "admission_created": False, "reason": "fixed masked refusal", "causes": causes, "unrelated": "private-data"} + monkeypatch.setattr(fixture.subprocess, "run", lambda *_args, **_kwargs: subprocess.CompletedProcess([], 2, "diagnostic prefix\n" + json.dumps(document) + "\n", "")) + with pytest.raises(fixture.NativeContainmentFailure, match="fixed masked refusal") as caught: + fixture._containment_cli(tmp_path, "original-observer", "component-observation", tmp_path / "private-resources.json", previous=tmp_path / "private-previous.json" if previous else None, refusal=refusal) + assert caught.value.diagnostics == {"schema": "native-recovery-containment-cli-failure-v1", "action": "verify" if previous else "observe", "returncode": 2, "causes": causes} + assert "private" not in json.dumps(caught.value.diagnostics) + assert (tmp_path / "original-observer.log").is_file() + + +def test_containment_cli_diagnostics_do_not_change_expected_refusal(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + document = {"schema": "current-recovery-containment-v1", "status": "unavailable", "grant_issued": False, "history_asserted": False, "admission_created": False, "reason": "expected refusal", "causes": [{"type": "ValueError", "message": "private-data"}]} + monkeypatch.setattr(fixture.subprocess, "run", lambda *_args, **_kwargs: subprocess.CompletedProcess([], 2, json.dumps(document) + "\n", "")) + assert fixture._containment_cli(tmp_path, "expected-refusal", "component-observation", tmp_path / "resources.json", refusal="expected refusal") == document + + +def test_unreserved_guard_observes_but_never_rewrites_original_inputs(tmp_path: Path) -> None: + context = _context(tmp_path) + before = {path.name: fixture._projection(path) for path in context.epoch_root.iterdir()} + assert fixture._epoch_snapshot(context.epoch_root) == fixture._epoch_snapshot(context.epoch_root) + fixture.assert_unreserved_epoch(context.epoch_root) + assert {path.name: fixture._projection(path) for path in context.epoch_root.iterdir()} == before + + +@pytest.mark.parametrize("name", ["reserved.json", "reserved-head.json", "intent.json", "intent-head.json", "consumed.json", "consumed-head.json", "receipt.json", "receipt-head.json", ".partial", "unexpected.json"]) +def test_any_original_publication_or_partial_name_permanently_aborts_fixture(tmp_path: Path, name: str) -> None: + context = _context(tmp_path) + path = context.epoch_root / name + _write_private(path, b"uncertain original bytes") + before = {item.name: fixture._projection(item) for item in context.epoch_root.iterdir()} + with pytest.raises(fixture.EpochAlreadyPublished, match="never repair or retry"): + fixture.assert_unreserved_epoch(context.epoch_root) + assert {item.name: fixture._projection(item) for item in context.epoch_root.iterdir()} == before + + +@pytest.mark.parametrize("change", ["symlink", "hardlink", "file-mode", "directory-mode", "missing-input"]) +def test_unreserved_guard_requires_bounded_original_objects(tmp_path: Path, change: str) -> None: + context = _context(tmp_path) + lock = context.epoch_root / "epoch.lock" + if change == "symlink": + lock.unlink() + lock.symlink_to(context.epoch_root / "inventory.json") + elif change == "hardlink": + os.link(lock, tmp_path / "alias") + elif change == "file-mode": + lock.chmod(0o640) + elif change == "directory-mode": + context.epoch_root.chmod(0o750) + else: + lock.unlink() + with pytest.raises(fixture.EpochAlreadyPublished): + fixture.assert_unreserved_epoch(context.epoch_root) + + +def test_epoch_inode_injection_restores_the_same_original_only_before_publication(tmp_path: Path) -> None: + context = _context(tmp_path) + original = context.epoch_root / "epoch.lock" + before = fixture._projection(original) + backup = tmp_path / "epoch.original" + inventory = fixture._projection(context.epoch_root / "inventory.json") + with fixture._replacement(context.epoch_root, original, backup, b""): + assert fixture._projection(original)[:2] != before[:2] + assert fixture._projection(backup) == before + fixture.assert_unreserved_epoch(context.epoch_root) + assert fixture._projection(original) == before + assert fixture._projection(context.epoch_root / "inventory.json") == inventory + assert not backup.exists() + + +def test_publication_during_epoch_injection_keeps_every_original_and_never_resets(tmp_path: Path) -> None: + context = _context(tmp_path) + path, backup = context.epoch_root / "epoch.lock", tmp_path / "epoch.original" + original = fixture._projection(path) + with pytest.raises(fixture.EpochAlreadyPublished, match="never repair or retry"): + with fixture._replacement(context.epoch_root, path, backup, b""): + injected = fixture._projection(path) + _write_private(context.epoch_root / "reserved.json", b"original partial record") + assert fixture._projection(path) == injected + assert fixture._projection(backup) == original + assert (context.epoch_root / "reserved.json").read_bytes() == b"original partial record" + assert {item.name for item in context.epoch_root.iterdir()} == {"epoch.lock", "inventory.json", "reserved.json"} + + +def test_modified_injection_is_retained_instead_of_destructively_restored(tmp_path: Path) -> None: + context = _context(tmp_path) + path, backup = tmp_path / "owner", tmp_path / "owner.original" + _write_private(path, b"original owner") + original = fixture._projection(path) + with pytest.raises(AssertionError, match="replacement changed"): + with fixture._replacement(context.epoch_root, path, backup, b"injected owner"): + path.write_bytes(b"unexpected issuer mutation") + assert path.read_bytes() == b"unexpected issuer mutation" + assert fixture._projection(backup) == original + + +def test_missing_owner_injection_never_creates_a_substitute_and_restores_the_original(tmp_path: Path) -> None: + context = _context(tmp_path) + path, backup = tmp_path / "owner.lock", tmp_path / "owner.original" + _write_private(path, b"original owner") + before = fixture._projection(path) + with fixture._missing_original(context.epoch_root, path, backup): + assert not path.exists() + assert fixture._projection(backup) == before + fixture.assert_unreserved_epoch(context.epoch_root) + assert fixture._projection(path) == before + assert not backup.exists() + + +def test_issuer_recreated_missing_owner_is_retained_as_a_failure(tmp_path: Path) -> None: + context = _context(tmp_path) + path, backup = tmp_path / "owner.lock", tmp_path / "owner.original" + _write_private(path, b"original owner") + before = fixture._projection(path) + with pytest.raises(AssertionError, match="recreated"): + with fixture._missing_original(context.epoch_root, path, backup): + _write_private(path, b"unexpected replacement") + assert fixture._projection(backup) == before + assert path.read_bytes() == b"unexpected replacement" + assert path.stat().st_ino != backup.stat().st_ino + + +def test_publication_during_missing_owner_case_never_restores_or_retries(tmp_path: Path) -> None: + context = _context(tmp_path) + path, backup = tmp_path / "owner.lock", tmp_path / "owner.original" + _write_private(path, b"original owner") + before = fixture._projection(path) + with pytest.raises(fixture.EpochAlreadyPublished, match="never repair or retry"): + with fixture._missing_original(context.epoch_root, path, backup): + _write_private(context.epoch_root / "reserved.json", b"uncertain original publication") + assert not path.exists() + assert fixture._projection(backup) == before + assert (context.epoch_root / "reserved.json").read_bytes() == b"uncertain original publication" + + +@pytest.mark.parametrize("change", ["contents", "inode", "added-entry", "removed-entry"]) +def test_alternate_authority_snapshot_detects_real_file_or_membership_changes(tmp_path: Path, change: str) -> None: + root = tmp_path / "alternate" + root.mkdir(mode=0o700) + path = root / "inventory.json" + _write_private(path, b"original alternate input") + before = fixture._alternate_input_snapshot(root) + assert fixture._alternate_input_snapshot(root) == before + if change == "contents": + path.write_bytes(b"modified alternate input") + elif change == "inode": + path.rename(tmp_path / "retained-original") + _write_private(path, b"original alternate input") + elif change == "added-entry": + _write_private(root / "reserved.json", b"unexpected publication") + else: + path.unlink() + assert fixture._alternate_input_snapshot(root) != before + + +def test_each_refusal_compares_injected_and_restored_original_state(tmp_path: Path) -> None: + context = _context(tmp_path) + original, backup = tmp_path / "owner", tmp_path / "owner.original" + _write_private(original, b"native original") + before = fixture._projection(original) + calls = [] + + def refusal(label: str) -> None: + assert fixture._projection(original)[:2] != before[:2] + calls.append(label) + + fixture._run_case(context, "component-refusal", lambda: fixture._replacement(context.epoch_root, original, backup, b"native original"), refusal, lambda: fixture._projection(original)) + assert calls == ["component-refusal"] + assert fixture._projection(original) == before + + +def test_refused_issuer_cannot_modify_unrelated_original_state(tmp_path: Path) -> None: + context = _context(tmp_path) + path = tmp_path / "original-operation-store" + _write_private(path, b"original") + + def invalid_refusal(_: str) -> None: + path.write_bytes(b"unexpected change") + + with pytest.raises(AssertionError, match="changed original/OAuth/worker"): + fixture._run_case(context, "component-refusal", nullcontext, invalid_refusal, lambda: fixture._projection(path)) + assert path.read_bytes() == b"unexpected change" + + +def test_refused_issuer_cannot_modify_epoch_inventory_without_a_record(tmp_path: Path) -> None: + context = _context(tmp_path) + + def invalid_refusal(_: str) -> None: + (context.epoch_root / "inventory.json").write_bytes(b"unexpected replacement configuration") + + with pytest.raises(AssertionError, match="changed an original epoch input"): + fixture._run_case(context, "component-refusal", nullcontext, invalid_refusal, lambda: "unchanged original state") + assert (context.epoch_root / "inventory.json").read_bytes() == b"unexpected replacement configuration" + + +def test_actual_ex_contender_prevents_independent_shared_fence_and_releases_only_itself(tmp_path: Path) -> None: + path = tmp_path / "original.lock" + _write_private(path, b"") + info = path.stat() + selected = RecoveryLock(str(path), info.st_dev, info.st_ino) + code = "import fcntl,sys;f=open(sys.argv[1],'rb');\ntry: fcntl.flock(f,fcntl.LOCK_SH|fcntl.LOCK_NB)\nexcept BlockingIOError: sys.exit(7)\n" + with fixture._busy_fence(selected): + assert subprocess.run((sys.executable, "-I", "-B", "-c", code, str(path)), timeout=10).returncode == 7 + assert subprocess.run((sys.executable, "-I", "-B", "-c", code, str(path)), timeout=10).returncode == 0 + assert path.stat().st_ino == selected.inode and path.read_bytes() == b"" + + +def test_actual_busy_fence_does_not_accept_a_different_selected_inode(tmp_path: Path) -> None: + path = tmp_path / "owner.lock" + _write_private(path, b"") + info = path.stat() + with pytest.raises(AssertionError, match="selected original"): + with fixture._busy_fence(RecoveryLock(str(path), info.st_dev, info.st_ino + 1)): + pytest.fail("a different original identity was accepted") + with path.open("rb") as stream: + fcntl.flock(stream.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + + +def test_real_process_barrier_and_pidfd_cleanup_leave_no_running_fixture_child() -> None: + code = "import sys;print('READY',flush=True);sys.stdin.buffer.read(1)" + with fixture._live_process((sys.executable, "-I", "-B", "-c", code)) as process: + pid = process.pid + assert process.poll() is None and Path(f"/proc/{pid}").exists() + assert process.returncode is not None + assert not Path(f"/proc/{pid}").exists() + + +def test_malformed_real_process_readiness_is_a_failure() -> None: + with pytest.raises(AssertionError, match="exact readiness"): + with fixture._live_process((sys.executable, "-I", "-B", "-c", "print('UNAVAILABLE',flush=True)")): + pytest.fail("an invalid native barrier was accepted") + + +def test_exited_nonleader_control_retains_the_same_real_zombie_for_an_independent_reader() -> None: + code = "import json,threading;event=threading.Event();\ndef occupy():\n print(json.dumps({'tid':threading.get_native_id()}),flush=True);event.wait()\nthread=threading.Thread(target=occupy);thread.start();thread.join()" + process = subprocess.Popen((sys.executable, "-I", "-B", "-c", code), stdout=subprocess.PIPE, stderr=subprocess.PIPE) + descriptor = os.pidfd_open(process.pid, 0) + try: + assert process.stdout is not None and select.select((process.stdout,), (), (), 10)[0] + tid = json.loads(process.stdout.readline())["tid"] + leader = fixture._task_observation(process.pid, process.pid) + nonleader = fixture._task_observation(process.pid, tid) + assert leader.pid == nonleader.pid and leader.tid != nonleader.tid + assert leader.network_namespace == nonleader.network_namespace == os.readlink("/proc/self/ns/net") + assert leader.state not in {"Z", "X", "x"} and nonleader.state not in {"Z", "X", "x"} + signal.pidfd_send_signal(descriptor, signal.SIGTERM) + exited = fixture._wait_unreaped_exit(process.pid, descriptor, leader.start_token, tid) + assert exited.leader.start_token == leader.start_token and exited.leader.state == "Z" + assert exited.leader.network_namespace is None and exited.nonleader_absent + assert exited.remaining_tids == (process.pid,) and exited.pidfd_readable + assert (exited.wait_code, exited.wait_status) == (os.CLD_KILLED, signal.SIGTERM) + probe = ( + "import json,os,sys;from pathlib import Path;p=Path('/proc')/sys.argv[1];s=(p/'stat').read_text().rsplit(')',1)[1].split();" + "assert s[0]=='Z' and s[19]==sys.argv[2];assert sorted(x.name for x in (p/'task').iterdir())==[sys.argv[1]];\n" + "try: os.readlink(p/'ns/net')\nexcept FileNotFoundError: print(json.dumps({'state':s[0],'start':s[19],'namespace':None}))\n" + "else: raise AssertionError('exited task retained nsproxy')" + ) + observed = subprocess.run((sys.executable, "-I", "-B", "-c", probe, str(process.pid), leader.start_token), capture_output=True, text=True, timeout=10, check=True) + assert json.loads(observed.stdout) == {"state": "Z", "start": leader.start_token, "namespace": None} + assert fixture._wait_unreaped_exit(process.pid, descriptor, leader.start_token, tid) == exited + with pytest.raises(AssertionError, match="measured namespace-free zombie"): + fixture._wait_unreaped_exit(process.pid, descriptor, str(int(leader.start_token) + 1), tid) + assert process.wait(timeout=10) == -signal.SIGTERM + with pytest.raises(ChildProcessError): + fixture._wait_unreaped_exit(process.pid, descriptor, leader.start_token, tid) + finally: + if process.poll() is None: + signal.pidfd_send_signal(descriptor, signal.SIGKILL) + process.communicate(timeout=10) + os.close(descriptor) + + +_RESOURCE_HOLDER = """ +import fcntl +import os +import sys +owner = open(sys.argv[1], 'rb') +fcntl.flock(owner, fcntl.LOCK_SH | fcntl.LOCK_NB) +controller = open(sys.argv[2], 'rb') +fcntl.flock(controller, fcntl.LOCK_EX | fcntl.LOCK_NB) +namespace = os.open('/proc/self/ns/net', os.O_RDONLY) +extra = open(sys.argv[1], 'rb') if sys.argv[3] == 'duplicate' else None +print('READY', flush=True) +sys.stdin.buffer.read(1) +""" + + +@pytest.mark.parametrize("duplicate", [False, True]) +def test_report_probe_discovers_actual_stopped_process_descriptors_without_inventing_fences(tmp_path: Path, duplicate: bool) -> None: + owner, controller = tmp_path / "owner.lock", tmp_path / "controller.lock" + for path in (owner, controller): + _write_private(path, b"") + namespace = os.open("/proc/self/ns/net", os.O_RDONLY | os.O_CLOEXEC) + try: + ns = os.fstat(namespace) + pin = OriginalNetworkPin(os.getpid(), _holder_start_token(os.getpid()), Path("/proc/sys/kernel/random/boot_id").read_text().strip(), namespace, os.readlink(f"/proc/self/fd/{namespace}"), ns.st_dev, ns.st_ino) + selected = _enrollment() + original = replace(selected.originals[0], owner=RecoveryLock(str(owner), owner.stat().st_dev, owner.stat().st_ino), namespace_pin=pin) + watcher = replace(selected.watchers[0], lock=RecoveryLock(str(controller), controller.stat().st_dev, controller.stat().st_ino)) + selected = replace(selected, originals=(original,), watchers=(watcher,)) + with fixture._live_process((sys.executable, "-I", "-B", "-c", _RESOURCE_HOLDER, str(owner), str(controller), "duplicate" if duplicate else "one")) as process: + with pytest.raises(AssertionError, match="SIGSTOP"): + fixture._issuer_resources(selected, process.pid) + process.send_signal(signal.SIGSTOP) + waited, status = os.waitpid(process.pid, os.WUNTRACED) + assert waited == process.pid and os.WIFSTOPPED(status) + try: + if duplicate: + with pytest.raises(AssertionError, match="absent or ambiguous"): + fixture._issuer_resources(selected, process.pid) + else: + actual = fixture._issuer_resources(selected, process.pid) + assert actual.holder_pid == process.pid and actual.holder_start_token == _holder_start_token(process.pid) + for values, expected in ((actual.original_owner_fds, owner), (actual.controller_fds, controller)): + assert len(values) == 1 + entry = Path(f"/proc/{process.pid}/fd/{values[0].descriptor}") + assert (entry.stat().st_dev, entry.stat().st_ino) == (expected.stat().st_dev, expected.stat().st_ino) + assert len(actual.namespace_fds) == 1 + assert os.readlink(f"/proc/{process.pid}/fd/{actual.namespace_fds[0].descriptor}") == pin.namespace + finally: + process.send_signal(signal.SIGCONT) + finally: + os.close(namespace) + + +@pytest.mark.parametrize("action", ["negatives", "reports"]) +def test_native_entry_refuses_unapproved_roots_before_any_fixture_effect(tmp_path: Path, action: str) -> None: + context = _context(tmp_path) + before = fixture._epoch_snapshot(context.epoch_root) + with pytest.raises(PermissionError, match="fixed disposable fixture roots"): + if action == "negatives": + fixture.run_native_negatives(context, lambda _: pytest.fail("an unauthorized issuer was started"), lambda: pytest.fail("unauthorized fixture state was sampled")) + else: + fixture.verify_native_containment_reports(context, os.getpid()) + assert fixture._epoch_snapshot(context.epoch_root) == before + assert {path.name for path in tmp_path.iterdir()} == {"epoch"} + + +def test_native_case_oracles_distinguish_actual_issuer_and_separate_report_work() -> None: + assert len(fixture.REQUIRED_NEGATIVE_CASES) == 17 + assert len(fixture.REQUIRED_REPORT_CASES) == 6 + assert fixture.REQUIRED_NEGATIVE_CASES.isdisjoint(fixture.REQUIRED_REPORT_CASES) + assert {"busy-original-owner-refused", "busy-controller-fence-refused", "missing-original-owner-refused", "replaced-epoch-lock-refused", "retained-old-network-task-and-listener-refused", "escaped-old-network-nonleader-refused", "old-published-listener-refused"} <= fixture.REQUIRED_NEGATIVE_CASES + assert {"forged-native-containment-refused", "cross-deployment-containment-refused", "stale-native-containment-refused", "escaped-nonleader-current-containment-refused", "measured-exited-nonleader-current-containment-reverified"} <= fixture.REQUIRED_REPORT_CASES + assert {"factory-root-argument-refused", "factory-inventory-argument-refused", "factory-epoch-root-environment-refused", "factory-inventory-environment-refused", "factory-authorization-environment-refused", "factory-containment-environment-refused"} <= fixture.REQUIRED_NEGATIVE_CASES + assert stat.S_IMODE(Path(fixture.__file__).stat().st_mode) & 0o111 == 0 diff --git a/tests/test_management_recovery_epoch_native_peer.py b/tests/test_management_recovery_epoch_native_peer.py new file mode 100644 index 000000000..b2d568b6b --- /dev/null +++ b/tests/test_management_recovery_epoch_native_peer.py @@ -0,0 +1,440 @@ +"""Real SCM_RIGHTS ABI components, separate from native VM peer acceptance. + +The message reader inspects actual kernel-populated msghdr/iovec/control memory +from a local socketpair. No production issuer, containment or grant is asserted. +The disposable VM independently exercises the actual ptrace/process barriers. +""" + +from __future__ import annotations + +import array +import ctypes +import os +import socket +from contextlib import ExitStack, contextmanager +from dataclasses import dataclass, replace +from pathlib import Path + +import pytest +from scripts.recovery_epoch_native_peer import ( + LATE_MUTATION_SCENARIOS, + MUTATION_SCENARIOS, + POST_RECEIPT_MUTATION_SCENARIOS, + REQUIRED_PEER_CASES, + KernelPeerMessageBarrier, + _container_ids, + _ControlHeader, + _decode_rights, + _descriptor_snapshot, + _failed_terminal, + _Iovec, + _local_socket_peer, + _MessageHeader, + _peek_pending_frame, + _peek_queued_packet, + _stopped_receiver, + _verify_target_loss, + mutate_after_publication, +) + +from auto_coder.management.original_attempt_evidence import OriginalFile, OriginalGeneration, OriginalLineageEvidence +from auto_coder.management.original_attempts import HistoryBinding, canonical +from auto_coder.management.recovery_epoch_peer import MAX_MESSAGE_BYTES, MAX_RESOURCES, OFFER_SCHEMA, encode_peer_message + +pytestmark = pytest.mark.usefixtures("_use_real_commands", "_use_real_sleep") + + +@dataclass +class _OwnMemory: + pid: int + returncode: int | None = None + + def wait(self, timeout: float | None = None) -> int: + raise AssertionError("this parser component cannot own a tracee") + + +@contextmanager +def _kernel_message(tmp_path: Path, count: int): + """Populate the actual Linux ABI by invoking libc recvmsg, without ptrace.""" + libc = ctypes.CDLL(None, use_errno=True) + libc.recvmsg.argtypes = [ctypes.c_int, ctypes.POINTER(_MessageHeader), ctypes.c_int] + libc.recvmsg.restype = ctypes.c_ssize_t + packet = encode_peer_message({"schema": OFFER_SCHEMA, "request_id": "request", "authorization_id": "authorization", "target_id": "target", "resources": []}) + data = ctypes.create_string_buffer(MAX_MESSAGE_BYTES + 1) + control = ctypes.create_string_buffer(socket.CMSG_SPACE(MAX_RESOURCES * array.array("i").itemsize)) + vector = _Iovec(ctypes.addressof(data), len(data)) + header = _MessageHeader(None, 0, ctypes.addressof(vector), 1, ctypes.addressof(control), len(control), 0) + receiver_fds: tuple[int, ...] = () + with ExitStack() as held: + sender, receiver = socket.socketpair() + held.enter_context(sender) + held.enter_context(receiver) + sources = tuple(held.enter_context((tmp_path / str(index)).open("w+b")).fileno() for index in range(count)) + ancillary = [(socket.SOL_SOCKET, socket.SCM_RIGHTS, array.array("i", sources))] if count else [] + assert sender.sendmsg([packet], ancillary) == len(packet) + result = libc.recvmsg(receiver.fileno(), ctypes.byref(header), socket.MSG_CMSG_CLOEXEC) + assert result == len(packet), os.strerror(ctypes.get_errno()) + if count: + # Capture only the actual expected received integers for cleanup, + # independently of the parser under test's handling of cmsghdr. + values = array.array("i") + values.frombytes(control.raw[ctypes.sizeof(_ControlHeader) : ctypes.sizeof(_ControlHeader) + count * values.itemsize]) + receiver_fds = tuple(values) + reader = object.__new__(KernelPeerMessageBarrier) + reader.issuer = _OwnMemory(os.getpid()) + reader.binding = {"request_id": "request", "authorization_id": "authorization", "target_id": "target"} + reader.schema = OFFER_SCHEMA + try: + yield reader, header, result, packet, sources, receiver_fds + finally: + for descriptor in receiver_fds: + os.close(descriptor) + + +@pytest.mark.parametrize("count", [0, 1, 2, MAX_RESOURCES]) +def test_real_recvmsg_memory_retains_exact_bytes_and_received_kernel_objects(tmp_path: Path, count: int) -> None: + with _kernel_message(tmp_path, count) as (reader, header, size, packet, originals, received): + encoded, descriptors = reader._message(ctypes.addressof(header), received_length=size) + assert encoded == packet and descriptors == received + assert reader._selected(encoded) + for original, actual in zip(originals, received): + assert original != actual + assert (os.fstat(original).st_dev, os.fstat(original).st_ino) == (os.fstat(actual).st_dev, os.fstat(actual).st_ino) + assert os.get_inheritable(actual) is False + + +@pytest.mark.parametrize("field,value", [("vector_count", 0), ("vector_count", 5), ("control_length", 4096), ("flags", socket.MSG_CTRUNC), ("flags", socket.MSG_TRUNC)]) +def test_actual_message_reader_refuses_incomplete_or_unbounded_abi_observations(tmp_path: Path, field: str, value: int) -> None: + with _kernel_message(tmp_path, 1) as (reader, header, size, _packet, _originals, _received): + setattr(header, field, value) + with pytest.raises(AssertionError, match="vectors or ancillary"): + reader._message(ctypes.addressof(header), received_length=size) + + +def test_actual_message_reader_refuses_a_return_larger_than_receiver_capacity(tmp_path: Path) -> None: + with _kernel_message(tmp_path, 1) as (reader, header, _size, _packet, _originals, _received): + with pytest.raises(AssertionError, match="iovec capacity"): + reader._message(ctypes.addressof(header), received_length=MAX_MESSAGE_BYTES + 2) + + +@pytest.mark.parametrize("scenario", ["", "late", "late-arbitrary-file", "late-../epoch", "after-intent-head"]) +def test_late_mutation_selector_refuses_unreviewed_actions_before_access(scenario: str) -> None: + with pytest.raises(ValueError, match="reviewed late mutation"): + with mutate_after_publication(None, scenario, os.getpid()): + pytest.fail("an unreviewed mutation was admitted") + + +def test_native_case_sets_are_explicit_and_distinct() -> None: + assert len(REQUIRED_PEER_CASES) == 4 + assert len(LATE_MUTATION_SCENARIOS) == 11 and len(set(LATE_MUTATION_SCENARIOS)) == 11 + assert POST_RECEIPT_MUTATION_SCENARIOS == ("post-receipt-ready-mutation",) + assert len(MUTATION_SCENARIOS) == 12 and not set(LATE_MUTATION_SCENARIOS).intersection(POST_RECEIPT_MUTATION_SCENARIOS) + assert not REQUIRED_PEER_CASES.intersection(LATE_MUTATION_SCENARIOS) + + +@pytest.mark.parametrize("malformation", ["header", "length", "type", "integer", "duplicate", "negative", "too-many"]) +def test_native_control_parser_refuses_nonexact_descriptor_records(malformation: str) -> None: + descriptor_data = array.array("i", [3]).tobytes() + header = _ControlHeader(ctypes.sizeof(_ControlHeader) + len(descriptor_data), socket.SOL_SOCKET, socket.SCM_RIGHTS) + if malformation == "header": + encoded = b"\x00" + else: + if malformation == "length": + header.length += 32 + elif malformation == "type": + header.kind = socket.SCM_CREDENTIALS + elif malformation == "integer": + descriptor_data += b"\x00" + header.length += 1 + elif malformation in {"duplicate", "negative", "too-many"}: + descriptor_data = array.array("i", [3, 3] if malformation == "duplicate" else [-1] if malformation == "negative" else range(MAX_RESOURCES + 1)).tobytes() + header.length = ctypes.sizeof(_ControlHeader) + len(descriptor_data) + encoded = bytes(header) + descriptor_data + with pytest.raises(AssertionError, match="native"): + _decode_rights(encoded) + + +def test_real_shared_ofd_reacquisition_changes_kernel_acquirer_visible_to_original_process(tmp_path: Path) -> None: + """Check the kernel premise of the VM fault, without substituting pidfd_getfd.""" + import fcntl + import select + import subprocess + import sys + + from auto_coder.management.recovery_epoch_peer import inspect_kernel_lock + + script = "import fcntl,sys; fd=int(sys.argv[1]); fcntl.flock(fd, fcntl.LOCK_SH); print('original-acquired',flush=True); sys.stdin.read(1)" + with (tmp_path / "original-owner.lock").open("w+b") as original: + os.fchmod(original.fileno(), 0o600) + info = os.fstat(original.fileno()) + process = subprocess.Popen([sys.executable, "-I", "-B", "-c", script, str(original.fileno())], pass_fds=(original.fileno(),), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + try: + assert select.select((process.stdout,), (), (), 10)[0] + assert process.stdout.readline() == "original-acquired\n" + before = inspect_kernel_lock(process.pid, original.fileno(), info.st_dev, info.st_ino) + assert before is not None and (before.mode, before.pid) == ("READ", process.pid) + fcntl.flock(original.fileno(), fcntl.LOCK_UN) + fcntl.flock(original.fileno(), fcntl.LOCK_SH | fcntl.LOCK_NB) + after = inspect_kernel_lock(process.pid, original.fileno(), info.st_dev, info.st_ino) + assert after is not None and (after.mode, after.pid) == ("READ", os.getpid()) + assert after != before and after == inspect_kernel_lock(os.getpid(), original.fileno(), info.st_dev, info.st_ino) + process.communicate("x", timeout=10) + assert process.returncode == 0 + finally: + if process.poll() is None: + process.kill() + process.communicate(timeout=10) + + +@pytest.mark.parametrize("encoded", ["a" * 12, "A" * 64, "a" * 64 + "\n" + "a" * 64, "name " + "a" * 64]) +def test_target_loss_inventory_rejects_truncated_alias_or_duplicate_identities(encoded: str) -> None: + with pytest.raises(AssertionError, match="incomplete or conflicting"): + _container_ids(encoded) + + +def test_target_loss_inventory_preserves_complete_ids_and_accepts_measured_empty_output() -> None: + assert _container_ids("b" * 64 + "\n" + "a" * 64 + "\n") == ("a" * 64, "b" * 64) + assert _container_ids("") == () + + +def test_target_loss_oracle_requires_real_pidfd_death_and_exact_object_absence() -> None: + """Exercise kernel lifetime proof; synthetic IDs are not Docker acceptance.""" + import select + import signal + import subprocess + import sys + + process = subprocess.Popen([sys.executable, "-I", "-B", "-c", "import signal; print('alive', flush=True); signal.pause()"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + descriptor = os.pidfd_open(process.pid, 0) + try: + assert select.select((process.stdout,), (), (), 10)[0] + assert process.stdout.readline() == "alive\n" + identifier, other = "a" * 64, "b" * 64 + with pytest.raises(AssertionError, match="pidfd has not independently"): + _verify_target_loss(descriptor, identifier, (identifier, other), (other,)) + signal.pidfd_send_signal(descriptor, signal.SIGKILL) + poll = select.poll() + poll.register(descriptor, select.POLLIN) + assert poll.poll(10000), "the actual subprocess death was not observed" + _verify_target_loss(descriptor, identifier, (identifier, other), (other,)) + for after in ((identifier, other), (), ("c" * 64,)): + with pytest.raises(AssertionError, match="exact target was not removed"): + _verify_target_loss(descriptor, identifier, (identifier, other), after) + assert process.wait(timeout=10) == -signal.SIGKILL + finally: + if process.poll() is None: + signal.pidfd_send_signal(descriptor, signal.SIGKILL) + process.communicate(timeout=10) + os.close(descriptor) + + +def _component_failed_lineage(root: Path) -> tuple[OriginalLineageEvidence, Path]: + """Use explicit synthetic records only for the selector's component oracle.""" + import hashlib + + request = "67bb23bb-35e4-45dd-b9dc-40ba21f912fb" + terminal = root / "00000007.json" + values = ( + (root / "00000000.json", canonical({"kind": "anchor", "attempt": "other-original-attempt"})), + (terminal, canonical({"kind": "never-created", "attempt": request, "value": {}})), + (root / "history.lock", b""), + ) + originals = [] + for path, encoded in values: + path.write_bytes(encoded) + path.chmod(0o600) + info = path.stat() + originals.append(OriginalFile(str(path), info.st_dev, info.st_ino, hashlib.sha256(encoded).hexdigest())) + branch = OriginalGeneration( + request_id=request, + instance_id="D", + management_id="d" * 64, + image_id="sha256:" + "e" * 64, + bootstrap_config_sha256="b" * 64, + management_config_sha256="c" * 64, + state_root=str(root / "state"), + state_device=1, + state_inode=2, + journal_root=str(root / "journal"), + journal_device=1, + journal_inode=3, + oauth_root=str(root / "oauth"), + oauth_device=1, + oauth_inode=4, + owner_path="", + owner_device=0, + owner_inode=0, + outcome="never-created", + expected_stores=(), + ) + return OriginalLineageEvidence(HistoryBinding(str(root), "component-deployment", "component-history", "sha256:" + "a" * 64), frontier="f" * 64, generations=(branch,), originals=tuple(originals)), terminal + + +def test_failed_branch_selector_uses_matched_terminal_original_not_first_history_path(tmp_path: Path) -> None: + lineage, terminal = _component_failed_lineage(tmp_path) + selected, observation = _failed_terminal(lineage) + assert selected == terminal and selected.name != "00000000.json" + assert observation["instance_id"] == "D" and observation["outcome"] == "never-created" + assert observation["request_id"] == lineage.generations[0].request_id + assert observation["frontier"] == lineage.frontier + assert observation["original_file"]["inode"] == terminal.stat().st_ino + assert "independent_lineage_verified" not in observation, "a pure selector must never claim that the producer ran" + + +@pytest.mark.parametrize("fault", ["absent", "changed-bytes", "wrong-request", "duplicate-record", "duplicate-branch", "unknown", "incomplete"]) +def test_failed_branch_selector_refuses_missing_changed_or_ambiguous_originals(tmp_path: Path, fault: str) -> None: + import hashlib + import json + + lineage, terminal = _component_failed_lineage(tmp_path) + selected = next(original for original in lineage.originals if original.path == str(terminal)) + if fault == "absent": + lineage = replace(lineage, originals=tuple(original for original in lineage.originals if original != selected)) + elif fault == "changed-bytes": + terminal.write_bytes(terminal.read_bytes() + b"\x00") + elif fault == "wrong-request": + value = json.loads(terminal.read_bytes()) + value["attempt"] = "different-failed-request" + encoded = canonical(value) + terminal.write_bytes(encoded) + lineage = replace(lineage, originals=tuple(replace(original, sha256=hashlib.sha256(encoded).hexdigest()) if original == selected else original for original in lineage.originals)) + elif fault == "duplicate-record": + duplicate = terminal.with_name("00000008.json") + duplicate.write_bytes(terminal.read_bytes()) + duplicate.chmod(0o600) + info = duplicate.stat() + lineage = replace(lineage, originals=(*lineage.originals, replace(selected, path=str(duplicate), device=info.st_dev, inode=info.st_ino))) + elif fault == "duplicate-branch": + lineage = replace(lineage, generations=(*lineage.generations, replace(lineage.generations[0], instance_id="another-failure"))) + elif fault == "unknown": + lineage = replace(lineage, generations=(replace(lineage.generations[0], outcome="unknown"),)) + else: + lineage = replace(lineage, unavailable=("an expected original is missing",)) + with pytest.raises(AssertionError, match="failed.branch|terminal record"): + _failed_terminal(lineage) + + +def test_live_stopped_receiver_oracle_cannot_count_running_or_dead_peer_as_unacknowledged_custody() -> None: + """Observe an actual stopped process; this is not native Docker acceptance.""" + import select + import signal + import subprocess + import sys + import time + + from scripts.recovery_epoch_fault_tracer import _token + + process = subprocess.Popen([sys.executable, "-I", "-B", "-c", "import signal; print('alive', flush=True); signal.pause()"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + descriptor = os.pidfd_open(process.pid, 0) + try: + assert select.select((process.stdout,), (), (), 10)[0] + assert process.stdout.readline() == "alive\n" + token = _token(process.pid) + snapshot = _descriptor_snapshot(process.pid) + with pytest.raises(AssertionError, match="not still stopped"): + _stopped_receiver(process.pid, token, descriptor, snapshot) + signal.pidfd_send_signal(descriptor, signal.SIGSTOP) + deadline = time.monotonic() + 10 + while Path(f"/proc/{process.pid}/stat").read_text().rsplit(")", 1)[1].split()[0] not in {"T", "t"}: + assert time.monotonic() < deadline + time.sleep(0.01) + observation = _stopped_receiver(process.pid, token, descriptor, snapshot) + assert observation["pidfd_alive"] is True and observation["pid"] == process.pid and observation["start_token"] == token + with pytest.raises(AssertionError, match="not still stopped"): + _stopped_receiver(process.pid, token, descriptor, snapshot[:-1]) + signal.pidfd_send_signal(descriptor, signal.SIGKILL) + poll = select.poll() + poll.register(descriptor, select.POLLIN) + assert poll.poll(10000) + with pytest.raises(AssertionError, match="lifetime ended"): + _stopped_receiver(process.pid, token, descriptor, snapshot) + assert process.wait(timeout=10) == -signal.SIGKILL + finally: + if process.poll() is None: + signal.pidfd_send_signal(descriptor, signal.SIGKILL) + process.communicate(timeout=10) + os.close(descriptor) + + +@pytest.mark.parametrize("blocking", [True, False]) +def test_actual_queued_grant_peek_preserves_bytes_and_shared_socket_flags(blocking: bool) -> None: + """Use a real duplicate OFD; native VM separately requires pidfd_getfd.""" + import fcntl + + expected = canonical({"schema": "management-recovery-epoch-grant-v1", "component": "queued-frame"}) + b"\n" + with ExitStack() as held: + sender, receiver = socket.socketpair() + held.enter_context(sender) + held.enter_context(receiver) + receiver.setblocking(blocking) + duplicate = os.dup(receiver.fileno()) + held.callback(os.close, duplicate) + before = fcntl.fcntl(receiver.fileno(), fcntl.F_GETFL) + sender.sendall(expected) + encoded, observed_before, observed_after = _peek_queued_packet(duplicate, expected) + assert encoded == expected and observed_before == observed_after == before + assert fcntl.fcntl(receiver.fileno(), fcntl.F_GETFL) == before + assert _local_socket_peer(duplicate) == {"pid": os.getpid(), "uid": os.geteuid(), "gid": os.getegid()} + # This is the first consuming read. Both native peeks left the whole + # original frame available to the intended receiver's own descriptor. + assert receiver.recv(len(expected) + 1, socket.MSG_DONTWAIT) == expected + with pytest.raises(BlockingIOError): + receiver.recv(1, socket.MSG_DONTWAIT) + + +@pytest.mark.parametrize("fault", ["empty", "closed", "partial", "foreign", "additional", "oversized"]) +def test_actual_queued_grant_peek_cannot_substitute_expected_records_for_received_bytes(fault: str) -> None: + import fcntl + + expected = canonical({"schema": "management-recovery-epoch-grant-v1", "request_id": "component-request"}) + b"\n" + with ExitStack() as held: + sender, receiver = socket.socketpair() + held.enter_context(sender) + held.enter_context(receiver) + before = fcntl.fcntl(receiver.fileno(), fcntl.F_GETFL) + payload = b"" + if fault == "closed": + sender.close() + elif fault == "partial": + payload = expected[:-1] + elif fault == "foreign": + payload = canonical({"schema": "management-recovery-epoch-grant-v1", "request_id": "foreign-request"}) + b"\n" + elif fault == "additional": + payload = expected + b"unexpected-second-frame\n" + elif fault == "oversized": + payload = b"x" * (MAX_MESSAGE_BYTES + 1) + if payload: + sender.sendall(payload) + with pytest.raises(AssertionError, match="queued grant|consumed grant"): + _peek_queued_packet(receiver.fileno(), expected) + assert fcntl.fcntl(receiver.fileno(), fcntl.F_GETFL) == before + if payload: + assert receiver.recv(MAX_MESSAGE_BYTES + 2, socket.MSG_DONTWAIT) == payload + + +def test_actual_socket_peer_reader_does_not_accept_datagram_transport() -> None: + with ExitStack() as held: + sender, receiver = socket.socketpair(type=socket.SOCK_DGRAM) + held.enter_context(sender) + held.enter_context(receiver) + assert _local_socket_peer(receiver.fileno()) is None + + +@pytest.mark.parametrize("blocking", [True, False]) +def test_empty_or_queued_post_receipt_observation_never_consumes_or_changes_flags(blocking: bool) -> None: + import fcntl + + with ExitStack() as held: + sender, receiver = socket.socketpair() + held.enter_context(sender) + held.enter_context(receiver) + receiver.setblocking(blocking) + flags = fcntl.fcntl(receiver.fileno(), fcntl.F_GETFL) + assert _peek_pending_frame(receiver.fileno()) == b"" + sender.sendall(b"actual native grant bytes\n") + assert _peek_pending_frame(receiver.fileno()) == b"actual native grant bytes\n" + assert fcntl.fcntl(receiver.fileno(), fcntl.F_GETFL) == flags + assert receiver.recv(1024, socket.MSG_DONTWAIT) == b"actual native grant bytes\n" + sender.close() + assert _peek_pending_frame(receiver.fileno()) == b"" + assert fcntl.fcntl(receiver.fileno(), fcntl.F_GETFL) == flags diff --git a/tests/test_management_recovery_epoch_observability.py b/tests/test_management_recovery_epoch_observability.py new file mode 100644 index 000000000..c6b0ca872 --- /dev/null +++ b/tests/test_management_recovery_epoch_observability.py @@ -0,0 +1,71 @@ +"""Real durable-phase emitters; no grant, operation, or readiness evidence.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from auto_coder.management import recovery_epoch_state as epoch + +pytestmark = pytest.mark.usefixtures("_use_real_commands", "_use_real_sleep") + +_EMIT = r""" +import json,os,sys +from dataclasses import asdict +from pathlib import Path +from auto_coder.logger_config import setup_logger +from auto_coder.management import recovery_epoch_state as epoch +from auto_coder.management.original_attempts import canonical +root=Path(sys.argv[1])/'epoch' +root.mkdir(mode=0o700) +fd=os.open(root/'epoch.lock',os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600) +os.close(fd) +directory,lock=root.stat(),(root/'epoch.lock').stat() +inventory=epoch.EpochInventory(1,str(root),directory.st_dev,directory.st_ino,lock.st_dev,lock.st_ino,'component-deployment','component-authorization',('component-enrollment',),'component-request','component-preparation','a'*64,'b'*64,'c'*64) +fd=os.open(root/'inventory.json',os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600) +with os.fdopen(fd,'wb') as stream: + stream.write(canonical(asdict(inventory))) +boot,pid,start=epoch.issuer_lifetime() +binding=epoch.EpochBinding(inventory.deployment_id,inventory.authorization_id,inventory.enrollment_ids,inventory.request_id,inventory.preparation_id,inventory.target_id,inventory.source_sha256,inventory.config_sha256,'d'*64,boot,pid,start) +setup_logger(stream=sys.stdout,log_file=str(root.parent/'events.log')) +sink=epoch.logger.add(root.parent/'events.jsonl',serialize=True,rotation='1 MB',enqueue=False,diagnose=False,filter=lambda record:'recovery_epoch_phase' in record['extra']) +try: + with epoch._hold_recovery_epoch(root,expected_uid=os.geteuid()) as state: + state.reserve(binding) + state.record_intent(binding) + receipt=state.consume(binding) + epoch.logger.complete() +finally: + epoch.logger.remove(sink) +print(json.dumps(asdict(receipt),sort_keys=True),flush=True) +""" + + +def test_actual_durable_phase_logs_reach_stdout_and_file_without_success_or_readiness(tmp_path: Path) -> None: + result = subprocess.run([sys.executable, "-B", "-c", _EMIT, str(tmp_path)], text=True, capture_output=True, check=True, timeout=20) + emitted = [json.loads(line)["record"] for line in (tmp_path / "events.jsonl").read_text().splitlines()] + phases = ("reserved", "intent", "consumed", "receipt") + assert [record["extra"]["recovery_epoch_phase"] for record in emitted] == list(phases) + persisted_text = (tmp_path / "events.log").read_text() + for phase, record in zip(phases, emitted): + durable = json.loads((tmp_path / "epoch" / (phase + ".json")).read_text()) + assert durable["phase"] == phase and durable["binding"]["request_id"] == "component-request" + assert record["file"]["path"] == str(Path(epoch.__file__).resolve()) + assert record["file"]["name"] == "recovery_epoch_state.py" and record["function"] == "_append" and record["line"] > 0 + assert record["level"]["name"] == "INFO" + extra = record["extra"] + assert extra["request_id"] == "component-request" and extra["target_id"] == "a" * 64 + assert extra["grant_delivered"] is False + assert not {"operation_id", "event_type", "status", "outcome", "application_ready", "current_ready", "ready"} & set(extra) + expected_message = f"Recovery epoch {phase} durably recorded; no grant delivery observed" + assert record["message"] == expected_message + assert expected_message in result.stdout and expected_message in persisted_text + receipt = json.loads(result.stdout.splitlines()[-1]) + assert receipt["grant_delivered"] is False + assert set(receipt) == {"inventory_sha256", "binding_sha256", "consumed_sha256", "receipt_sha256", "grant_delivered"} + assert os.stat(tmp_path / "epoch" / "receipt.json").st_size > 0 diff --git a/tests/test_management_recovery_epoch_peer.py b/tests/test_management_recovery_epoch_peer.py new file mode 100644 index 000000000..2684fe959 --- /dev/null +++ b/tests/test_management_recovery_epoch_peer.py @@ -0,0 +1,368 @@ +"""Real local processes and kernel custody, not Docker/recovery grant evidence.""" + +from __future__ import annotations + +import array +import fcntl +import json +import os +import select +import socket +import subprocess +import sys +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +from dataclasses import dataclass, replace +from pathlib import Path +from typing import Iterator + +import pytest + +from auto_coder.management.recovery_epoch_peer import ExpectedPeer, HeldFence, HeldResource, PeerCustody, PeerSession, PeerUnavailable, inspect_held_resource + +pytestmark = pytest.mark.usefixtures("_use_real_commands", "_use_real_sleep") + +# The consumer speaks the public wire schema with real independent opens and +# flocks. No product observer or kernel authority is replaced by this fixture. +_CONSUMER = r""" +import array, fcntl, json, os, secrets, select, socket, sys, time +mailbox_fd, paths_json, mode, inherited_json = sys.argv[1:] +paths, inherited = json.loads(paths_json), json.loads(inherited_json) +owned = {} +for name, operation in (('original', fcntl.LOCK_SH), ('target', fcntl.LOCK_EX)): + if (name == 'original' and mode == 'inherited-original') or (name == 'target' and mode == 'inherited-target'): + owned[name] = inherited[name] + else: + owned[name] = os.open(paths[name], os.O_RDONLY | os.O_NOFOLLOW) + if not (name == 'original' and mode == 'unlocked-original'): + fcntl.flock(owned[name], operation | fcntl.LOCK_NB) +mailbox = socket.socket(fileno=int(mailbox_fd)) +connection, issuer_endpoint = socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM) +mailbox.sendmsg([b'p'], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, array.array('i', [issuer_endpoint.fileno()]))]) +issuer_endpoint.close() +mailbox.close() +connection.settimeout(10) +request = { + 'schema': 'management-recovery-peer-challenge-v1', + 'request_id': 'request-one', 'authorization_id': 'authorization-one', + 'target_id': 'target-one', 'challenge': secrets.token_hex(32), +} +if mode == 'wrong-target': request['target_id'] = 'target-two' +if mode == 'bad-challenge': request['challenge'] = 'old' +wire = json.dumps(request).encode() + b'\n' +if mode == 'oversized-challenge': wire = b' ' * 65537 + wire +if mode == 'duplicate-challenge': wire = wire.rstrip()[:-1] + b',"challenge":"' + b'b' * 64 + b'"}\n' +if mode == 'rights-challenge': + connection.sendmsg([wire], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, array.array('i', [owned['original']]))]) +else: + connection.sendall(wire) +print('challenge-sent', flush=True) +if mode in ('wrong-target', 'bad-challenge', 'oversized-challenge', 'duplicate-challenge', 'rights-challenge'): + sys.stdin.readline() + raise SystemExit(0) +if mode == 'queued-only': + if not select.select([connection], [], [], 10)[0]: raise RuntimeError('offer was never queued') + print('offer-queued', flush=True) + sys.stdin.readline() + raise SystemExit(0) +received = [] +content = bytearray() +while b'\n' not in content: + data, ancillary, flags, _ = connection.recvmsg(65537, socket.CMSG_SPACE(256), socket.MSG_CMSG_CLOEXEC) + if not data or flags & (socket.MSG_TRUNC | socket.MSG_CTRUNC): raise RuntimeError('incomplete offer') + content.extend(data) + for level, kind, payload in ancillary: + if (level, kind) != (socket.SOL_SOCKET, socket.SCM_RIGHTS): raise RuntimeError('unexpected control') + values = array.array('i'); values.frombytes(payload); received.extend(values) +offer = json.loads(content) +descriptors = [] +for resource in offer['resources']: + index = resource['transfer_index'] + descriptor = received[index] if index is not None else owned[resource['name']] + owned[resource['name']] = descriptor + descriptors.append({'name': resource['name'], 'fd': descriptor}) +print('offer-received', flush=True) +if sys.stdin.readline().strip() != 'receipt': raise SystemExit(0) +receipt = {key: offer[key] for key in ('request_id', 'authorization_id', 'target_id', 'challenge', 'receipt_nonce')} +receipt.update(schema='management-recovery-custody-receipt-v1', descriptors=descriptors) +if mode == 'omitted': descriptors.pop() +if mode == 'duplicate-fd': descriptors[-1]['fd'] = descriptors[0]['fd'] +if mode == 'duplicate-name': descriptors[-1]['name'] = descriptors[0]['name'] +if mode == 'unknown-name': descriptors[-1]['name'] = 'undeclared' +if mode == 'stale-nonce': receipt['receipt_nonce'] = '0' * 64 +if mode == 'wrong-fd': descriptors[2]['fd'] = os.open(paths['other'], os.O_RDONLY) +if mode == 'reopened-controller': descriptors[2]['fd'] = os.open(paths['controller'], os.O_RDONLY) +if mode == 'closed-controller': os.close(owned['controller']) +if mode == 'boolean-fd': descriptors[0]['fd'] = True +wire = json.dumps(receipt).encode() + b'\n' +if mode == 'rights-receipt': + connection.sendmsg([wire], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, array.array('i', [owned['original']]))]) +elif mode == 'drip-receipt': + for byte in wire: + try: connection.sendall(bytes([byte])) + except BrokenPipeError: break + time.sleep(0.04) +else: + connection.sendall(wire) +print('receipt-sent', flush=True) +for command in sys.stdin: + command = command.strip() + if command == 'exit': break + if command.startswith('close-'): + os.close(owned[command.removeprefix('close-')]) + elif command.startswith('unlock-'): + fcntl.flock(owned[command.removeprefix('unlock-')], fcntl.LOCK_UN) + else: raise RuntimeError('unsupported fixture command') + print(command, flush=True) +""" + + +@dataclass +class PeerCase: + process: subprocess.Popen[str] + connection: socket.socket + expected: ExpectedPeer + fences: tuple[HeldFence, ...] + resources: tuple[HeldResource, ...] + paths: dict[str, str] + + def line(self, expected: str) -> None: + assert self.process.stdout is not None + assert select.select([self.process.stdout], [], [], 10)[0], "peer did not reach the observable protocol barrier" + actual = self.process.stdout.readline().strip() + assert actual == expected, (actual, expected, self.process.poll()) + + def command(self, value: str) -> None: + assert self.process.stdin is not None + self.process.stdin.write(value + "\n") + self.process.stdin.flush() + + +@contextmanager +def peer_case(tmp_path: Path, mode: str = "valid", *, namespace: bool = False) -> Iterator[PeerCase]: + paths = {name: str(tmp_path / name) for name in ("original", "target", "controller", "other")} + descriptors: dict[str, int] = {} + process = None + connection = None + mailbox, child_mailbox = socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM) + mailbox.settimeout(10) + try: + for name, path in paths.items(): + descriptors[name] = os.open(path, os.O_RDWR | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600) + fcntl.flock(descriptors["original"], fcntl.LOCK_SH | fcntl.LOCK_NB) + fcntl.flock(descriptors["controller"], fcntl.LOCK_EX | fcntl.LOCK_NB) + inherited: dict[str, int] = {} + if mode == "inherited-original": + inherited["original"] = descriptors["original"] + elif mode == "inherited-target": + fcntl.flock(descriptors["target"], fcntl.LOCK_EX | fcntl.LOCK_NB) + inherited["target"] = descriptors["target"] + process = subprocess.Popen( + [sys.executable, "-I", "-B", "-c", _CONSUMER, str(child_mailbox.fileno()), json.dumps(paths), mode, json.dumps(inherited)], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + pass_fds=(*inherited.values(), child_mailbox.fileno()), + ) + child_mailbox.close() + # The independent consumer creates the session pair, so SO_PEERCRED is + # its real PID/UID. The inherited pair only transports the issuer end. + # This needs no listening socket, proc substitution or credential mock. + payload, ancillary, flags, _ = mailbox.recvmsg(1, socket.CMSG_SPACE(array.array("i").itemsize), socket.MSG_CMSG_CLOEXEC) + assert payload == b"p" and not flags & (socket.MSG_TRUNC | socket.MSG_CTRUNC) and len(ancillary) == 1 + level, kind, data = ancillary[0] + assert (level, kind) == (socket.SOL_SOCKET, socket.SCM_RIGHTS) + received = array.array("i") + received.frombytes(data) + assert len(received) == 1 + connection = socket.socket(fileno=received[0]) + start = Path(f"/proc/{process.pid}/stat").read_text().rsplit(")", 1)[1].split()[19] + expected = ExpectedPeer(process.pid, start, Path("/proc/sys/kernel/random/boot_id").read_text().strip(), os.geteuid(), "request-one", "authorization-one", "target-one") + fences = tuple(HeldFence(name, descriptors[name], kind) for name, kind in (("original", "original-sh"), ("target", "target-ex"), ("controller", "transferred-ex"))) + resources = () + if namespace: + descriptors["namespace"] = os.open("/proc/self/ns/net", os.O_RDONLY | os.O_CLOEXEC) + resources = (HeldResource("namespace", descriptors["namespace"]),) + case = PeerCase(process, connection, expected, fences, resources, paths) + case.line("challenge-sent") + yield case + finally: + if connection is not None: + connection.close() + mailbox.close() + child_mailbox.close() + if process is not None: + if process.poll() is None: + try: + assert process.stdin is not None + process.stdin.write("exit\n") + process.stdin.flush() + except BrokenPipeError: + pass + try: + process.communicate(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.communicate(timeout=5) + for descriptor in descriptors.values(): + os.close(descriptor) + + +def receive_custody(case: PeerCase, session: PeerSession) -> PeerCustody: + with ThreadPoolExecutor(max_workers=1) as workers: + future = workers.submit(session.offer_and_verify_custody, case.fences, case.resources) + case.line("offer-received") + assert not future.done(), "received descriptors alone must not establish acknowledged custody" + case.command("receipt") + case.line("receipt-sent") + return future.result(timeout=10) + + +@pytest.mark.parametrize("namespace", [False, True]) +def test_real_peer_independently_owns_fences_and_receives_exclusive_custody(tmp_path: Path, namespace: bool) -> None: + with peer_case(tmp_path, namespace=namespace) as case, PeerSession(case.connection, case.expected) as session: + custody = receive_custody(case, session) + assert custody.peer == case.expected + assert len(custody.challenge) == len(custody.receipt_nonce) == 64 + assert len(custody.resources) == 3 + int(namespace) + original, target, controller = custody.resources[:3] + assert original.peer_lock is not None and original.peer_lock.pid == case.process.pid and original.peer_lock.mode == "READ" + assert original.identity.lock is not None and original.identity.lock.pid == os.getpid() + assert target.peer_lock is not None and target.peer_lock.pid == case.process.pid and target.peer_lock.mode == "WRITE" + assert controller.peer_lock == controller.identity.lock + assert controller.peer_lock is not None and controller.peer_lock.pid == os.getpid() + if namespace: + assert custody.resources[-1].identity.kind == "network-namespace" and custody.resources[-1].peer_lock is None + session.check(custody) + assert not hasattr(session, "send_grant") + + +@pytest.mark.parametrize("mode,barrier", [("queued-only", "offer-queued"), ("valid", "offer-received")]) +def test_queued_or_received_descriptors_without_acknowledgment_never_complete(tmp_path: Path, mode: str, barrier: str) -> None: + with peer_case(tmp_path, mode) as case, PeerSession(case.connection, case.expected, timeout=0.5) as session: + with ThreadPoolExecutor(max_workers=1) as workers: + future = workers.submit(session.offer_and_verify_custody, case.fences) + case.line(barrier) + assert not future.done(), "the producer must wait for the actual challenge-bound receipt" + with pytest.raises(PeerUnavailable, match="could not be verified"): + future.result(timeout=5) + assert case.connection.fileno() == -1 + for fence in case.fences: + os.fstat(fence.descriptor) + + +@pytest.mark.parametrize( + "mode", + ["omitted", "duplicate-fd", "duplicate-name", "unknown-name", "stale-nonce", "wrong-fd", "reopened-controller", "closed-controller", "boolean-fd", "rights-receipt", "inherited-original", "inherited-target", "unlocked-original"], +) +def test_actual_bad_or_unheld_custody_refuses(tmp_path: Path, mode: str) -> None: + with peer_case(tmp_path, mode) as case, PeerSession(case.connection, case.expected) as session: + with pytest.raises(PeerUnavailable): + receive_custody(case, session) + assert case.connection.fileno() == -1 + + +@pytest.mark.parametrize("mode", ["wrong-target", "bad-challenge", "oversized-challenge", "duplicate-challenge", "rights-challenge"]) +def test_challenge_is_bounded_exact_and_cannot_supply_descriptors(tmp_path: Path, mode: str) -> None: + with peer_case(tmp_path, mode) as case: + before = len(tuple(Path("/proc/self/fd").iterdir())) + with pytest.raises((PeerUnavailable, ValueError)): + PeerSession(case.connection, case.expected) + assert case.connection.fileno() == -1 + assert len(tuple(Path("/proc/self/fd").iterdir())) <= before - 1 + + +@pytest.mark.parametrize("change", ["pid", "uid", "start_token", "boot_id"]) +def test_socket_credentials_and_current_process_lifetime_are_independent(tmp_path: Path, change: str) -> None: + with peer_case(tmp_path) as case: + values = {"pid": os.getpid(), "uid": os.geteuid() + 1, "start_token": str(int(case.expected.start_token) + 1), "boot_id": "00000000-0000-4000-8000-000000000001"} + with pytest.raises(PeerUnavailable): + PeerSession(case.connection, replace(case.expected, **{change: values[change]})) + assert case.connection.fileno() == -1 + + +@pytest.mark.parametrize("command", ["close-original", "unlock-original", "unlock-target", "close-controller", "unlock-controller", "close-namespace"]) +def test_custody_is_rechecked_after_receipt(tmp_path: Path, command: str) -> None: + with peer_case(tmp_path, namespace=True) as case, PeerSession(case.connection, case.expected) as session: + custody = receive_custody(case, session) + case.command(command) + case.line(command) + with pytest.raises(PeerUnavailable): + session.check(custody) + assert case.connection.fileno() == -1 + + +def test_exited_peer_never_remains_current(tmp_path: Path) -> None: + with peer_case(tmp_path) as case, PeerSession(case.connection, case.expected) as session: + custody = receive_custody(case, session) + case.command("exit") + case.process.wait(timeout=5) + with pytest.raises(PeerUnavailable): + session.check(custody) + + +@pytest.mark.parametrize("change", ["replacement", "missing", "protection", "host-unlock"]) +def test_changed_issuer_fence_refuses_after_receipt(tmp_path: Path, change: str) -> None: + with peer_case(tmp_path) as case, PeerSession(case.connection, case.expected) as session: + custody = receive_custody(case, session) + path = Path(case.paths["original"]) + if change in ("replacement", "missing"): + path.rename(path.with_name("preserved-original")) + if change == "replacement": + descriptor = os.open(path, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0o600) + os.close(descriptor) + elif change == "protection": + path.chmod(0o644) + else: + fcntl.flock(case.fences[0].descriptor, fcntl.LOCK_UN) + with pytest.raises(PeerUnavailable): + session.check(custody) + + +def test_duplicate_held_resource_cannot_be_disguised_with_another_name(tmp_path: Path) -> None: + with peer_case(tmp_path) as case, PeerSession(case.connection, case.expected) as session: + duplicate = replace(case.fences[0], name="another-original") + with pytest.raises(PeerUnavailable, match="duplicated"): + session.offer_and_verify_custody((*case.fences, duplicate)) + + +def test_custody_from_a_different_session_is_not_accepted(tmp_path: Path) -> None: + with peer_case(tmp_path) as case, PeerSession(case.connection, case.expected) as session: + custody = receive_custody(case, session) + with pytest.raises(PeerUnavailable, match="another session"): + session.check(replace(custody, receipt_nonce="0" * 64)) + + +def test_session_can_offer_only_once(tmp_path: Path) -> None: + with peer_case(tmp_path) as case, PeerSession(case.connection, case.expected) as session: + receive_custody(case, session) + with pytest.raises(PeerUnavailable, match="already offered"): + session.offer_and_verify_custody(case.fences) + + +def test_regular_file_cannot_be_claimed_as_namespace_resource(tmp_path: Path) -> None: + with peer_case(tmp_path) as case: + with pytest.raises(PeerUnavailable, match="network namespace"): + inspect_held_resource(HeldResource("pretend-namespace", case.fences[0].descriptor)) + + +def test_partial_receipt_does_not_extend_the_total_exchange_deadline(tmp_path: Path) -> None: + with peer_case(tmp_path, "drip-receipt") as case, PeerSession(case.connection, case.expected, timeout=0.2) as session: + with ThreadPoolExecutor(max_workers=1) as workers: + future = workers.submit(session.offer_and_verify_custody, case.fences) + case.line("offer-received") + case.command("receipt") + with pytest.raises(PeerUnavailable): + future.result(timeout=2) + + +def test_peer_exit_before_receipt_never_establishes_custody(tmp_path: Path) -> None: + with peer_case(tmp_path) as case, PeerSession(case.connection, case.expected) as session: + with ThreadPoolExecutor(max_workers=1) as workers: + future = workers.submit(session.offer_and_verify_custody, case.fences) + case.line("offer-received") + case.command("exit") + case.process.wait(timeout=5) + with pytest.raises(PeerUnavailable): + future.result(timeout=2) diff --git a/tests/test_management_recovery_epoch_state.py b/tests/test_management_recovery_epoch_state.py new file mode 100644 index 000000000..3511369b7 --- /dev/null +++ b/tests/test_management_recovery_epoch_state.py @@ -0,0 +1,417 @@ +"""Native persistence tests, separate from complete host-issuer acceptance. + +Inventories here are isolated state fixtures. They do not establish historical +completeness, independent containment or authorization to launch a supervisor. +""" + +import fcntl +import hashlib +import inspect +import json +import os +import select +import subprocess +import sys +from dataclasses import asdict, replace +from pathlib import Path + +import pytest + +from auto_coder.management import recovery_epoch_state as epoch +from auto_coder.management.original_attempts import canonical + +pytestmark = pytest.mark.usefixtures("_use_real_commands", "_use_real_sleep") + + +@pytest.fixture +def _use_real_commands() -> None: + pass + + +@pytest.fixture +def _use_real_sleep() -> None: + pass + + +def _write(path: Path, value: bytes) -> None: + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, "wb") as stream: + stream.write(value) + + +def _inventory(tmp_path: Path) -> tuple[Path, epoch.EpochInventory]: + root = tmp_path / "epoch" + root.mkdir(mode=0o700) + _write(root / "epoch.lock", b"") + directory, lock = root.stat(), (root / "epoch.lock").stat() + inventory = epoch.EpochInventory(1, str(root), directory.st_dev, directory.st_ino, lock.st_dev, lock.st_ino, "deployment-A", "reviewed-A", ("original-A", "original-B"), "request-A", "preparation-A", "a" * 64, "b" * 64, "c" * 64) + _write(root / "inventory.json", canonical(asdict(inventory))) + return root, inventory + + +def _binding(inventory: epoch.EpochInventory) -> epoch.EpochBinding: + boot, pid, start = epoch.issuer_lifetime() + return epoch.EpochBinding(inventory.deployment_id, inventory.authorization_id, inventory.enrollment_ids, inventory.request_id, inventory.preparation_id, inventory.target_id, inventory.source_sha256, inventory.config_sha256, "d" * 64, boot, pid, start) + + +def _snapshot(root: Path) -> dict[str, tuple[int, int, bytes]]: + return {path.name: (path.stat().st_dev, path.stat().st_ino, path.read_bytes()) for path in root.iterdir() if path.is_file()} + + +def _complete(state: epoch.RecoveryEpochState, binding: epoch.EpochBinding) -> epoch.EpochReceipt: + state.reserve(binding) + state.record_intent(binding) + return state.consume(binding) + + +def _line(process: subprocess.Popen[str]) -> str: + assert process.stdout is not None + assert select.select([process.stdout], [], [], 15)[0], "native process did not reach its observation barrier" + return process.stdout.readline().strip() + + +def test_durable_phases_bind_exact_inventory_and_never_reissue(tmp_path): + root, inventory = _inventory(tmp_path) + initial = _snapshot(root) + binding = _binding(inventory) + with epoch._hold_recovery_epoch(root, expected_uid=os.geteuid()) as state: + assert state.inventory == inventory + receipt = _complete(state, binding) + assert receipt.grant_delivered is False + assert receipt.inventory_sha256 == hashlib.sha256(initial["inventory.json"][2]).hexdigest() + assert receipt.binding_sha256 == hashlib.sha256(canonical(asdict(binding))).hexdigest() + assert receipt.consumed_sha256 == hashlib.sha256((root / "consumed.json").read_bytes()).hexdigest() + assert receipt.receipt_sha256 == hashlib.sha256((root / "receipt.json").read_bytes()).hexdigest() + with pytest.raises(epoch.EpochUnavailable, match="already consumed"): + state.consume(binding) + retained = _snapshot(root) + assert {"inventory.json", "epoch.lock"} | {phase + suffix for phase in ("reserved", "intent", "consumed", "receipt") for suffix in (".json", "-head.json")} == set(retained) + previous = "" + for sequence, phase in enumerate(("reserved", "intent", "consumed", "receipt")): + record = json.loads(retained[phase + ".json"][2]) + assert record == {"schema_version": 1, "phase": phase, "sequence": sequence, "previous": previous, "inventory_sha256": receipt.inventory_sha256, "binding": json.loads(canonical(asdict(binding)))} + head = json.loads(retained[phase + "-head.json"][2]) + assert head["record_sha256"] == hashlib.sha256(retained[phase + ".json"][2]).hexdigest() + assert head["previous"] == previous + previous = hashlib.sha256(retained[phase + "-head.json"][2]).hexdigest() + with pytest.raises(epoch.EpochUnavailable, match="closed"): + state.reserve(binding) + with pytest.raises(epoch.EpochUnavailable, match="existing, partial or conflicting"): + with epoch._hold_recovery_epoch(root, expected_uid=os.geteuid()): + pytest.fail("a lost receipt became a second invocation") + assert _snapshot(root) == retained + assert {name: retained[name] for name in initial} == initial + + +@pytest.mark.parametrize( + "field,value", + [ + ("deployment_id", "foreign"), + ("authorization_id", "unreviewed"), + ("enrollment_ids", ("different",)), + ("request_id", "other"), + ("preparation_id", "other"), + ("target_id", "e" * 64), + ("source_sha256", "e" * 64), + ("config_sha256", "e" * 64), + ("evidence_sha256", "unknown"), + ("host_pid", True), + ("host_start_token", "1"), + ("host_boot_id", "different"), + ], +) +def test_binding_differences_refuse_before_persistence(tmp_path, field, value): + root, inventory = _inventory(tmp_path) + initial = _snapshot(root) + with epoch._hold_recovery_epoch(root, expected_uid=os.geteuid()) as state: + with pytest.raises(epoch.EpochUnavailable): + state.reserve(replace(_binding(inventory), **{field: value})) + assert _snapshot(root) == initial + + +@pytest.mark.parametrize("field,value", [("authorization_id", "reviewed-B"), ("evidence_sha256", "e" * 64), ("host_start_token", "999")]) +def test_conflicting_binding_after_durable_reservation_poisons_invocation(tmp_path, field, value): + root, inventory = _inventory(tmp_path) + binding = _binding(inventory) + with pytest.raises(epoch.EpochUnavailable, match="closed, failed or inherited"): + with epoch._hold_recovery_epoch(root, expected_uid=os.geteuid()) as state: + state.reserve(binding) + with pytest.raises(epoch.EpochUnavailable): + state.record_intent(replace(binding, **{field: value})) + before = _snapshot(root) + with pytest.raises(epoch.EpochUnavailable, match="closed, failed or inherited"): + state.record_intent(binding) + assert _snapshot(root) == before + with pytest.raises(epoch.EpochUnavailable, match="existing, partial or conflicting"): + with epoch._hold_recovery_epoch(root, expected_uid=os.geteuid()): + pytest.fail("an interrupted authorization was reused") + assert _snapshot(root) == before + + +@pytest.mark.parametrize("name", ["inventory.json", "epoch.lock"]) +def test_missing_or_aliased_fixed_files_are_never_recreated(tmp_path, name): + root, _ = _inventory(tmp_path) + path = root / name + os.link(path, tmp_path / "alias") + before = _snapshot(root) + with pytest.raises(epoch.EpochUnavailable): + with epoch._hold_recovery_epoch(root, expected_uid=os.geteuid()): + pytest.fail("a hardlinked fixed file was accepted") + assert _snapshot(root) == before + path.unlink() + before = _snapshot(root) + with pytest.raises(epoch.EpochUnavailable): + with epoch._hold_recovery_epoch(root, expected_uid=os.geteuid()): + pytest.fail("a missing fixed file was synthesized") + assert _snapshot(root) == before + + +@pytest.mark.parametrize("mutation", ["unknown-field", "missing-field", "noncanonical", "wrong-root", "wrong-lock", "boolean-schema", "duplicate-enrollment"]) +def test_fixed_inventory_has_a_finite_exact_schema(tmp_path, mutation): + root, inventory = _inventory(tmp_path) + value = asdict(inventory) + if mutation == "unknown-field": + value["alternate_root"] = str(tmp_path) + elif mutation == "missing-field": + del value["authorization_id"] + elif mutation == "wrong-root": + value["root_inode"] = inventory.root_inode + 1 + elif mutation == "wrong-lock": + value["lock_inode"] = inventory.lock_inode + 1 + elif mutation == "boolean-schema": + value["schema_version"] = True + elif mutation == "duplicate-enrollment": + value["enrollment_ids"] = ["original-A", "original-A"] + encoded = json.dumps(value).encode() if mutation == "noncanonical" else canonical(value) + (root / "inventory.json").write_bytes(encoded) + before = _snapshot(root) + with pytest.raises(epoch.EpochUnavailable): + with epoch._hold_recovery_epoch(root, expected_uid=os.geteuid()): + pytest.fail("an invalid or aliased inventory was accepted") + assert _snapshot(root) == before + + +@pytest.mark.parametrize("name", ["reserved.json", "reserved-head.json", "intent.json", "consumed.json", "receipt-head.json", "pending.json"]) +def test_partial_or_conflicting_state_is_preserved_and_unavailable(tmp_path, name): + root, _ = _inventory(tmp_path) + _write(root / name, b'{"interrupted":') + before = _snapshot(root) + with pytest.raises(epoch.EpochUnavailable, match="existing, partial or conflicting"): + with epoch._hold_recovery_epoch(root, expected_uid=os.geteuid()): + pytest.fail("uncertain state was treated as a fresh epoch") + assert _snapshot(root) == before + + +@pytest.mark.parametrize("changed", ["inventory", "lock", "record", "root", "released"]) +def test_changes_after_reservation_refuse_without_repair(tmp_path, changed): + root, inventory = _inventory(tmp_path) + binding = _binding(inventory) + before: dict[str, tuple[int, int, bytes]] = {} + preserved = root + with pytest.raises(epoch.EpochUnavailable): + with epoch._hold_recovery_epoch(root, expected_uid=os.geteuid()) as state: + state.reserve(binding) + if changed == "root": + preserved = tmp_path / "detached" + root.rename(preserved) + root.mkdir(mode=0o700) + elif changed == "released": + fcntl.flock(state._lock, fcntl.LOCK_UN) + else: + path = root / {"inventory": "inventory.json", "lock": "epoch.lock", "record": "reserved.json"}[changed] + original = path.read_bytes() + path.unlink() + _write(path, original) + before = _snapshot(preserved) + state.record_intent(binding) + assert _snapshot(preserved) == before + assert "intent.json" not in before + + +@pytest.mark.parametrize("boundary", range(1, 25)) +@pytest.mark.parametrize("fail_after_sync", [False, True]) +def test_every_record_and_head_fsync_failure_preserves_uncertainty(tmp_path, monkeypatch, boundary, fail_after_sync): + root, inventory = _inventory(tmp_path) + original_fsync = os.fsync + count = 0 + + def interrupted(descriptor): + nonlocal count + count += 1 + if count == boundary and not fail_after_sync: + raise OSError("injected epoch persistence interruption") + original_fsync(descriptor) + if count == boundary and fail_after_sync: + raise OSError("injected epoch persistence interruption") + + with monkeypatch.context() as patch: + patch.setattr(os, "fsync", interrupted) + with pytest.raises(epoch.EpochUnavailable, match="injected epoch persistence interruption"): + with epoch._hold_recovery_epoch(root, expected_uid=os.geteuid()) as state: + _complete(state, _binding(inventory)) + assert count == boundary + before = _snapshot(root) + assert "reserved.json" in before + with pytest.raises(epoch.EpochUnavailable, match="existing, partial or conflicting"): + with epoch._hold_recovery_epoch(root, expected_uid=os.geteuid()): + pytest.fail("an uncertain persistence result was reset") + assert _snapshot(root) == before + + +@pytest.mark.parametrize("phase", ["reserved", "intent", "consumed", "receipt"]) +def test_cancellation_after_durable_publication_poisons_the_same_invocation(tmp_path, phase): + root, inventory = _inventory(tmp_path) + binding = _binding(inventory) + + def interrupt_after_real_publication(message): + if message.record["extra"].get("recovery_epoch_phase") == phase: + raise KeyboardInterrupt("cancel after durable " + phase) + + sink = epoch.logger.add(interrupt_after_real_publication, catch=False) + try: + with pytest.raises(epoch.EpochUnavailable, match="closed, failed or inherited"): + with epoch._hold_recovery_epoch(root, expected_uid=os.geteuid()) as state: + with pytest.raises(KeyboardInterrupt, match="cancel after durable"): + _complete(state, binding) + assert (root / (phase + ".json")).is_file() + assert (root / (phase + "-head.json")).is_file() + before = _snapshot(root) + # Catching cancellation must not resurrect the same in-memory + # holder and complete issuance after an interrupted boundary. + with pytest.raises(epoch.EpochUnavailable, match="closed, failed or inherited"): + state.consume(binding) + assert _snapshot(root) == before + finally: + epoch.logger.remove(sink) + with pytest.raises(epoch.EpochUnavailable, match="existing, partial or conflicting"): + with epoch._hold_recovery_epoch(root, expected_uid=os.geteuid()): + pytest.fail("canceled publication was reset by another invocation") + assert _snapshot(root) == before + + +@pytest.mark.parametrize("boundary", [3, 6, 9, 12, 15, 18, 21, 24]) +def test_publication_readback_cannot_adopt_a_same_byte_replacement(tmp_path, monkeypatch, boundary): + root, inventory = _inventory(tmp_path) + original_fsync = os.fsync + count = 0 + replaced = None + before: dict[str, tuple[int, int, bytes]] = {} + names = [phase + suffix for phase in ("reserved", "intent", "consumed", "receipt") for suffix in (".json", "-head.json")] + + def replace_after_real_publication(descriptor): + nonlocal count, replaced, before + original_fsync(descriptor) + count += 1 + if count == boundary: + path = root / names[boundary // 3 - 1] + original_inode = path.stat().st_ino + replacement = tmp_path / "unfsynced-copy" + _write(replacement, path.read_bytes()) + replaced = replacement.stat().st_ino + assert replaced != original_inode + replacement.replace(path) + before = _snapshot(root) + + monkeypatch.setattr(os, "fsync", replace_after_real_publication) + with pytest.raises(epoch.EpochUnavailable, match="append identity or readback differs"): + with epoch._hold_recovery_epoch(root, expected_uid=os.geteuid()) as state: + _complete(state, _binding(inventory)) + assert count == boundary and replaced is not None + assert _snapshot(root) == before + with pytest.raises(epoch.EpochUnavailable, match="existing, partial or conflicting"): + with epoch._hold_recovery_epoch(root, expected_uid=os.geteuid()): + pytest.fail("replaced publication was repaired by another invocation") + assert _snapshot(root) == before + + +def test_two_native_contenders_reach_the_held_deployment_flock(tmp_path): + root, _ = _inventory(tmp_path) + first_code = """ +import os, sys +from pathlib import Path +from auto_coder.management.recovery_epoch_state import _hold_recovery_epoch +from tests.test_management_recovery_epoch_state import _binding +with _hold_recovery_epoch(Path(sys.argv[1]), expected_uid=os.geteuid()) as state: + state.reserve(_binding(state.inventory)) + print('RESERVED_WITH_FLOCK', flush=True) + sys.stdin.readline() +""" + contender_code = """ +import os, sys +from pathlib import Path +from auto_coder.management.recovery_epoch_state import _hold_recovery_epoch, EpochUnavailable +try: + with _hold_recovery_epoch(Path(sys.argv[1]), expected_uid=os.geteuid()): + raise AssertionError('second contender entered') +except EpochUnavailable as error: + print(str(error), flush=True) +""" + first = subprocess.Popen([sys.executable, "-c", first_code, str(root)], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + try: + assert _line(first) == "RESERVED_WITH_FLOCK" + before = _snapshot(root) + second = subprocess.run([sys.executable, "-c", contender_code, str(root)], capture_output=True, text=True, timeout=15, check=True) + assert second.stdout.strip() == "recovery epoch admission contested" + assert first.poll() is None + assert _snapshot(root) == before + finally: + first.kill() + first.communicate(timeout=10) + assert first.returncode == -9 + with pytest.raises(epoch.EpochUnavailable, match="existing, partial or conflicting"): + with epoch._hold_recovery_epoch(root, expected_uid=os.geteuid()): + pytest.fail("host death erased the deployment reservation") + assert _snapshot(root) == before + + +@pytest.mark.parametrize("boundary", [3, 6, 9, 12, 15, 18, 21, 24]) +def test_native_host_death_after_each_record_or_head_keeps_the_epoch_burned(tmp_path, boundary): + root, _ = _inventory(tmp_path) + code = """ +import os, sys +from pathlib import Path +from auto_coder.management.recovery_epoch_state import _hold_recovery_epoch +from tests.test_management_recovery_epoch_state import _binding, _complete +original = os.fsync +count = 0 +def barrier(fd): + global count + original(fd) + count += 1 + if count == int(sys.argv[2]): + print('DURABLE_BOUNDARY:' + str(count), flush=True) + sys.stdin.readline() +with _hold_recovery_epoch(Path(sys.argv[1]), expected_uid=os.geteuid()) as state: + os.fsync = barrier + _complete(state, _binding(state.inventory)) +""" + process = subprocess.Popen([sys.executable, "-c", code, str(root), str(boundary)], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + try: + assert _line(process) == "DURABLE_BOUNDARY:" + str(boundary) + before = _snapshot(root) + finally: + process.kill() + process.communicate(timeout=10) + assert process.returncode == -9 + with pytest.raises(epoch.EpochUnavailable, match="existing, partial or conflicting"): + with epoch._hold_recovery_epoch(root, expected_uid=os.geteuid()): + pytest.fail("host death produced a reusable recovery grant") + assert _snapshot(root) == before + + +def test_production_factory_ignores_environment_roots_and_has_no_override(tmp_path, monkeypatch): + assert list(inspect.signature(epoch.hold_recovery_epoch).parameters) == [] + seen = [] + + def stop_before_open(root, *, uid): + seen.append((root, uid)) + raise epoch.EpochUnavailable("fixed-root unit observation") + + monkeypatch.setenv("RECOVERY_EPOCH_ROOT", str(tmp_path)) + monkeypatch.setenv("AUTO_CODER_RECOVERY_EPOCH_ROOT", str(tmp_path)) + monkeypatch.setattr(epoch.os, "geteuid", lambda: 0) + monkeypatch.setattr(epoch, "_approved_directory", stop_before_open) + with pytest.raises(epoch.EpochUnavailable, match="fixed-root unit observation"): + with epoch.hold_recovery_epoch(): + pytest.fail("unit seam should stop before real fixed-root access") + assert seen == [(Path("/var/lib/auto-coder/recovery-epoch"), 0)] diff --git a/tests/test_management_recovery_fixture_finalization.py b/tests/test_management_recovery_fixture_finalization.py new file mode 100644 index 000000000..7f4f0925b --- /dev/null +++ b/tests/test_management_recovery_fixture_finalization.py @@ -0,0 +1,233 @@ +"""Real original SQLite chronology with explicit component Docker state inputs. + +Only the separate native VM proves actual stopped Docker/host producers. These +regressions exercise their bounded fixture finalization against real SQLite, +RetainedJournal and QuarantineHistory without changing those production readers. +""" + +from __future__ import annotations + +import gc +import json +import os +import subprocess +import sys +from contextlib import closing +from pathlib import Path +from types import SimpleNamespace + +import pytest +from scripts import test_management_recovery_epoch_host as fixture + +from auto_coder.management.bootstrap_store import BootstrapStore +from auto_coder.management.operations import OperationPhase +from auto_coder.management.quarantine_history import QuarantineHistory, QuarantineUnavailable +from auto_coder.management.readiness_observation import _connect as _readiness_store +from auto_coder.management.recovery_mailbox import RecoveryMailbox +from auto_coder.management.retained_journal import RetainedJournal +from auto_coder.management.self_update_handoff import HandoffStore +from tests.test_management_archived_cutover_container import Chain, Generation +from tests.test_management_quarantine_history import _store + + +@pytest.fixture +def originals(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + epoch = tmp_path / "epoch" + epoch.mkdir() + (epoch / "epoch.lock").touch() + monkeypatch.setattr(fixture, "EPOCH_ROOT", epoch) + monkeypatch.setattr(fixture, "AUTHORIZATION_FILE", tmp_path / "authorization.json") + chain = Chain.__new__(Chain) + chain.root = tmp_path / "deployment" + chain.root.mkdir() + chain.generations = [] + cutover = subprocess.Popen([sys.executable, "-I", "-B", "-c", "pass"]) + assert cutover.wait(timeout=10) == 0 + chain.processes = [cutover] + chain.docker = lambda *_args: json.dumps({"Status": "exited", "Running": False, "Pid": 0, "Restarting": False}) + journals, captures = {}, [] + for name in ("A", "B", "C"): + root = chain.root / name + root.mkdir() + journal, _ = _store(root / "journal", name, management=True, phase=OperationPhase.NEEDS_ATTENTION if name == "C" else OperationPhase.SUCCEEDED) + journals[name] = journal + state = root / "state" + state.mkdir(mode=0o700) + previous_umask = os.umask(0o077) + try: + with closing(BootstrapStore(state, name)._connect()): + pass + with closing(HandoffStore(state, name)._connect()): + pass + RecoveryMailbox(state, name).initialize() + finally: + os.umask(previous_umask) + assert {path.name for path in state.glob("*.sqlite3")} == {"management-child.sqlite3", "management-self-update.sqlite3", "management-recovery.sqlite3"} + for path in state.glob("*.sqlite3"): + Chain.checkpoint(path) + if name in {"A", "B"}: + # Exercise the production observation-store constructor. This + # component payload makes no authenticated-readiness assertion. + with closing(_readiness_store(SimpleNamespace(state_root=state))) as connection, connection: + assert connection.execute("PRAGMA journal_mode").fetchone() == ("delete",) + connection.execute("INSERT INTO proof VALUES (?, ?)", (name, json.dumps({"component_observation": name}))) + generation = Generation(name, root, "manager-" + name, [], proxy="proxy-" + name) + chain.generations.append(generation) + original = SimpleNamespace(generation_id=name, instance_id=name, state="started", management_id=generation.container, proxy_id=generation.proxy) + captures.append(SimpleNamespace(original=original)) + failed = Generation("D", chain.root / "D", "never-created-manager", [], proxy="never-created-proxy") + failed.root.mkdir() + (failed.root / "retained-never-created").write_bytes(b"original failed branch") + chain.generations.append(failed) + return chain, tuple(captures), journals + + +def _late_archived_read(journal) -> None: + # This earlier checkpoint precedes the genuine later archived read. + Chain.checkpoint(journal.database) + assert not Path(str(journal.database) + "-shm").exists() + with RetainedJournal(journal.database, journal.fence, database_identity=(journal.database_device, journal.database_inode), fence_identity=(journal.fence_device, journal.fence_inode), prior_management_instance="A") as reader: + assert len(reader.read()) == 1 + assert Path(str(journal.database) + "-wal").stat().st_size == 0 + assert Path(str(journal.database) + "-shm").stat().st_size > 0 + + +def _retained_bytes(chain) -> dict[str, tuple[bytes, int, int, int]]: + result = {} + for path in sorted(chain.root.rglob("*")): + if path.is_file() and not path.is_symlink(): + info = path.stat() + result[str(path)] = (path.read_bytes(), info.st_dev, info.st_ino, info.st_ctime_ns) + return result + + +def test_final_checkpoint_follows_last_archived_read_and_preserves_original_authority(originals, monkeypatch: pytest.MonkeyPatch) -> None: + chain, captures, journals = originals + actual, checkpoints = Chain.checkpoint, [] + + def observed(database: Path) -> None: + checkpoints.append(database) + actual(database) + + collection = gc.isenabled() + gc.disable() + try: + _late_archived_read(journals["A"]) + before = _retained_bytes(chain) + with pytest.raises(QuarantineUnavailable, match="nonempty transaction sidecar"): + QuarantineHistory((journals["A"],)) + monkeypatch.setattr(Chain, "checkpoint", observed) + fixture._finalize_original_stores(chain, captures) + after = _retained_bytes(chain) + assert {name: value for name, value in before.items() if not name.endswith(("-wal", "-shm", "-journal"))} == after + assert (chain.root / "D" / "retained-never-created").read_bytes() == b"original failed branch" + assert len(checkpoints) == 12 and {path.name for path in checkpoints} == {"management-child.sqlite3", "management-self-update.sqlite3", "management-recovery.sqlite3", "management-operations.sqlite3"} + assert len([name for name in after if name.endswith(".sqlite3")]) == 14 + assert not (chain.root / "C" / "state" / "management-readiness.sqlite3").exists() + for name, journal in journals.items(): + with QuarantineHistory((journal,)) as history: + report = history.inspect() + assert report.historical_keys == 1 + assert report.unresolved_targets == (1 if name == "C" else 0) + assert report.launch_authority is False + assert _retained_bytes(chain) == after + finally: + if collection: + gc.enable() + + +@pytest.mark.parametrize("name", ["management-self-update.sqlite3", "management-recovery.sqlite3"]) +def test_all_real_original_mailboxes_are_required_and_never_reconstructed(originals, name: str) -> None: + chain, captures, _ = originals + missing = chain.root / "B" / "state" / name + missing.unlink() + before = _retained_bytes(chain) + with pytest.raises(FileNotFoundError): + fixture._finalize_original_stores(chain, captures) + assert not missing.exists() and _retained_bytes(chain) == before + + +@pytest.mark.parametrize("generation", ["A", "B"]) +def test_completed_readiness_observations_are_required_and_never_reconstructed(originals, generation: str) -> None: + chain, captures, journals = originals + _late_archived_read(journals["A"]) + missing = chain.root / generation / "state" / "management-readiness.sqlite3" + missing.unlink() + before = _retained_bytes(chain) + with pytest.raises(FileNotFoundError): + fixture._finalize_original_stores(chain, captures) + assert not missing.exists() and _retained_bytes(chain) == before + + +def test_unresolved_cannot_acquire_a_readiness_observation_during_finalization(originals) -> None: + chain, captures, journals = originals + _late_archived_read(journals["A"]) + with closing(_readiness_store(SimpleNamespace(state_root=chain.root / "C" / "state"))): + pass + before = _retained_bytes(chain) + with pytest.raises(AssertionError, match="unknown store"): + fixture._finalize_original_stores(chain, captures) + assert _retained_bytes(chain) == before + + +@pytest.mark.parametrize("suffix", ["-wal", "-shm", "-journal"]) +def test_observation_sidecars_refuse_without_settling_any_original_store(originals, suffix: str) -> None: + chain, captures, journals = originals + _late_archived_read(journals["A"]) + sidecar = chain.root / "B" / "state" / ("management-readiness.sqlite3" + suffix) + sidecar.touch(mode=0o600) + before = _retained_bytes(chain) + with pytest.raises(AssertionError, match="readiness observation still has a transaction sidecar"): + fixture._finalize_original_stores(chain, captures) + assert _retained_bytes(chain) == before + + +@pytest.mark.parametrize("fault", ["nonempty-wal", "hot-journal", "missing-store", "unknown-store", "linked-sidecar", "running-manager", "running-proxy", "live-cutover", "published-inventory", "installed-authorization", "target-prepared", "wrong-capture", "missing-capture"]) +def test_finalization_refuses_uncertain_scope_before_any_checkpoint(originals, fault: str) -> None: + chain, captures, journals = originals + _late_archived_read(journals["A"]) + stopped = chain.docker + if fault in {"nonempty-wal", "hot-journal"}: + path = Path(str(journals["B"].database) + ("-wal" if fault == "nonempty-wal" else "-journal")) + path.write_bytes(b"retained transaction uncertainty") + path.chmod(0o600) + elif fault == "missing-store": + journals["B"].database.unlink() + elif fault == "unknown-store": + (journals["B"].database.parent / "unknown.sqlite3").write_bytes(b"unknown original store") + elif fault == "linked-sidecar": + Path(str(journals["B"].database) + "-shm").symlink_to(journals["A"].database) + elif fault in {"running-manager", "running-proxy"}: + selected = "manager-B" if fault == "running-manager" else "proxy-B" + chain.docker = lambda *args: json.dumps({"Status": "running", "Running": True, "Pid": 123, "Restarting": False}) if args[-1] == selected else stopped(*args) + elif fault == "live-cutover": + chain.processes = [SimpleNamespace(poll=lambda: None)] + elif fault == "published-inventory": + (fixture.EPOCH_ROOT / "inventory.json").write_bytes(b"already selected recovery epoch") + elif fault == "installed-authorization": + fixture.AUTHORIZATION_FILE.write_bytes(b"already authorized recovery") + elif fault == "target-prepared": + (chain.root / "target").mkdir() + elif fault == "wrong-capture": + captures[1].original.management_id = "another-manager" + else: + captures = captures[:-1] + before = _retained_bytes(chain) + with pytest.raises((AssertionError, ValueError, FileNotFoundError)): + fixture._finalize_original_stores(chain, captures) + assert _retained_bytes(chain) == before + assert Path(str(journals["A"].database) + "-shm").stat().st_size > 0 + + +def test_finalization_rejects_any_database_change_even_if_checkpoint_returns(originals, monkeypatch: pytest.MonkeyPatch) -> None: + chain, captures, _ = originals + actual = Chain.checkpoint + + def changed(database: Path) -> None: + actual(database) + with database.open("ab") as stream: + stream.write(b"unexpected original database change") + + monkeypatch.setattr(Chain, "checkpoint", changed) + with pytest.raises(AssertionError, match="changed original database bytes or identity"): + fixture._finalize_original_stores(chain, captures) diff --git a/tests/test_management_recovery_proxy_profiles.py b/tests/test_management_recovery_proxy_profiles.py new file mode 100644 index 000000000..ba15fcc44 --- /dev/null +++ b/tests/test_management_recovery_proxy_profiles.py @@ -0,0 +1,76 @@ +"""Real protected mount checks for recovery's two explicit proxy profiles. + +The typed selector and Docker description are component inputs, not evidence +of a real running deployment. Full native capture remains a separate VM gate. +""" + +from __future__ import annotations + +import hashlib +import os +import pwd +import tempfile +from contextlib import contextmanager +from dataclasses import replace +from pathlib import Path + +import pytest + +from auto_coder.management.containment_route_manifest import DeploymentRootBinding +from auto_coder.management.recovery_containment_model import RecoveryContainmentUnavailable, RecoveryFile +from auto_coder.management.recovery_containment_profile import digest, validate_proxy_profile +from tests.test_management_recovery_containment import _enrollment + + +@contextmanager +def proxy_case(destinations: tuple[tuple[str, bool], ...]): + with tempfile.TemporaryDirectory(prefix="ac-recovery-proxy-", dir=pwd.getpwuid(os.geteuid()).pw_dir) as directory: + root = Path(directory) + config = root / "Caddyfile" + config.write_bytes(b"{\n admin off\n auto_https off\n}\n") + config.chmod(0o600) + selected = _enrollment().originals[0] + bindings = [] + mounts = [] + for index, (destination, writable) in enumerate(destinations): + path = config if destination == "/etc/caddy/Caddyfile" else root / str(index) + if path != config: + path.mkdir(mode=0o700) + info = path.stat() + bindings.append(DeploymentRootBinding(selected.proxy_id, destination, str(path), writable, info.st_dev, info.st_ino, info.st_mode)) + mounts.append({"Type": "bind", "Source": str(path), "Destination": destination, "RW": writable}) + info = config.stat() + selected = replace(selected, proxy_config=RecoveryFile(str(config), hashlib.sha256(config.read_bytes()).hexdigest(), info.st_dev, info.st_ino), mounts=tuple(bindings)) + inspection = { + "Id": selected.proxy_id, + "Image": selected.proxy_image_id, + "HostConfig": {"Privileged": False, "PidMode": "", "CapAdd": None, "Devices": [], "SecurityOpt": None, "NetworkMode": "container:" + selected.management_id}, + "Config": {"User": "", "WorkingDir": "/srv", "Entrypoint": ["caddy"], "Cmd": ["run", "--config", "/etc/caddy/Caddyfile", "--adapter", "caddyfile"]}, + "Mounts": mounts, + } + yield selected, inspection + + +@pytest.mark.parametrize("destinations", [(("/etc/caddy/Caddyfile", False),), (("/etc/caddy/Caddyfile", False), ("/data", True), ("/config", True))]) +def test_recovery_proxy_accepts_exact_ordinary_or_original_storage_effects(destinations): + with proxy_case(destinations) as (selected, inspection): + assert validate_proxy_profile(selected, inspection, expected_uid=os.geteuid()) == digest([inspection]) + + +@pytest.mark.parametrize( + "destinations", + [ + (("/etc/caddy/Caddyfile", False), ("/data", True)), + (("/etc/caddy/Caddyfile", False), ("/config", True)), + (("/etc/caddy/Caddyfile", False), ("/data", False), ("/config", True)), + (("/etc/caddy/Caddyfile", False), ("/data", True), ("/config", False)), + (("/etc/caddy/Caddyfile", True),), + (("/etc/caddy/Caddyfile", False), ("/unselected", True)), + (("/etc/caddy/Caddyfile", False), ("/unselected", False)), + (("/etc/caddy/Caddyfile", False), ("/data", True), ("/config", True), ("/extra", False)), + ], +) +def test_recovery_proxy_refuses_partial_or_additional_effects(destinations): + with proxy_case(destinations) as (selected, inspection): + with pytest.raises(RecoveryContainmentUnavailable, match="mount effects"): + validate_proxy_profile(selected, inspection, expected_uid=os.geteuid()) diff --git a/tests/test_management_recovery_vm.py b/tests/test_management_recovery_vm.py new file mode 100644 index 000000000..3115ef4ea --- /dev/null +++ b/tests/test_management_recovery_vm.py @@ -0,0 +1,238 @@ +"""Component checks execute the real guest setup and systemd fact parser.""" + +from __future__ import annotations + +import json +import os +import subprocess +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from auto_coder.management import containment_route_manifest as routes + +GUEST = Path(__file__).resolve().parents[1] / "scripts/recovery_vm/guest.sh" +MAINTENANCE_COMMAND = ".venv/bin/python - \"$output/maintenance-profile.json\" <<'PY'\n" +PAIRS = ( + ("systemd-tmpfiles-clean.timer", "systemd-tmpfiles-clean.service"), + ("apt-daily.timer", "apt-daily.service"), + ("apt-daily-upgrade.timer", "apt-daily-upgrade.service"), + ("dpkg-db-backup.timer", "dpkg-db-backup.service"), + ("e2scrub_all.timer", "e2scrub_all.service"), + ("fstrim.timer", "fstrim.service"), +) +TIMERS = tuple(timer for timer, _ in PAIRS) +SERVICES = tuple(service for _, service in PAIRS) +UNITS = tuple(unit for pair in PAIRS for unit in pair) + + +class SystemdFixture: + """Supply systemctl readback while retaining real parser and mask objects.""" + + def __init__(self, root: Path, fault: str | None = None, fault_unit: str = SERVICES[0]) -> None: + self.root = root + self.fault = fault + self.fault_unit = fault_unit + self.calls: list[tuple[str, ...]] = [] + self.stopped: set[str] = set() + self.masked = False + self.reads = {unit: 0 for unit in UNITS} + self.timer_rows: list[dict[str, str]] = [] + + def command(self, *arguments: str) -> str: + assert arguments[0] == "systemctl" and arguments[-1] in UNITS + unit = arguments[-1] + service = unit in SERVICES + if arguments[1] == "cat": + return "# Installed maintenance unit\n" + ("[Service]\nExecStart=/usr/bin/true\n" if service else "[Timer]\nOnCalendar=daily\n") + assert arguments[1] == "show" + state = { + "Id": unit, + "LoadState": "masked" if self.masked else "loaded", + "UnitFileState": "masked" if self.masked else "static", + "ActiveState": "inactive" if unit in self.stopped else "active", + "SubState": "dead" if unit in self.stopped else "running" if service else "waiting", + "NeedDaemonReload": "no", + "FragmentPath": str(self.root / unit) if self.masked else "/usr/lib/systemd/system/" + unit, + } + if service: + state["MainPID"] = "0" if unit in self.stopped else "123" + if unit == self.fault_unit: + if not self.masked and unit not in self.stopped and self.fault and self.fault.startswith("initial:"): + _, key, value = self.fault.split(":", 2) + state[key] = value + if not self.masked and self.fault == "missing": + state.update(LoadState="not-found", UnitFileState="", ActiveState="inactive", FragmentPath="") + if not self.masked and unit in self.stopped and self.fault == "still-running": + state.update(ActiveState="active", SubState="running", MainPID="123") + if self.masked: + self.reads[unit] += 1 + if unit == self.fault_unit and self.fault and self.fault.startswith("property:"): + _, key, value = self.fault.split(":", 2) + if value == "missing": + state.pop(key) + else: + state[key] = value + if unit == self.fault_unit and self.fault == "changed-mask" and self.reads[unit] == 2: + path = self.root / unit + path.rename(self.root / "retained-original-mask") + path.symlink_to("/dev/null") + if not service: + self.timer_rows.append(state.copy()) + return "\n".join(key + "=" + value for key, value in state.items()) + + def run(self, command: tuple[str, ...], *, check: bool, timeout: int) -> subprocess.CompletedProcess[str]: + assert check is True and timeout == 30 + self.calls.append(command) + if command[:3] == ("systemctl", "stop", "--"): + unit = command[3] + assert unit in UNITS + if unit in SERVICES: + assert set(TIMERS) <= self.stopped, "Every selected timer must stop before any service" + if self.fault == "stop-failed" and unit == self.fault_unit: + raise subprocess.CalledProcessError(1, command) + self.stopped.add(unit) + elif command == ("systemctl", "mask", "--", *UNITS): + assert self.stopped == set(UNITS) + for unit in UNITS: + path = self.root / unit + if self.fault == "regular-mask" and unit == self.fault_unit: + path.write_text("not a mask") + else: + path.symlink_to("/dev/zero" if self.fault == "wrong-mask" and unit == self.fault_unit else "/dev/null") + self.masked = True + else: + assert command == ("systemctl", "daemon-reload") and self.masked + return subprocess.CompletedProcess(command, 0, "", "") + + +@pytest.fixture +def maintenance(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + text = GUEST.read_text() + assert text.count(MAINTENANCE_COMMAND) == 1 + body = text.split(MAINTENANCE_COMMAND, 1)[1].split("\nPY\n", 1)[0] + namespace = {"__name__": "recovery_vm_maintenance_component"} + exec(compile(body, str(GUEST), "exec"), namespace) + mask_root = tmp_path / "masks" + mask_root.mkdir() + namespace["MASK_ROOT"] = mask_root + actual_reader = routes._systemd_unit + namespace["_systemd_unit"] = lambda unit: actual_reader(unit, trusted_uid=os.geteuid()) + + def prepare(fault: str | None = None, fault_unit: str = SERVICES[0]) -> SystemdFixture: + fixture = SystemdFixture(mask_root, fault, fault_unit) + monkeypatch.setattr(routes, "_command", fixture.command) + namespace["subprocess"] = SimpleNamespace(run=fixture.run) + return fixture + + return namespace["inhibit_maintenance"], prepare, tmp_path / "maintenance-profile.json", mask_root + + +def test_guest_inhibits_only_exact_pairs_before_originals_and_control_copy() -> None: + text = GUEST.read_text() + setup = text.index(MAINTENANCE_COMMAND) + assert text.index("verify_installed_watcher\n") < setup + assert setup < text.index("driver_module=scripts.test_management_recovery_epoch_host") + assert setup < text.index("scripts.recovery_epoch_mutation_control prepare") + assert setup < text.index('fd = os.open("/etc/auto-coder-disposable-containment-host.json"') + assert setup < text.index('.venv/bin/python -m "$driver_module"') + subprocess.run(("bash", "-n", str(GUEST)), check=True, timeout=10) + + +def test_real_guest_block_stops_all_timers_before_services_and_retains_actual_masks(maintenance) -> None: + inhibit, prepare, output, masks = maintenance + systemd = prepare() + inhibit(output) + assert systemd.calls == [ + *(("systemctl", "stop", "--", unit) for unit in (*TIMERS, *SERVICES)), + ("systemctl", "mask", "--", *UNITS), + ("systemctl", "daemon-reload"), + ] + evidence = json.loads(output.read_text()) + assert evidence["schema"] == "auto-coder-recovery-vm-maintenance-profile-v1" and evidence["status"] == "inhibited" + assert [fact["unit"] for fact in evidence["initial"]] == list(UNITS) + assert [fact["unit"] for fact in evidence["stopped"]] == list(UNITS) + assert [fact["unit"] for fact in evidence["observed"]] == list(UNITS) + assert all(fact["main_pid"] == 123 for fact in evidence["initial"] if fact["unit"] in SERVICES) + assert all(fact["active_state"] == "inactive" and fact["main_pid"] == 0 for fact in evidence["stopped"]) + assert all(fact["inhibited"] and fact["unit_file_state"] == "masked" and fact["main_pid"] == 0 and not fact["needs_daemon_reload"] for fact in evidence["observed"]) + assert all("MainPID" not in row for row in systemd.timer_rows) + assert systemd.reads == dict.fromkeys(UNITS, 2) + assert [record["unit"] for record in evidence["masks"]] == list(UNITS) + for record in evidence["masks"]: + path = masks / record["unit"] + info = path.lstat() + assert record == {"unit": path.name, "path": str(path), "device": info.st_dev, "inode": info.st_ino, "ctime_ns": info.st_ctime_ns, "mode": info.st_mode, "uid": info.st_uid, "target": "/dev/null"} + assert os.readlink(path) == "/dev/null" + + +@pytest.mark.parametrize("unit", UNITS) +def test_existing_mask_is_preserved_without_stopping_or_resetting(maintenance, unit: str) -> None: + inhibit, prepare, output, masks = maintenance + path = masks / unit + path.symlink_to("/dev/null") + before = path.lstat() + systemd = prepare() + with pytest.raises(ValueError, match="mask paths are not empty"): + inhibit(output) + assert path.lstat() == before and os.readlink(path) == "/dev/null" + assert systemd.calls == [] + assert json.loads(output.read_text())["status"] == "failed" + + +@pytest.mark.parametrize("unit", UNITS) +@pytest.mark.parametrize("fault", ("initial:ActiveState:failed", "initial:NeedDaemonReload:yes")) +def test_unavailable_initial_unit_refuses_before_any_preparation(maintenance, unit: str, fault: str) -> None: + inhibit, prepare, output, masks = maintenance + systemd = prepare(fault, unit) + with pytest.raises(ValueError, match="installed maintenance"): + inhibit(output) + assert systemd.calls == [] and not tuple(masks.iterdir()) + assert json.loads(output.read_text())["status"] == "failed" + + +@pytest.mark.parametrize("service", SERVICES) +@pytest.mark.parametrize("fault", ("missing", "stop-failed", "still-running")) +def test_missing_or_unstopped_service_never_installs_a_mask(maintenance, service: str, fault: str) -> None: + inhibit, prepare, output, masks = maintenance + systemd = prepare(fault, service) + with pytest.raises((ValueError, subprocess.CalledProcessError)): + inhibit(output) + assert not tuple(masks.iterdir()) + assert all(command[1] == "stop" for command in systemd.calls) + if fault == "missing": + assert systemd.calls == [] + evidence = json.loads(output.read_text()) + assert evidence["status"] == "failed" and evidence["failure_type"] + assert "observed" not in evidence + + +@pytest.mark.parametrize("pair", PAIRS, ids=lambda pair: pair[0].removesuffix(".timer")) +@pytest.mark.parametrize( + "fault", + ( + "property:LoadState:loaded", + "property:UnitFileState:static", + "property:ActiveState:active", + "property:SubState:running", + "property:MainPID:123", + "property:MainPID:missing", + "property:NeedDaemonReload:yes", + "regular-mask", + "wrong-mask", + "changed-mask", + ), +) +def test_guest_refuses_incomplete_or_changed_actual_inhibition(maintenance, pair: tuple[str, str], fault: str) -> None: + inhibit, prepare, output, masks = maintenance + timer, service = pair + systemd = prepare(fault, timer if fault == "changed-mask" else service) + with pytest.raises(ValueError): + inhibit(output) + evidence = json.loads(output.read_text()) + assert evidence["status"] == "failed" and evidence["failure_type"] + assert systemd.calls[-1] == ("systemctl", "daemon-reload") + assert len(systemd.calls) == len(UNITS) + 2 + assert (masks / timer).is_symlink() + assert all("reset-failed" not in command and "unmask" not in command for command in systemd.calls) diff --git a/tests/test_management_recovery_watcher_source.py b/tests/test_management_recovery_watcher_source.py new file mode 100644 index 000000000..3fa58c353 --- /dev/null +++ b/tests/test_management_recovery_watcher_source.py @@ -0,0 +1,244 @@ +"""Real-file component checks; native systemd acceptance remains a VM gate.""" + +from __future__ import annotations + +import hashlib +import os +import pwd +import sys +import tempfile +from dataclasses import asdict, dataclass, replace +from pathlib import Path + +import pytest + +from auto_coder.management import recovery_watcher_source as watcher +from auto_coder.management.containment_route_manifest import SystemdRouteFact +from auto_coder.management.recovery_containment_model import RecoveryContainmentUnavailable, RecoveryFile, RecoveryLock, RecoveryWatcher + +pytestmark = pytest.mark.usefixtures("_use_real_commands") + + +def _bound(path: Path) -> RecoveryFile: + info = path.stat() + return RecoveryFile(str(path), hashlib.sha256(path.read_bytes()).hexdigest(), info.st_dev, info.st_ino) + + +def _identity(path: Path) -> tuple[int, ...]: + info = path.lstat() + return info.st_dev, info.st_ino, info.st_mode, info.st_uid, info.st_gid, info.st_nlink, info.st_size, info.st_mtime_ns, info.st_ctime_ns + + +@dataclass +class UnitFixture: + root: Path + paths: tuple[Path, ...] + mask: Path + source: Path + selected: RecoveryWatcher + fact: SystemdRouteFact + content: bytes + commands: list[tuple[str, ...]] + + def observe(self): + return watcher.observe_recovery_watcher_unit(self.selected, self.fact, expected_uid=os.geteuid()) + + def select_source(self, content: bytes) -> None: + self.source.write_bytes(content) + self.selected = replace(self.selected, unit_source=_bound(self.source)) + + +@pytest.fixture +def unit_fixture(monkeypatch: pytest.MonkeyPatch): + with tempfile.TemporaryDirectory(prefix="ac-watcher-source-", dir=pwd.getpwuid(os.geteuid()).pw_dir) as directory: + root = Path(directory) + roots = tuple(root / name for name in ("etc-units", "run-units", "vendor-units")) + for path in roots: + path.mkdir(mode=0o700) + paths = (*roots, root / "absent-parent" / "absent-units") + config, module, lock = root / "watcher.json", root / "proxy_watch_cli.py", root / "proxy-watch.lock" + for path in (config, module, lock): + path.write_bytes(b"{}\n") + path.chmod(0o600) + unit = "ac-watch-proxy.service" + source, mask = roots[1] / unit, roots[0] / unit + command = sys.executable + " -I -B -m auto_coder.management.proxy_watch_cli --config " + str(config) + content = ("[Unit]\nDescription=Retained original watcher\n[Service]\nType=simple\nExecStart=" + command + "\nRestart=no\nStandardOutput=append:" + str(root / "watcher-output.log") + "\nStandardError=journal\n").encode() + source.write_bytes(content) + source.chmod(0o600) + mask.symlink_to("/dev/null") + info = lock.stat() + selected = RecoveryWatcher(unit, _bound(config), _bound(module), _bound(source), RecoveryLock(str(lock), info.st_dev, info.st_ino)) + fact = SystemdRouteFact(unit, "a" * 64, "masked", "inactive", 0, True, True) + commands = [] + + def command_read(*args, **kwargs): + commands.append(args) + assert args == ("systemctl", "show", "--property=UnitPath", "--value") + return " ".join(str(path) for path in paths) + "\n" + + monkeypatch.setattr(watcher, "_command", command_read) + yield UnitFixture(root, paths, mask, source, selected, fact, content, commands) + + +def test_masked_watcher_joins_real_retained_fragment_without_effective_commands(unit_fixture) -> None: + fixture = unit_fixture + assert fixture.fact.command_sources == () + source_before, mask_before = _identity(fixture.source), _identity(fixture.mask) + result = fixture.observe() + assert result.source == fixture.selected.unit_source + assert result.interpreter == sys.executable + assert result.load_paths == tuple(str(path) for path in fixture.paths) + assert result.mask_path == str(fixture.mask) + assert result.load_roots[-1][1:] == (0, 0, 0, 0, 0) + assert fixture.source.read_bytes() == fixture.content + assert _identity(fixture.source) == source_before and _identity(fixture.mask) == mask_before + assert len(fixture.commands) == 2 + assert RecoveryWatcher.parse(asdict(fixture.selected)) == fixture.selected + + +@pytest.mark.parametrize("field,value", (("inhibited", False), ("main_pid", 12), ("needs_daemon_reload", True), ("unit", "other.service"))) +def test_retained_source_cannot_replace_actual_technical_inhibition(unit_fixture, field, value) -> None: + unit_fixture.fact = replace(unit_fixture.fact, **{field: value}) + with pytest.raises(RecoveryContainmentUnavailable, match="technically inhibited"): + unit_fixture.observe() + assert not unit_fixture.commands + + +@pytest.mark.parametrize("change", ("missing-isolation", "missing-bytecode-flag", "module", "config", "extra-argument", "environment", "working-directory", "root-directory", "pre-command", "post-command", "duplicate-command", "install-alias", "section-repeat", "specifier", "control")) +def test_retained_unit_requires_the_literal_finite_command_profile(unit_fixture, change: str) -> None: + content = unit_fixture.content.decode() + if change == "missing-isolation": + content = content.replace(" -I -B ", " -B ") + elif change == "missing-bytecode-flag": + content = content.replace(" -I -B ", " -I ") + elif change == "module": + content = content.replace("auto_coder.management.proxy_watch_cli", "other.module") + elif change == "config": + content = content.replace(unit_fixture.selected.configuration.path, str(unit_fixture.root / "different.json")) + elif change == "extra-argument": + content = content.replace("\nRestart=", " --other\nRestart=") + elif change == "duplicate-command": + content += "ExecStart=/usr/bin/true\n" + elif change == "install-alias": + content += "[Install]\nAlias=other.service\n" + elif change == "section-repeat": + content += "[Service]\n" + elif change == "specifier": + content = content.replace("Description=Retained", "Description=%n Retained") + elif change == "control": + content = content.replace("Type=simple", "Type=simple\r") + else: + directive = {"environment": "Environment=PYTHONPATH=/other", "working-directory": "WorkingDirectory=/other", "root-directory": "RootDirectory=/other", "pre-command": "ExecStartPre=/usr/bin/true", "post-command": "ExecStartPost=/usr/bin/true"}[change] + content += directive + "\n" + unit_fixture.select_source(content.encode()) + with pytest.raises(RecoveryContainmentUnavailable): + unit_fixture.observe() + + +def test_same_python_binary_in_another_venv_is_not_the_same_interpreter_selection(unit_fixture) -> None: + other = unit_fixture.root / "other-venv/bin/python" + other.parent.mkdir(parents=True) + other.symlink_to(sys.executable) + assert other.samefile(sys.executable) + unit_fixture.select_source(unit_fixture.content.replace(sys.executable.encode(), str(other).encode())) + with pytest.raises(RecoveryContainmentUnavailable, match="interpreter/module/configuration"): + unit_fixture.observe() + + +@pytest.mark.parametrize("change", ("bytes", "inode", "writable", "hardlink")) +def test_retained_source_must_be_the_protected_original_object(unit_fixture, change: str) -> None: + fixture = unit_fixture + if change == "bytes": + fixture.source.write_bytes(fixture.content + b"# changed\n") + elif change == "inode": + fixture.source.rename(fixture.root / "retained-old-fragment") + fixture.source.write_bytes(fixture.content) + fixture.source.chmod(0o600) + elif change == "writable": + fixture.source.chmod(0o666) + else: + os.link(fixture.source, fixture.root / "other-link") + with pytest.raises(RecoveryContainmentUnavailable): + fixture.observe() + + +@pytest.mark.parametrize("name", ("ac-watch-proxy.service.d", "ac-watch-.service.d", "ac-.service.d", "service.d")) +def test_unit_dash_prefix_and_type_wide_drop_ins_refuse(unit_fixture, name: str) -> None: + (unit_fixture.paths[2] / name).mkdir() + with pytest.raises(RecoveryContainmentUnavailable, match="drop-ins"): + unit_fixture.observe() + + +@pytest.mark.parametrize("kind", ("competing-fragment", "extra-mask", "missing-mask", "source-above-mask", "source-outside-load-path")) +def test_real_unit_load_path_selection_cannot_be_ambiguous(unit_fixture, monkeypatch: pytest.MonkeyPatch, kind: str) -> None: + fixture = unit_fixture + if kind == "competing-fragment": + (fixture.paths[2] / fixture.selected.unit).write_bytes(fixture.content) + elif kind == "extra-mask": + (fixture.paths[2] / fixture.selected.unit).symlink_to("/dev/null") + elif kind == "missing-mask": + fixture.mask.unlink() + elif kind == "source-above-mask": + monkeypatch.setattr(watcher, "_command", lambda *args, **kwargs: " ".join(str(path) for path in (fixture.paths[1], fixture.paths[0], *fixture.paths[2:]))) + else: + monkeypatch.setattr(watcher, "_command", lambda *args, **kwargs: " ".join(str(path) for path in (fixture.paths[0], *fixture.paths[2:]))) + with pytest.raises(RecoveryContainmentUnavailable): + fixture.observe() + + +@pytest.mark.parametrize("kind", ("direct", "missing-vendor-target", "name-chain", "relative")) +def test_aliases_follow_systemd_unit_names_even_when_literal_target_is_missing(unit_fixture, kind: str) -> None: + fixture = unit_fixture + alias = fixture.paths[0] / "generic.service" + if kind == "direct": + alias.symlink_to(fixture.source) + elif kind == "missing-vendor-target": + target = fixture.paths[2] / fixture.selected.unit + assert not target.exists() + alias.symlink_to(target) + elif kind == "name-chain": + alias.symlink_to(fixture.paths[2] / "middle.service") + (fixture.paths[1] / "middle.service").symlink_to(fixture.paths[2] / fixture.selected.unit) + else: + (fixture.paths[1] / "generic.service").symlink_to(fixture.selected.unit) + with pytest.raises(RecoveryContainmentUnavailable, match="alias"): + fixture.observe() + + +@pytest.mark.parametrize("change", ("mask", "alias", "source")) +def test_retained_unit_readback_repeats_source_and_load_path_authority(unit_fixture, monkeypatch: pytest.MonkeyPatch, change: str) -> None: + fixture = unit_fixture + real_read = watcher.read_bound_file + calls = 0 + + def changing_read(*args, **kwargs): + nonlocal calls + value = real_read(*args, **kwargs) + calls += 1 + if calls == 1: + if change == "mask": + fixture.mask.rename(fixture.root / "old-mask") + fixture.mask.symlink_to("/dev/null") + elif change == "alias": + (fixture.paths[0] / "new-alias.service").symlink_to(fixture.paths[2] / fixture.selected.unit) + else: + fixture.source.write_bytes(fixture.content + b"# changed after first read\n") + return value + + monkeypatch.setattr(watcher, "read_bound_file", changing_read) + with pytest.raises(RecoveryContainmentUnavailable): + fixture.observe() + + +@pytest.mark.parametrize("change", ("missing-source", "wrong-name", "instance")) +def test_enrollment_requires_an_unambiguous_retained_watcher_fragment(unit_fixture, change: str) -> None: + raw = asdict(unit_fixture.selected) + if change == "missing-source": + raw.pop("unit_source") + elif change == "wrong-name": + raw["unit_source"]["path"] = str(unit_fixture.source.parent / "other.service") + else: + raw["unit"] = "ac-watch@one.service" + with pytest.raises(RecoveryContainmentUnavailable): + RecoveryWatcher.parse(raw) diff --git a/tests/test_management_worker_lifecycle.py b/tests/test_management_worker_lifecycle.py index fcbf16e59..f420889c8 100644 --- a/tests/test_management_worker_lifecycle.py +++ b/tests/test_management_worker_lifecycle.py @@ -8,6 +8,7 @@ import time from concurrent.futures import ThreadPoolExecutor from pathlib import Path +from types import SimpleNamespace import pytest from starlette.testclient import TestClient @@ -730,6 +731,97 @@ def observe_status(): assert status_entered.is_set() +@pytest.mark.parametrize("receipt_state", ("complete", "pending", "digest-conflict", "missing", "expired-pending")) +def test_supervisor_reconstruction_between_identity_checks_retains_original_effect(real_boundary, tmp_path: Path, monkeypatch, receipt_state: str) -> None: + """Control typed Supervisor observations around the real dispatcher and SQLite journal.""" + base, _base_grants, identity, signed, _uids, state = real_boundary + origin = base.oauth.canonical_issuer + oauth = OAuthSettings(base.oauth.database_path, origin, base.oauth.firebase_project_id, (OAuthClient("alice-client", (origin + "/alice-callback",), allowed_scopes=(READ_SCOPE, DEPLOY_SCOPE)),), enabled_scopes=(READ_SCOPE, DEPLOY_SCOPE)) + policy = tmp_path / "operator.yaml" + policy.write_text("profile: beta\ntargets:\n beta-worker:\n channel: beta\n repositories: [fixture/beta]\n", encoding="utf-8") + settings = ManagementSettings(tmp_path, oauth, port=base.port, operating_policy_path=policy, operation_state_root=tmp_path / "journal") + bridge = LifecycleBridge() + bridge.selected = "bootstrap" + assert bridge.worker is not None + bridge.worker["runtime_id"] = "bootstrap" + app = create_management_app(settings, OAuthGrantService(oauth, identity, clock=lambda: state["now"]), worker_clients={"beta": bridge}) # type: ignore[dict-item] + runtime = app.state.management_runtime + + async def no_background_monitor() -> None: + # Every lifecycle iteration is driven explicitly at the identity boundary. + pass + + monkeypatch.setattr(runtime, "start_monitor", no_background_monitor) + with TestClient(app, base_url=origin) as client: + bearer = _issue(client, settings, signed, "alice", "alice-client", READ_SCOPE + " " + DEPLOY_SCOPE)["access_token"] + arguments = {"target_id": "beta-worker", "idempotency_key": "owner-changes-between-observations", "expected_supervisor_generation": "supervisor-a", "expected_process_run_id": "run-old", "expected_deployment_generation": "generation-old", "expected_runtime_id": "bootstrap"} + admitted = _call(client, bearer, "auto_coder_stop_worker", arguments) + operation_id = str(admitted["operation_id"]) + effect_id = "d-" + operation_id + for _ in range(8): + _drive(app, operation_id, 1) + if effect_id in bridge.receipts: + break + else: + raise AssertionError("original drain was never dispatched") + original = runtime.journal.lifecycle_snapshot(operation_id) + original_record = runtime.journal.get(operation_id).record + assert original is not None and original[1]["stage"] == "DRAINING" + assert original[1]["steps"]["drain"]["state"] == "SENT_UNKNOWN" + original_receipt = dict(bridge.receipts[effect_id]) + actions = list(bridge.actions) + original_inspect = bridge.inspect_supervisor + inspections = 0 + + def reconstruct_between_checks() -> WorkerNetworkResult: + nonlocal inspections + inspections += 1 + observed = original_inspect() + if inspections == 1: + # The first check saw the old owner; the step sees its replacement. + bridge.supervisor_generation = "supervisor-reconstructed" + return observed + + monkeypatch.setattr(bridge, "inspect_supervisor", reconstruct_between_checks) + _drive(app, operation_id, 1) + assert inspections == 2 + assert runtime.journal.lifecycle_snapshot(operation_id) == original + assert runtime.journal.get(operation_id).record == original_record + assert bridge.actions == actions + monkeypatch.setattr(bridge, "inspect_supervisor", original_inspect) + + if receipt_state in {"pending", "expired-pending"}: + bridge.receipts[effect_id] = {**original_receipt, "status": "DRAINING"} + elif receipt_state == "digest-conflict": + bridge.receipts[effect_id] = {**original_receipt, "intent_digest": "0" * 64} + elif receipt_state == "missing": + monkeypatch.setattr(bridge, "read_worker_operation", lambda _operation: bridge._result("UNKNOWN", {})) + if receipt_state == "expired-pending": + monkeypatch.setattr("auto_coder.management.worker_lifecycle.time", SimpleNamespace(time=lambda: original[1]["deadline"] + 1)) + + _drive(app, operation_id, 8) + if receipt_state == "pending": + assert runtime.journal.lifecycle_snapshot(operation_id) == original + assert bridge.actions == actions + bridge.receipts[effect_id] = original_receipt + _drive(app, operation_id, 8) + result = _call(client, bearer, "auto_coder_operation_get", {"target_id": "beta-worker", "operation_id": operation_id}) + final = runtime.journal.lifecycle_snapshot(operation_id) + assert final is not None + assert final[1]["deadline"] == original[1]["deadline"] + assert final[1]["original_deadline"] == original[1]["original_deadline"] + assert all(final[1]["steps"]["drain"][field] == original[1]["steps"]["drain"][field] for field in ("operation_id", "sent_at", "owner_generation", "attempt_id", "correlation_identity", "expected_digest")) + assert bridge.actions == actions == [("REQUEST_GRACEFUL_STOP", effect_id)] + if receipt_state in {"complete", "pending"}: + assert result["status"] == "SUCCEEDED", result + assert final[1]["current_supervisor_generation"] == "supervisor-reconstructed" + else: + assert result["status"] == "NEEDS_ATTENTION", result + assert result["reason"] == "original_supervisor_incarnation_changed" + assert _call(client, bearer, "auto_coder_stop_worker", {**arguments, "idempotency_key": "competing-after-owner-change"})["status"] == "BUSY" + assert _call(client, bearer, "auto_coder_stop_worker", arguments)["operation_id"] == operation_id + + def test_full_policy_allows_release_process_restart_but_never_beta_source_update(real_boundary, tmp_path: Path) -> None: base, _base_grants, identity, signed, _uids, state = real_boundary origin = base.oauth.canonical_issuer diff --git a/tests/test_worktree_isolation.py b/tests/test_worktree_isolation.py index 577975fbc..6ea03dc0d 100644 --- a/tests/test_worktree_isolation.py +++ b/tests/test_worktree_isolation.py @@ -517,6 +517,28 @@ def test_tracked_file_permissions_survive_preparation_and_noop_handoff(tmp_path: assert stat.S_IMODE(tracked.stat().st_mode) == 0o600 +@pytest.mark.parametrize("is_noedit", [False, True]) +def test_metadata_only_change_preserves_exact_caller_index_during_preparation(tmp_path: Path, is_noedit: bool) -> None: + repo = _init_repo(tmp_path) + tracked = repo / "tracked.txt" + observed = tracked.stat() + # Force a real cached-stat mismatch without a wall-clock race or content edit. + os.utime(tracked, ns=(observed.st_atime_ns, observed.st_mtime_ns - 2_000_000_000)) + tracked.chmod(0o600) + index = repo / ".git" / "index" + original_index = index.read_bytes() + + with isolated_local_llm_worktree(repo, is_noedit=is_noedit) as workspace: + assert index.read_bytes() == original_index + private = Path(workspace) / "tracked.txt" + assert private.read_text() == "initial content\n" + assert stat.S_IMODE(private.stat().st_mode) == 0o600 + + assert index.read_bytes() == original_index + assert tracked.read_text() == "initial content\n" + assert stat.S_IMODE(tracked.stat().st_mode) == 0o600 + + def test_disposing_one_private_workspace_preserves_peer_workspace_and_caller_refs(tmp_path: Path) -> None: repo = _init_repo(tmp_path) initial_commit = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo, text=True).strip()