From 14d25c7a91655e7709036a2bf00e25e2e11f853c Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sat, 26 Sep 2026 09:30:32 -0700 Subject: [PATCH 01/51] docs(plans): add the Bazel-owned host integration lane plan --- ...actor-bazel-owned-host-integration-plan.md | 564 ++++++++++++++++++ 1 file changed, 564 insertions(+) create mode 100644 docs/plans/2026-09-26-0021-refactor-bazel-owned-host-integration-plan.md diff --git a/docs/plans/2026-09-26-0021-refactor-bazel-owned-host-integration-plan.md b/docs/plans/2026-09-26-0021-refactor-bazel-owned-host-integration-plan.md new file mode 100644 index 000000000..61ce199b3 --- /dev/null +++ b/docs/plans/2026-09-26-0021-refactor-bazel-owned-host-integration-plan.md @@ -0,0 +1,564 @@ +--- +title: Bazel-Owned Host Integration Lane - Plan +type: refactor +date: 2026-09-26 +deepened: 2026-09-26 +topic: bazel-owned-host-integration +artifact_contract: ce-unified-plan/v1 +product_contract_source: ce-brainstorm +execution: code +--- + +# Bazel-Owned Host Integration Lane - Plan + +## Goal Capsule + +- **Objective:** The d2b host-integration lane runs as Bazel tests, so a contributor gets check selection, results in the Bazel graph, and a faster local loop, and the repository stops maintaining two build systems for one test tier. +- **Means:** Bazel owns the lane end to end; nix is reduced to a hermetic build action that produces the guest image. +- **Product authority:** This Product Contract owns behavior and scope. `tests/AGENTS.md` owns the type-10 classification; root `AGENTS.md` owns the profile and changelog rules. `docs/plans/2026-08-24-001-refactor-bazel-backed-host-integration-binaries-plan.md` is superseded on its R3, which reserved VM orchestration to nix. The cutover amends the repository instruction sentences that R3 reverses — the host-lane binary-injection rule in root `AGENTS.md`, the type-10 tier rows and host-lane bundle handoff in `tests/AGENTS.md`, and the handoff descriptions in `docs/contributing/critical-subsystems.md`, `docs/contributing/gates-and-lints.md`, `docs/reference/compatibility.md`, and `docs/reference/support-matrix.md` — in the same change that removes the handoff. The heavy-gate rule those files also describe is not carried forward: the guard was deleted and R16 does not require it. +- **Product Contract preservation:** changed at planning time — R3, R5, R6, R7, R9, R16, R17, F1, AE2, AE10. The two user-directed changes are the single lane test target and dropping the heavy-gate semaphore; the rest are research corrections to a snapshot-capability premise, a missing host precondition, the disk layout those two assume, and two contradictions a document review found between requirements that planning-time evidence resolved. +- **Execution profile:** Deep, local-only, delivered as a port of one check at a time rather than a single cutover. Within the port the branch stays green because each unported check keeps executing its existing assertions. +- **Stop conditions:** Stop if the guest's attached writable devices prove not snapshot-capable, before the pool is built; if a restored run costs at least as much as a fresh boot for the same check; or, at the point in the cutover where the deferred check is retired, if no retained check is found to carry the host isolation from Gateway relay credentials it asserts. + +--- + +## Product Contract + +### Summary + +The type-10 VM lane moves out of nix orchestration and into Bazel: a small pool of guests is built as a hermetic Bazel action, booted once each, and replayed per check through snapshot and restore, with the checks' guest assertions moving to Rust as each one is ported. `make test-host-integration` remains the contributor's command and now invokes a single lane test target; the nix VM fixtures are removed as their checks port. + +### Problem Frame + +`make test-host-integration` is the last major surface where nix, not Bazel, schedules work. Every other test tier in the repository runs through the Bazel graph, and `docs/plans/2026-08-19-002-refactor-build-test-ownership-cleanup-plan.md` set the direction that Make targets, CI jobs, contributor docs, and tests expose Bazel only. The VM lane was carved out at the time because Bazel owned the binaries but not the run. + +The cost of the carve-out is structural rather than episodic. The lane has no Bazel target at all, so it has no selection, no JUnit or BEP history, and no shared cache with the rest of the suite; `Makefile:295` serializes the checks by default; and `flake.nix:610-612` smuggles the Bazel-built d2b binaries into the guest through `builtins.getEnv`, which means the guest closure is keyed on an environment variable rather than on declared inputs. Two build systems own one test tier, and the boundary between them is a shell script. + +### Requirements + +**Guest image construction** + +- R1. Each guest image must be produced by a Bazel build action whose inputs are declared labels, so the image is a graph output that caches and rebuilds on source change. +- R2. The Bazel-built d2b host binaries must reach the guest as declared inputs; the guest must never rebuild them through nix. +- R3. The `D2B_HOST_TOOL_BUNDLE` and `D2B_CH_CONTROLLER_BUNDLE` environment-variable handoff must be removed from the flake and the Makefile. + +**Lane execution** + +- R4. Bazel must own the guest lifecycle inside the lane test target: spawn, readiness wait, assertion run, restore between checks, and teardown. +- R5. A small pool of guests is booted once each, and every check runs against a snapshot-restored copy of the guest it is assigned to, where assignment matches the check's emulator invocation; a check that cannot be snapshot-restored runs on its own single-use guest instead, and a pool member that has run a nested guest is retired rather than restored. +- R6. Every writable block device a guest attaches must be materialized in a format that supports internal snapshots, and a device that is not must fail the lane rather than run without restore. +- R7. The lane must require `/dev/kvm`, nested virtualization, and nested-state save support on the host; a host missing any of them must stop with a clear message instead of falling back to emulation. +- R8. The pool's aggregate guest footprint must stay within a stated host budget covering memory, vCPU count, and the lane working directory, sized per guest shape with the pool size derived during Planning; checks assigned to one guest run sequentially while the pool itself runs concurrently. +- R9. A contributor must be able to run one named check by filtering the lane target, without booting the checks that were not selected. +- R10. The lane must reproduce each check's emulator invocation — its memory, vCPU count, disk size, the drive layout including the writable-store root drive, and any per-check device options such as the vsock device — rather than booting a single uniform guest shape. + +**Assertion layer** + +- R11. Each check's guest assertions must end in Rust, compiled and linted by the repository's Rust gates. +- R12. A check that has not yet been ported must keep executing its existing assertions and must keep gating the lane, so no check loses coverage during the transition. The lane owns that check's guest for the whole run, and must re-provide the legacy driver's guest-control surface so its assertions execute unchanged. +- R13. A failing check must report the diagnostics the current driver produces: the failing stage, the resource rows and unit journals that explain it, and the zone debug dump. + +**Cutover** + +- R14. At cutover the `vmChecks` flake output and the Makefile shell recipe must be removed, and the repository's own instruction and reference documents must be updated to describe the Bazel lane; each `runNixOSTest` fixture is removed when its own check is ported. +- R15. The deferred Gateway-isolation check must be removed rather than migrated, and this work does not replace its coverage. +- R16. `make test-host-integration` must remain the contributor's entry point, invoking the lane's single Bazel test target. + +**Contributor surface** + +- R17. Every selected check must surface as its own result in the lane's test output, carrying that check's diagnostics, not one aggregated pass or fail for the suite. +- R18. The guest binaries must be built under a repository-committed build profile; the lane must not depend on a caller-supplied profile override. + +### Key Decisions + +- **Bazel owns the lane end to end.** (session-settled: user-directed — chosen over wrapping the existing nix lane and over consuming a prebuilt guest from outside the lane: one build system, not two.) Governs R1, R4, R14. +- **Nix is reduced to a hermetic guest-image build action.** (session-settled: user-directed — chosen over fetching a published guest image: the guest has to track source changes inside the Bazel graph.) Governs R1, R2, R3. +- **The lane stays contributor-local.** (session-settled: user-directed — chosen over a required PR gate on BuildBuddy: the lane is a local pre-PR surface, which is also what makes the virtualization precondition assertable rather than negotiable.) Governs R7, R8, R18. +- **A small pool of guests is reused through snapshot and restore.** (session-settled: user-directed — chosen over a guest per check and over a single sequential guest: cut total boots from one-per-check to one-per-pool-member while keeping a parallel wave.) Governs R5, R6, R8. +- **Virtualization is a precondition, not a fallback.** (session-settled: user-directed — the lane runs on the contributor's own machine, so emulation is a silent degradation rather than a needed capability.) Governs R7. +- **Assertions port to Rust one check at a time.** (session-settled: user-directed — chosen over keeping the Python testScripts permanently and over porting all eleven in one cutover: the lane is Bazel-native from day one without a coverage cliff.) Governs R11, R12, R13. +- **The pool runs concurrently; one guest runs its checks sequentially.** (session-settled: user-directed — chosen over a single sequential guest: snapshot and restore mutates one guest, so parallelism has to come from the pool.) Governs R8. +- **A check that cannot be snapshot-restored gets a single-use guest.** (session-settled: user-directed — chosen over leaving it on the legacy lane and over deciding at a de-risking spike: its coverage is preserved and the migration never blocks on proving snapshot safety.) Governs R5. + +### How This Work Fits Together + + + +This plan covers the type-10 VM lane only. That split is the current understanding, not a committed roadmap. + +- Remote execution of the lane on BuildBuddy, an executor pool, or in CI as a required gate — *Deferred*: a later plan may take it up once the lane has a stable local run and the virtualization precondition is settled. +- The type-9 container lane and the live-host scripts — *Can proceed independently of* this plan; they share the Make facade but not the VM harness. +- Porting the checks to Rust — in scope for this plan and sequenced inside it: the boot layer lands first, each check then ports per F2, and cutover fires when the last check asserts in Rust. +- Bumping the pinned nixpkgs Bazel ruleset — *Can proceed independently of* this plan; this work stays inside the existing pin. + +### Key Flows + +- F1. A contributor runs the lane + - **Trigger:** A contributor invokes `make test-host-integration` on a host with hardware virtualization and nested-state support. + - **Actors:** The Make facade, the lane test target, the guest-image build action, the guest pool, the checks. + - **Steps:** The facade invokes the single lane test target; the guest-image action materializes the image from declared inputs; the target boots each pool member once, waits for activation, and snapshots it; each selected check restores its assigned guest, runs, and hands the guest back; a member that has run a nested guest is retired instead of restored; teardown releases the guests. + - **Outcome:** Every selected check reports through the lane's test output with its result and diagnostics. + - **Covers:** R1, R4, R5, R8, R10, R16, R17, R18. + +```mermaid +flowchart TB + A[make test-host-integration] --> B[lane test target
result never cached] + B --> C[guest-image build action
declared inputs, cacheable] + C --> D[boot pool member
KVM + nested state required] + D --> E[wait activation, snapshot] + E --> F[restore snapshot] + F --> G[run selected check] + G --> H{ran a nested guest?} + H -->|yes| I[retire this pool member] + H -->|no| J[hand guest to next check] + J --> F + I --> D + G --> K[per-check result
+ diagnostics] + K --> L[teardown] +``` + +- F2. A check is ported to Rust + - **Trigger:** A contributor picks a check whose assertions still execute in the legacy driver. + - **Actors:** The contributor, the check's Rust test target, the legacy fixture. + - **Steps:** The check's current assertions pass unchanged against a guest the lane hands it through the re-provided driver surface; the assertions move to Rust behind the same readiness and diagnostic contract; the legacy fixture for that check is deleted; the lane stays green at every step. + - **Outcome:** The check asserts in Rust with unchanged coverage, and one fewer legacy fixture remains. + - **Covers:** R11, R12, R13. + +- F3. The lane is cut over + - **Trigger:** All eleven checks assert in Rust. + - **Actors:** The contributor, the Make facade, the flake, the contributor documentation. + - **Steps:** The `vmChecks` output, the Makefile shell recipe, and the deferred check are removed; the instructions and contributor docs that describe the nix lane are updated to describe the Bazel lane. + - **Outcome:** One lane, one build system, and no nix VM orchestration left in the tree. + - **Covers:** R14, R15, R16. + +### Acceptance Examples + +- AE1. Contributor host without the virtualization capability + - **Covers:** R7. + - **Given:** a host missing `/dev/kvm`, or one whose virtualization lacks nested-state support. + - **When:** the contributor invokes `make test-host-integration`. + - **Then:** the lane stops with a message naming the missing capability, and no check runs under emulation and none against a guest that cannot be snapshotted. + +- AE2. Contributor runs one check + - **Covers:** R9, R17. + - **Given:** a contributor filters the lane target to a single check. + - **When:** the lane runs. + - **Then:** only that check's guest is booted, and its result and log appear as that check's own entry in the lane's test output. + +- AE3. A check fails mid-suite + - **Covers:** R13, R17. + - **Given:** a ported check fails an assertion after the guest is restored. + - **When:** the failure is reported. + - **Then:** the report carries the failing stage, the resource rows and unit journals that explain it, and the zone debug dump, under that check's own entry. + +- AE4. A guest attaches a non-snapshot-capable device + - **Covers:** R6. + - **Given:** a guest configuration attaches a writable device with no internal-snapshot support. + - **When:** the lane tries to snapshot that guest. + - **Then:** the lane fails with a snapshot-capability error rather than running the suite without restore. + +- AE5. A check not yet ported + - **Covers:** R12. + - **Given:** a check whose assertions still execute in the legacy driver. + - **When:** the lane runs. + - **Then:** that check's assertions execute against a lane-owned guest and gate the lane exactly as before the port. + +- AE6. Guest closure needs a d2b binary + - **Covers:** R2, R3. + - **Given:** a guest image is built for a source state where a d2b binary changed. + - **When:** the image is built. + - **Then:** the binary comes from the Bazel graph, the image rebuilds because that input changed, and nix compiles no replacement. + +- AE7. The nested cloud-hypervisor check + - **Covers:** R5. + - **Given:** a check that runs a nested guest, or one that cannot be snapshot-restored. + - **When:** the lane schedules it. + - **Then:** it runs on a guest booted fresh for that run and never restored, and that guest is retired rather than returned to the pool, while the reused pool covers the other checks. + +- AE8. Contributor supplies a build-profile override + - **Covers:** R18. + - **Given:** a contributor exports a Bazel profile override before invoking the lane. + - **When:** the lane builds the guest binaries. + - **Then:** the guest build uses the repository-committed profile and the override does not change the guest closure. + +- AE9. Restored run is no cheaper than a fresh boot + - **Covers:** R5, R8. + - **Given:** a pool member measured both from a fresh boot and from a restored snapshot. + - **When:** the restored measurement is at least as large as the fresh-boot measurement. + - **Then:** the pool is not extended past its first member until restore is shown to be cheaper. + +- AE10. A guest built from the current disk layout + - **Covers:** R5, R6. + - **Given:** a guest whose attached writable devices — the root drive and the shared state disk — are in a snapshot-capable configuration, and whose in-guest writable-store images are files inside that guest rather than attached devices. + - **When:** the lane boots that guest and restores it between checks. + - **Then:** the guest boots and restores, and the suite proceeds on the reused pool rather than the single-use tier. + +### Success Criteria + +- Before any port lands, the current lane's wall-clock is recorded on the reference host for the full eleven-check suite, for one named check from each of the two guest shapes, and at the current lane's supported check concurrency. +- The full eleven-check lane and a single named check each complete faster than their recorded baselines on the same host. +- A contributor can identify which check failed, and why, from the lane's test output alone. +- With all checks ported, a repo-wide `grep` for the removed environment variables, the `vmChecks` output, and the heavy-gate semaphore returns nothing, including in the repository's own instruction and reference documents. +- A restored run of a check costs measurably less than a fresh boot of that same check, measured per guest shape before the pool grows past its first member. + +### Scope Boundaries + +**Deferred for later** + +- Running the lane on BuildBuddy, an executor pool, or in CI as a required gate. The virtualization precondition is the reason this is deferred rather than merely out of scope: a remote runner has no nested virtualization to give the guest. +- Bringing the type-9 container lane and the live-host scripts into the Bazel graph. +- Rebuilding the repository's cross-lane heavy-gate guard, which this plan neither reinstates nor depends on. +- Bumping the pinned nixpkgs Bazel ruleset to a current version. + +**Outside this product's identity** + +- d2b's contract that NixOS configuration is the source of truth for Zones and their resources. The guest stays a NixOS system built by the module system; this change alters who runs the test, not how a guest is declared. + +**Cost of the decisions above** + +- R7 removes the emulation fallback that worked at roughly six times the boot cost, so contributor hosts without nested virtualization lose the type-10 tier entirely rather than running it slowly. This plan provides no replacement verification surface for those hosts. + +### Dependencies / Assumptions + +- The contributor's host provides `/dev/kvm` with nested virtualization and nested-state save support, which the nested cloud-hypervisor check requires of the guest and which snapshotting an outer guest requires. +- The guest-image action needs a nix build that realizes the system closure into a Bazel-declared output from label inputs. The repository's existing nix-inside-Bazel test harness does not provide this — it runs against the host store over a working-tree flake reference inside an uncacheable, unsandboxed test action — and no nix Bazel ruleset version provides a cacheable nix-build action, so the action is authored here. The substitute reachability the current recipe's cache preflight and closure upload provide must be carried into the action, and the flake must arrive as a declared input rather than a working-tree reference, before R1 can hold. +- The guest's attached writable devices are already in a snapshot-capable configuration: the root drive is qcow2 and the shared state disk is attached with a writable overlay. This is a property to verify, not a conversion to perform; the two writable-store ext4 images are files inside the guest, not attached devices, and a single raw attached device would fail the snapshot outright. + +### Outstanding Questions + +**Deferred to Planning** + +- Which checks share a pool member, and whether that grouping is declared in the repository or derived from each check's declared weight. A grouping that drifts as checks are added is a maintenance surface of its own. +- How the pool size is chosen and whether it is contributor-tunable. +- Whether the host memory budget that bounds R8 uses the repository's existing per-lane memory ceiling shape or a lane-specific one, and whether the budget is contributor-tunable. The existing ceiling meters a single process tree, which cannot cover guests the lane spawns as its own children. + +### Sources / Research + +- `tests/AGENTS.md:12-15, 23, 68` — the type-10 classification and the "push coverage down toward type 1" rule. +- Root `AGENTS.md:209-215, 229-230` — the no-profile-override rule that R18 answers, and the existing requirement that the guest consume Bazel-built binaries. +- `Makefile:8-10, 11-22, 158-159, 178-361` — the one-class-per-target dispatcher invariant, the lane's membership in the local-target class, the canned Bazel alias, and the serial default at `:295`. +- `flake.nix:606-681` — the `vmChecks` output, the non-recursive fixture discovery, and the two `builtins.getEnv` bundle reads at `:610-612`. +- `tests/host-integration/lib.nix:519-641, 653-796` — the shared node configuration and the diagnostics prelude the Rust assertion layer must reproduce, and the split between reusable configuration and driver-coupled diagnostics. +- `tests/host-integration/lib.nix:529-536, 627-630` — the shared state disk, attached with a writable overlay. +- `runtime-cloud-hypervisor-guest-preflight.nix:637-644` — the in-guest virtualization and vhost-net assertions behind the single-use tier. +- `bazel/checks/fixtures/defs.bzl:1-63` — the one existing Starlark rule that runs nix as a cacheable build action; the model for the guest-image action. +- `bazel/checks/nix/defs.bzl:3-9, 81-125` — the existing nix-inside-Bazel test harness, whose tags establish the non-cacheable convention the lane follows. +- `tests/unit/meta/rust-main-packages-suite-guard.sh:150-173` — the guard that force-registers any crate carrying a test aggregate and forbids positive tags on such aggregates; the reason the lane's targets live outside the main package suite. +- `nixos-modules/base.nix:67-68` and `nixos-modules/lib.nix:322, 421-429` — sshd enabled by default in the guest base, the guest's ssh capability, and the repository's existing QMP readiness vocabulary. +- `CHANGELOG.md:1086-1088` and commit `2c2f8149b` — the deletion of the heavy-gate orchestration, which R16 and four documentation sites previously described as current. +- `docs/plans/2026-08-24-001-refactor-bazel-backed-host-integration-binaries-plan.md` — the superseded plan; its R3 and its "Bazel does not become the scheduler for the NixOS VM test" boundary are what this contract reverses. +- `docs/plans/2026-08-19-002-refactor-build-test-ownership-cleanup-plan.md:266, 269-270` — the direction that tests expose Bazel only and receive binaries from Bazel rather than building at test runtime. +- Planning research dossiers, kept at `/tmp/compound-engineering-1000/ce-plan-research/d57f3743/`: repository patterns, QEMU and nix best practices, framework documentation, and flow analysis. +- Emulator snapshot semantics from the research dossier: internal snapshots are supported only by the qcow2 format, a single writable non-snapshot-capable device fails the whole snapshot, and restoring a guest that has a live nested guest is documented undefined behavior on one vendor while working on another — which is why a member that has run a nested guest is retired rather than restored. +- Bazel execution model from the research dossier: a test action reaches a resource created outside it only by opting out of the sandbox or by an explicit mount pair, a sandboxed test's only writable surface is its own temporary directory, a test result defaults to replaying a cached verdict, and the current Bazel release no longer exposes the host temporary directory to sandboxed actions. +- Measured on the contributor's host: a hardware-virtualized boot reaches a running d2b daemon in 13.6s, against 84.0s under emulation, a 6.2x difference on the same guest image. This is the basis for treating virtualization as a precondition. It does not price the refactor, and it does not describe the whole suite: it is a single boot of the default guest shape, while the two writable-store checks replace the root drive with a bootable one and the repository's own comment says that path adds many minutes to startup and can hang. Both the speed criterion and the restore stop condition are therefore measured per guest shape, and the writable-store shape's cold-boot cost is recorded before the pool is sized. + +--- + +## Planning Contract + +### Key Technical Decisions + +- KTD1. The heavy-gate semaphore is not reinstated. (session-settled: user-directed — chosen over rebuilding the guard: the repository deleted it deliberately and six documentation sites still describe it as current, so correcting the contract is cheaper than reviving dead infrastructure.) Lane-local teardown and the lane's own stop conditions cover the self-race the guard used to prevent, and the two normative sites that record a `RETAIN` disposition for the semaphore namespace are a different edit class from a prose refresh and need a named owner in U8. Governs R16. +- KTD2. One lane-level test target owns the whole pool lifecycle, and the make target stays in the local class with a one-line recipe that names the repository-committed build profile itself rather than inheriting whatever profile a caller exported, the way the generate target already pins its own. (session-settled: user-directed — chosen over per-check targets with a facade that boots the pool first: the Make dispatcher expands a target to exactly one canned Bazel call under a one-class-per-target invariant, so a two-command facade would break that convention.) Governs R4, R9, R17, R18. +- KTD3. The guest image is built by a rule authored in this repository, modeled on the existing fixture rule, and the emulator is taken from the repository's own pinned nix package set rather than a new third-party Bazel ruleset. (session-settled: user-approved — no nix Bazel ruleset provides a cacheable nix-build action at the pinned or the current version, and the rules that do exist keep realization in the repository-fetch phase; taking the emulator from the same pinned set as the guest avoids adding an external module and keeps emulator and guest at one nixpkgs revision.) Governs R1, R2. +- KTD4. The guest-image action and the lane target both run unsandboxed and local, and hermeticity comes from nix's own configuration rather than Bazel's isolation. (session-settled: user-approved — nix cannot build inside a sandboxed action because its own sandbox requires root, and the fallback degrades silently rather than failing.) Governs R1, R7. +- KTD5. Guests are snapshotted and restored in-process, and a pool member that has run a nested guest is retired rather than restored. The rejected path is live migration: it rolls back memory and device state but not block content, and block migration was removed from the current emulator, so it cannot deliver the rollback the pool needs. It stays the fallback if restore turns out to cost more than a fresh boot. Governs R5. +- KTD6. The lane target's result is never cacheable, and the lane never runs with streamed test output, which would serialize it. Governs R9, R17. +- KTD7. Per-check guest configuration moves out of the runNixOSTest fixtures into a module the lane evaluates, before any fixture is deleted. Governs R10, R14. +- KTD8. The pool's bound covers the aggregate guest footprint across memory, vCPU count, and the lane working directory, read from the re-homed node module's declared guest fields rather than restated as a lane constant. The repository's existing per-lane memory meter cannot decide admission: it samples resident pages, so it under-reports an idle guest's reservation, and it wraps a single process tree where the guests are the lane's own children. Pool size is derived at lane start from that budget against the host's available memory, not chosen by hand; the budget is declared next to the per-check guest configuration rather than supplied as a contributor override, so adding a check cannot silently change what the pool can hold. Checks are assigned to members by matching emulator invocation, which is what fixes a member's device and block configuration for its whole life, so the pool is sized from the number of distinct invocations rather than from a guest-shape count. Governs R5, R8, R10. +- KTD9. The nixpkgs Bazel ruleset stays pinned at its current version; the bump is separate work. Governs R1. + +### High-Level Technical Design + +The lane has three cooperating pieces and a lifecycle that no existing rule in this repository covers. + +```mermaid +flowchart LR + subgraph graph["Bazel graph"] + IMG["guest image rule
cacheable, declared inputs"] + LANE["lane test target
never cached, local"] + end + subgraph tools["runfiles of the lane action"] + QEMU["emulator binary
+ runtime data"] + HARNESS["Rust harness
pool, checks, diagnostics"] + CFG["per-check guest config
evaluated by nix"] + end + IMG --> LANE + QEMU --> HARNESS + CFG --> HARNESS + HARNESS --> POOL["guest pool"] + POOL --> GUEST1["guest member A"] + POOL --> GUEST2["guest member B"] + GUEST1 --> CHECKS["selected checks
one at a time per guest"] + GUEST2 --> CHECKS + CHECKS --> JUNIT["one JUnit document
a testcase per check"] +``` + +The pool lifecycle is the part with the most failure surface, because restore is not a fresh boot: + +```mermaid +stateDiagram-v2 + [*] --> Booting + Booting --> Activating: emulator up, activation complete + Activating --> Ready: snapshot taken + Ready --> Restoring: next check assigned + Restoring --> Running + Running --> Ready: check clean, guest reusable + Running --> Retiring: check ran a nested guest + Running --> Retiring: check cannot be snapshot-restored + Ready --> Retiring: lane finishing + Retiring --> [*] +``` + +Two invariants hold across every transition. No guest is ever snapshotted or restored while a nested guest is alive inside it. And the snapshot is taken after activation completes and before any check runs, so a restored guest is always a guest that has never been touched by a check. + +### Implementation Constraints + +- The lane's targets live outside the main Rust package suite. The repository's test census force-registers any crate carrying a test aggregate into the main package suite and rejects positive tags on such aggregates, so the harness crate carries no test aggregate and the lane's test targets are registered directly by the lane's own build file. +- A snapshot-capable guest needs every writable attached device to support internal snapshots. Today the root drive is qcow2 and the shared state disk is attached with a writable overlay; a single raw attached device fails the snapshot for the whole guest, so the lane verifies this at boot rather than converting formats. +- The guest image build must not depend on a working-tree flake reference, because that would key a "cacheable" output on mutable state. The flake and its lock arrive as declared inputs, and the cache the current recipe configures is reached through nix's substituter configuration inside the action. +- The lane's own process holds the guests, so a lane-scoped working directory must outlive the individual check runs and cannot rely on a sandboxed temporary directory, which the current Bazel release no longer exposes to sandboxed actions. +- Check selection is a filter on the lane target rather than a target selection, so the existing check-name environment variables become filter inputs rather than Bazel label selection. + +### Sequencing + +The guest-image rule and the guest-configuration re-homing come first because the harness cannot spawn anything without them. The legacy driver guest-control surface lands before the lane test target is registered, because at that boundary the make target would otherwise point at a lane no check can run — a coverage hole R12 forbids. Only then do the pool and reporting layer, then the ports. + +### Alternative Approaches Considered + +- **Wrap the existing nix lane in a Bazel test target.** Keeps the nix driver and its Python assertions, and delivers selection, JUnit, and a shared cache cheaply. Rejected by the brainstorm: it leaves the repository maintaining two orchestrators for one tier, and the nix driver's own store, privilege, and device model does not survive as a non-interactive test action. +- **Adopt a QEMU Bazel ruleset for the guest launcher.** The available ruleset ships a hermetic emulator binary and toolchain providers but no VM-launching rule, and lists one on its roadmap; a second, unrelated ruleset requires a host-installed emulator, which defeats the point. Adopting either still leaves the launcher to be written, so the ruleset reduces to an emulator-binary dependency. +- **Use live migration instead of snapshot and restore for guest reset.** Migration rolls back memory and device state but not block content, and block migration was removed from the current emulator, so it cannot deliver the rollback the pool needs. Recorded as the fallback if restore cost measurement fails. +- **Rebuild the cross-lane heavy-gate guard.** Rejected: the repository removed it deliberately, and the plan's own stop conditions and lane-local teardown cover the new self-race risk without it. + +### Risks & Dependencies + +- Sourcing the emulator from the pinned nix set trades an external dependency for a version coupling: a nixpkgs bump changes the emulator under the lane, so guest image and emulator move together and a snapshot taken by one version is never assumed restorable by another. +- Snapshot compatibility is emulator-version-sensitive. A pool member booted under one emulator version is not assumed restorable under another, so the lane's emulator version is part of the guest identity. +- The guest-image action depends on a reachable substituter for the guest closure, the same reachability the current recipe guarantees through a cache preflight and a closure upload. If the action cannot reach it, the image build fails where the old recipe would have degraded. +- The lane's correctness rests on no guest being restored while a nested guest is alive. That is a runtime invariant the pool must enforce, and the only check that can violate it is the one that creates nested guests. + +### Documentation Plan + +- The repository's own instruction and reference documents are updated in the same change that removes the environment-variable handoff, since the grep gate that proves the removal spans them. +- The heavy-gate semaphore stops being described as current at every site that still says it is, per KTD1, and the deletion is recorded rather than left to be rediscovered. +- Contributor documentation describes the check-name variables as filter inputs on the lane target, so the variables contributors already use keep working, alongside the virtualization precondition. +- Every unit ships a changelog fragment; the changelog gate makes one mandatory for any change touching a non-prose path, so it applies to U1 through U8 rather than only to the cutover units. + +### System-Wide Impact + +Three boundaries meet at this lane, each owned by a different authority, and the change crosses all three. + +- **The make dispatcher.** Every public target is classified into exactly one environment class, and a local-class target's recipe expands to one canned Bazel call. The lane keeps its class and its recipe collapses to the single target, which is the same shape the existing performance target uses; a reader cannot infer this from the target's name alone, so KTD2 states the class. +- **The Bazel suite graph and its test census.** Layer-1 is composed from an explicit list, so staying out of it is by omission. The main-package suite is a separate fixed inventory read by a guard that force-registers any crate carrying a test aggregate, which is why the harness holds none. A new crate is not free, though: it must be a workspace member, carry a committed-scope row, satisfy the blocking census, and appear in the packages filegroup or it is not built, not linted, and silently invisible to the guard that would otherwise notice. +- **The flake's public surface.** The lane is the only consumer of the VM output being removed, but that output is part of what the flake publishes, so its removal is observable to anything else evaluating the flake. The guest declaration itself is unchanged and stays in the flake. +- **The contributor documentation surface.** Six files describe the removed handoff or the deleted guard, two of them normative records carrying a disposition rather than prose. Those are a different edit class from a documentation refresh and need a named owner rather than a sweep. + +### Sources & Research + +Planning research dossiers are kept at `/tmp/compound-engineering-1000/ce-plan-research/d57f3743/` — repository patterns, emulator and nix best practices, framework documentation, and flow analysis. The grounding dossier from the brainstorm phase is at `/tmp/compound-engineering-1000/ce-brainstorm/20260925-hostint-bazel/grounding.md`. The Sources section under the Product Contract carries the per-claim citations both phases rely on. + +--- + +## Implementation Units + +### U1. Build the guest image as a declared-input Bazel action + +- **Goal:** A rule that evaluates the guest's NixOS configuration from declared label inputs and emits the guest artifacts as a cacheable graph output, replacing the environment-variable handoff. +- **Requirements:** R1, R2, R3. +- **Dependencies:** none. +- **Files:** `bazel/checks/vm/defs.bzl` (new), `bazel/checks/vm/BUILD.bazel` (new), `nix/test-support/guest-image.nix` (new), `nix/test-support/bazel-host-tools.nix` (modify), `flake.nix` (modify — add the guest evaluation as a declared entry point, keeping the two environment reads until U4), `Makefile` (modify — move the substituter preflight into the action's inputs), `.bazelrc` (add the committed guest-build profile), `MODULE.bazel` (modify — add the emulator as an entry on the existing nix package extension), `MODULE.bazel.lock` (modify — the repository's lockfile mode errors rather than regenerating), `changelog.d/` (add). +- **Approach:** + 1. Model the rule on the repository's one existing cacheable nix action rather than on the nix test harness, which is a test wrapper with a different shape. + 2. Declare the flake and its lock as label inputs so the action's key reflects the source, not a working-tree reference. + 3. Configure the nix store, evaluation store, build-users setting, and substituters from inside the action so hermeticity does not depend on the developer's shell. + 4. Feed the d2b host binaries in as a declared label set, and keep the legacy environment-variable handoff in place until U4 collapses the recipe. Removing it here would leave the still-current nix fixtures falling back to nix-built tools, which R2 forbids, and would make the recorded baseline a different guest from the one the lane finally runs. R3 completes at U4, not here. + 5. Do not carry the recipe's closure upload into this action: a network side effect would make it uncacheable, contradicting R1. The upload retires with the recipe and its replacement is the build cache the image now lands in; the retirement is recorded in the changelog rather than left silent. + 5. Register the committed profile in the shared bazelrc so no caller supplies one. +- **Execution note:** Add a characterization check first that the action's output matches what the current recipe realizes for the same source state, before optimizing anything about it. +- **Patterns to follow:** `bazel/checks/fixtures/defs.bzl:1-63` for the action shape; `nix/test-support/bazel-host-tools.nix` for the bundle inventory and the hard failure on an incomplete handoff. +- **Test scenarios:** + - A guest image builds for an unmodified source state and is byte-comparable to what the current recipe realizes. + - A change to a d2b host binary invalidates the image and the rebuilt image contains the new binary; nix compiles no replacement. + - A change to a guest module invalidates the image and the rebuilt image reflects the change. + - An incomplete binary set fails the action rather than producing an image with a missing tool. + - Building with a developer's shell pointing at a different store still produces the same image. + - The action fails when the substituter is unreachable, rather than silently producing an image from a partial closure. +- **Verification:** The image is produced by `bazel build` as a graph output, the two environment variables no longer appear in the flake, and a second build of an unchanged tree reuses the cached output. + +### U2. Re-home per-check guest configuration + +- **Goal:** Move the reusable NixOS node configuration out of the runNixOSTest fixtures into a module the lane evaluates, so a fixture can be deleted when its check ports without taking its guest declaration with it. +- **Requirements:** R10, R14. +- **Dependencies:** U1. +- **Files:** `tests/host-integration/lib.nix` (modify — split), `nix/test-support/host-integration-node.nix` (new), one `tests/host-integration/*.nix` file per check (modify), `changelog.d/` (add). +- **Approach:** + 1. Separate the shared node configuration and the per-check module contributions from the driver-coupled diagnostics prelude, keeping each side intact. + 2. Give the re-homed module a stable interface the lane's harness can evaluate per check, independent of any test driver. + 3. Leave every fixture's assertion body untouched so the lane stays green through this unit. +- **Test expectation:** none — a pure relocation that adds no test target. The unit's evidence is the existing lane staying green and the U1 image build covering the re-homed module; the scenarios below are what an implementer checks by hand, not gated coverage. +- **Patterns to follow:** the configuration half of `tests/host-integration/lib.nix:519-641` as the source of truth for what moves. +- **Test scenarios:** + - The re-homed module evaluates to the same guest configuration as the fixture's inline configuration for every check. + - A per-check module contribution applied through the re-homed path produces the same guest as it does through the fixture. + - No fixture file loses an assertion while the split lands. +- **Verification:** Every check's guest evaluates identically before and after the move, and the fixtures still run through the current lane. + +### U3. Author the guest-spawning harness + +- **Goal:** A harness that reproduces each check's emulator invocation, boots its guest, waits for activation, and tears it down — the half of the lane that replaces the nix driver's boot responsibility. +- **Requirements:** R4, R6, R7, R10. +- **Dependencies:** U1, U2. +- **Files:** `packages/d2b-vm-harness/` (new crate — pool-free first pass), `Cargo.toml` (modify — workspace member; without it the crate has no generated dependency defs and cannot be built at all), `Cargo.lock` (modify — both clippy gates run locked), `BUILD.bazel` (modify — the packages filegroup enumerates every crate), `packages/xtask/src/provider_crate_policy.rs` (modify — committed-scope row, or the crate-layout gate fails), `packages/xtask/data/blocking-census-baseline.json` (modify, or the established allow-attribute convention — a harness that spawns processes and sleeps trips the blocking census), `bazel/checks/vm/BUILD.bazel` (modify — including the lane suite naming the harness's clippy targets, so the crate is linted by the unit that creates it), `MODULE.bazel` (modify if the dependency set grows), `MODULE.bazel.lock` (modify, if that manifest changes), `changelog.d/` (add). +- **Approach:** + 1. Consume the emulator binary and its runtime data from the pinned nix package set as runfile labels, the shape the existing nix-inside-Bazel harness already uses for the nix binary, so the emulator is at the same nixpkgs revision the guest image is realized from; select the accelerator explicitly instead of relying on a default that falls back to emulation. + 2. Reproduce the per-check invocation shape — memory, vCPU count, disk size, drive layout, and per-check device options — from the re-homed configuration rather than from a single uniform guest. + 3. Wait for activation through a readiness signal the repository already has vocabulary for, and treat a guest that never activates as a lane failure rather than a hang. + 4. Assert the host's virtualization capabilities up front, including nested-state save support, and fail with a message naming what is missing. + 5. Verify at boot that every attached writable device supports internal snapshots, so a non-snapshot-capable device fails the lane before any check runs. + 6. Hold no test aggregate on the crate, which is what keeps the repository's test census from force-registering it into the main package suite. The suite that would normally carry it is the lane's own, which names the harness's clippy targets directly, so the harness is still linted by the repository's Rust gates without an aggregate. + 7. Give every reusable-pool guest a machine-identity device and a hardware random source, and take the snapshot only once the guest reports its random pool is initialised — a guest restored before that point comes back with a different identity and a colder random pool than the checks were written against. +- **Execution note:** Prove the readiness and teardown paths against a real guest before adding snapshot support, so a boot failure is never confused with a restore failure. +- **Patterns to follow:** the readiness vocabulary in `nixos-modules/lib.nix:421-429`; the accelerator and device handling already written in the qemu-media provider's process builder. +- **Test scenarios:** + - A guest with the default invocation boots, activates, and tears down cleanly. + - A check whose configuration asks for a different memory, vCPU count, and disk size gets exactly that invocation. + - A check whose configuration attaches an extra device gets that device and the guest still boots. + - A host missing nested-state save support stops the lane with a message naming it, before any guest boots. + - A guest configured with a non-snapshot-capable writable device fails the lane at boot with a snapshot-capability error. + - A guest that never reaches activation fails the lane within a bounded time rather than hanging. + - Repeated boot and teardown cycles leave no emulator process or working directory behind. +- **Verification:** Every check's guest boots under the lane's own harness with the invocation its configuration declares, and the host preconditions are enforced before any guest starts. + +### U4. Add snapshot and restore pool management and the lane test target + +- **Goal:** The lane test target that boots the pool once, restores per selected check, retires members that cannot be reused, and reports per-check results — the unit that makes the lane a Bazel test. +- **Requirements:** R4, R5, R8, R9, R16, R17. +- **Dependencies:** U3, U5. +- **Files:** `packages/d2b-vm-harness/src/pool.rs` (new), `packages/d2b-vm-harness/src/report.rs` (new), `bazel/checks/vm/BUILD.bazel` (modify — add the pool target to the lane suite), `bazel/checks/BUILD.bazel` (modify — register the lane suite), `Makefile` (modify — collapse the shell recipe to the single target, keeping the target in the local class, pinning the committed build profile in the recipe, and keeping the non-x86_64 skip as a guard on the lane target), `tests/AGENTS.md` (modify — the type-10 tier row only), `changelog.d/` (add). +- **Approach:** + 1. Register one lane test target that owns the pool for its whole run, and make the make target a thin invocation of it. + 2. Take the snapshot after activation completes and before any check runs, so a restored guest is always one no check has touched. + 3. Serialize checks within a guest and run guests concurrently, sizing the pool against a host memory budget. + 4. Retire rather than restore any member that has run a nested guest or that is otherwise not reusable. + 5. Mark the result uncacheable and keep streamed test output off the lane, either of which would otherwise defeat the lane. + 6. Emit one JUnit document with a testcase per selected check, carrying that check's diagnostics. + 7. Take check selection as a filter on the target and honor the existing single-check selection variables as filter inputs. + 8. Hold a lane working directory that outlives individual check runs and does not depend on a sandboxed temporary directory. +- **Execution note:** Measure restored-run wall-clock against fresh-boot wall-clock for one check before growing the pool past a single member, and stop if restore is not cheaper. +- **Patterns to follow:** the non-cacheable and local-only tag convention in `bazel/checks/nix/defs.bzl:3-9`; the thin make alias shape used by the existing local targets. +- **Test scenarios:** + - The full lane runs every selected check and reports one result per check. + - Filtering to a single check boots only that check's guest. + - Two checks sharing a guest run in sequence on that guest and the second sees the first's guest state as handed back, not as a fresh boot. + - A check that runs a nested guest causes its guest to be retired, and the remaining checks still complete. + - A check that fails reports its stage, resource rows, unit journals, and zone debug dump under its own result entry, and the lane continues to the remaining checks. + - A second identical lane invocation re-runs every check rather than replaying a cached verdict. + - Streaming the lane's test output is refused or does not serialize the guests. + - The lane exceeds its memory budget on a small host by reducing the pool, not by failing. + - Teardown leaves no guest process or working directory behind after a mid-suite failure. +- **Verification:** `make test-host-integration` runs the lane through Bazel, every selected check reports individually, and the pool reduces wall-clock against the recorded baselines. + +### U5. Provide the legacy driver guest-control surface + +- **Goal:** Re-provide the guest-control helpers and the diagnostics prelude the unported checks call, so every check keeps gating the lane unchanged until its own port. +- **Requirements:** R12, R13. +- **Dependencies:** U3. +- **Files:** `packages/d2b-vm-harness/src/legacy.rs` (new), `tests/host-integration/lib.nix` (modify — extract the diagnostics prelude so both surfaces share it), `tests/host-integration/*.nix` (modify — point at the shared prelude), `changelog.d/` (add). +- **Approach:** + 1. Implement the full set of guest-control helpers the fixtures call — command execution with a bounded timeout, service-state waiting, file waiting, retrying command success, and explicit success and failure assertions — against the lane's own guest. + 2. Port the diagnostics prelude to the same surface so a failing unported check reports the same stage, rows, journals, and zone debug as today. + 3. Keep the guest's ssh capability and the fixtures' use of it unchanged, so assertion bodies need no edits. + 4. Keep the prelude in one place so the ported Rust assertions and the legacy surface report identically. +- **Test expectation:** none as a new behavior surface. The existing fixtures are the coverage; the unit adds no test target of its own, and its proof is that the lane is green with the new surface before any port begins. +- **Patterns to follow:** the helper set and diagnostics in `tests/host-integration/lib.nix:653-796`, which is the specification for this unit. +- **Test scenarios:** + - Every unported check runs unchanged against the lane's guest and gates the lane. + - A deliberately failing unported check reports the same diagnostics the current driver produces for the same failure. + - A command that never succeeds times out within its declared bound and reports the last observed output. + - A service that never reaches its expected state reports the unit status and journal. + - A file that never appears reports after its declared bound rather than hanging. +- **Verification:** The full lane is green with zero checks ported, and a deliberately broken unported check fails with diagnostics indistinguishable from the current driver. + +### U6. Port the first check's assertions to Rust + +- **Goal:** Convert one check's guest assertions to Rust behind the lane's own test target, establishing the pattern the remaining ports follow. +- **Requirements:** R11, R12, R13, R17. +- **Dependencies:** U4, U5. +- **Files:** `packages/d2b-vm-harness/tests/daemon_smoke.rs` (new), `tests/host-integration/daemon-smoke.nix` (delete), `bazel/checks/vm/BUILD.bazel` (modify), `changelog.d/` (add). +- **Approach:** + 1. Start with the daemon smoke check: the narrowest assertion surface, no nested guest, and the one the repository already treats as the archetype for this tier, so the pattern is proved on the easy case and U7's loop has a stable first pick. + 2. Reuse the lane's guest-control primitives rather than reimplementing them, and assert the same conditions the fixture asserted. + 3. Emit the same diagnostics on failure, through the same reporting path the legacy surface uses. + 4. Delete the fixture in the same change, so the port and its retirement land together. +- **Execution note:** Diff the ported check's assertions against the fixture it replaces before deleting the fixture, so a dropped assertion is caught rather than inherited. +- **Test scenarios:** + - The ported check passes against a healthy guest and reports under its own result entry. + - Each assertion the fixture made is present in the port; a missing one fails the port's own review check. + - A deliberately broken guest condition makes the ported check fail with the same diagnostics the fixture produced. + - The lane is green with one Rust check and the rest legacy. +- **Verification:** One check asserts in Rust, its fixture is gone, the lane is green, and the remaining fixtures still gate it. + +### U7. Port the remaining checks and cut the lane over + +- **Goal:** Port the remaining checks one at a time and complete the cutover once the last one asserts in Rust. +- **Requirements:** R11, R12, R13, R14, R15, R16, R17. +- **Dependencies:** U6. +- **Files:** `packages/d2b-vm-harness/tests/*.rs` (new, one per remaining check), `tests/host-integration/*.nix` (delete as each check ports), `tests/host-integration/deferred/host-zone-gateway-isolation.nix` (delete), `flake.nix` (modify — remove the `vmChecks` output), `changelog.d/` (add). The make target's recipe was already collapsed to the single target in U4, so the cutover here is the flake output and the fixtures, not the recipe. +- **Approach:** + 1. Port one check per change, retiring its fixture in the same change, keeping the lane green throughout. + 2. Port the nested guest check last, and give it a single-use guest that is retired after the run. + 3. Review the retained checks for one that carries the host isolation from Gateway relay credentials the deferred check asserts. If one is found the removal proceeds and that check keeps the coverage; if none is, stop rather than ship a silent loss, and record the decision either way. + 4. Remove the `vmChecks` output and the recipe's nix orchestration once no fixture depends on them. + 5. Update the instruction and reference documents that describe the environment-variable handoff in the same change. +- **Test scenarios:** + - Each ported check passes against its guest and reports under its own result entry. + - The lane is green at every intermediate state, with any mix of ported and legacy checks. + - The nested guest check runs on a single-use guest, and its guest is retired rather than returned to the pool. + - The full eleven-check lane passes with no legacy fixture remaining. + - No nix VM orchestration remains reachable from the make target. + - The deferred Gateway-isolation check no longer exists in the tree and its removal is recorded. +- **Verification:** All eleven checks assert in Rust, no `runNixOSTest` fixture remains, `make test-host-integration` runs entirely through the lane, and the documented grep gate passes. + +### U8. Sweep the documentation the migration invalidates + +- **Goal:** Make the repository's own documents match the shipped lane, including the guard that no longer exists. +- **Requirements:** R14, R16. +- **Dependencies:** U1, U7. +- **Files:** `AGENTS.md` (modify — the host-lane binary-injection rule), `tests/AGENTS.md` (modify — prose only; U4 owns the type-10 tier row), `tests/README.md`, `docs/contributing/gates-and-lints.md`, `docs/contributing/critical-subsystems.md`, `docs/reference/compatibility.md`, `docs/reference/support-matrix.md`, `packages/d2b-provider-device-usbip/integration/README.md` (modify — describes the semaphore as current), `docs/specs/providers/ADR-046-provider-volume-local.md`, `docs/specs/providers/ADR-046-provider-runtime-cloud-hypervisor.md`, `docs/specs/ADR-046-current-code-migration-map.md` (modify — carries a `RETAIN` disposition for the semaphore namespace, a different edit class that needs a named owner rather than a prose refresh), `specs/001-adr046-d2b3-completion/plan.md` (modify — states the semaphore contract), `CHANGELOG.md` (modify), `changelog.d/` (add). +- **Approach:** + 1. Replace the environment-variable handoff description with the declared-input handoff in every site that states it as current. + 2. Stop describing the heavy-gate semaphore as current and record its deletion rather than leaving the next reader to rediscover it. + 3. Update the contributor-facing description of the lane: filter-based check selection, the virtualization precondition, and the loss of the emulation fallback. + 4. Move the type-10 tier out of the description that keeps Layer-2 surfaces outside the Bazel scheduler, since the lane is now in that graph. +- **Test expectation:** none — documentation only. The grep scenario below is the check. +- **Test scenarios:** + - A repository-wide grep for the removed environment variables, the `vmChecks` output, and the heavy-gate semaphore returns nothing outside the changelog, the fragment directory, this plan, the audits and explanations, the specifications, and third-party trees, which legitimately keep a record of the removed names. + - Each documentation site that described the handoff now describes the declared-input handoff consistently. + - Contributor documentation describes check selection as a filter and names the virtualization precondition. +- **Verification:** The grep gate passes, and no document in the repository describes a guard or a handoff that no longer exists. + +--- + +## Verification Contract + +- `make check` — the full Layer-1 aggregate. Must stay green at every unit boundary, since the port's premise is that the branch is always releasable. +- `make test-host-integration` — the lane. Before U1, this is the current nix recipe and is the baseline; from U4, it is the lane test target. +- `make check-tier0` — the fast policy and source-hygiene subset, for the units that only touch build wiring. +- The recorded wall-clock baselines compared against the lane's full-suite and single-check runs. +- The restored-versus-fresh-boot measurement, taken before the pool grows past one member, and the marker-based equivalence gate that proves a restored guest matches a fresh boot on a member of every distinct invocation. +- The repository-wide grep gate over the removed environment variables, the `vmChecks` output, and the heavy-gate semaphore. +- No release-validation gate applies: this work changes no packaged artifact, only the lane that tests one. + +## Definition of Done + +- All eleven checks assert in Rust, each reporting its own result and diagnostics, and the full lane passes. +- No `runNixOSTest` fixture, no `vmChecks` output, and no nix VM orchestration remain reachable from `make test-host-integration`. +- The guest image is a cacheable Bazel graph output keyed on declared inputs, and a change to a guest module or a d2b host binary rebuilds it. +- The lane is measurably faster than the recorded baseline for both the full suite and a single named check, and a restored run is measurably cheaper than a fresh boot. +- A contributor can run one named check by filter and identify which check failed, and why, from the lane's test output alone. +- The repository's own instruction, contributor, and reference documents describe the shipped lane, name no removed handoff, and no longer describe the deleted heavy-gate semaphore as current. +- A changelog fragment records the migration and the retired coverage. +- Abandoned approaches from this work are removed, not left in the tree: any experimental launcher, any retained nix orchestration kept "just in case", any superseded guest-configuration shim, and any unused dependency added along the way. +- The lane's harness crate carries no test aggregate, so it stays out of the main package suite, and the lane's targets carry the tags that keep them out of the Layer-1 aggregate and out of remote execution. From 72e31fb44d2ac89ee2c68691e2db0539b30ae9d4 Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sat, 26 Sep 2026 09:30:32 -0700 Subject: [PATCH 02/51] feat(vm): build the host-integration guest image as a Bazel action The flake, its lock, the d2b module sources and the host binaries are declared label inputs, so a host-binary change moves the image's action key. Nix store, substituters, build-users-group, sandbox and HOME are configured inside the action, so hermeticity comes from nix's own configuration rather than the calling shell. The output is the bootable image - kernel, initrd and a qcow2 root disk with a pinned filesystem UUID and build clock - not a copy of the toplevel symlink tree, which Bazel rejects as a tree artifact. The manifest keeps the host-tool inventory and the per-check invocation fields the lane reproduces. The emulator comes from the existing pinned nix package extension rather than a new Bazel module, keeping emulator and guest at one nixpkgs revision. The recipe's closure upload retires: a network side effect in a build action would make the image uncacheable. The environment-variable handoff stays until the make recipe collapses, so the live lane keeps receiving Bazel-built binaries. --- .bazelrc | 12 + MODULE.bazel | 10 +- MODULE.bazel.lock | 20 +- Makefile | 10 +- bazel/checks/vm/BUILD.bazel | 51 +++++ bazel/checks/vm/defs.bzl | 214 ++++++++++++++++++ changelog.d/bazel-owned-guest-image.md | 14 ++ flake.nix | 138 +++++++---- nix/test-support/bazel-host-tools.nix | 16 +- nix/test-support/guest-image.nix | 121 ++++++++++ .../d2b-provider-display-wayland/BUILD.bazel | 2 +- .../BUILD.bazel | 2 +- .../d2b-provider-test-controller/BUILD.bazel | 2 +- 13 files changed, 553 insertions(+), 59 deletions(-) create mode 100644 bazel/checks/vm/BUILD.bazel create mode 100644 bazel/checks/vm/defs.bzl create mode 100644 changelog.d/bazel-owned-guest-image.md create mode 100644 nix/test-support/guest-image.nix diff --git a/.bazelrc b/.bazelrc index f1fa16fdc..5d00a805d 100644 --- a/.bazelrc +++ b/.bazelrc @@ -41,6 +41,18 @@ build:local --@rules_rust//rust/settings:extra_exec_rustc_flags= build:local --action_env=PATH build:local --test_env=PATH +# The host-integration lane's guest build profile. The lane builds the d2b +# host binaries that reach the guest through this config, named here +# rather than at a call site so a contributor's exported Bazel profile +# cannot change the guest closure: the guest image action realizes the +# binaries this config produces, whatever the caller set. +build:guest --config=local +# The guest binaries are content-addressed inputs of the guest image, so a +# caller-inherited action environment would change the guest closure. The +# profile pins the environment those builds see, the same way the remote +# profile pins it. +build:guest --action_env=PATH=/run/current-system/sw/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + # Remote profiles use the d2b BuildBuddy workspace. Authentication is # the credential helper only. Direct API-key headers leak into BEP. build:remote --remote_executor=grpcs://d2b.buildbuddy.io diff --git a/MODULE.bazel b/MODULE.bazel index f79a48c72..e9986c5b0 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -69,7 +69,15 @@ nix_packages.attr( attr = "python3", repo = "@nixpkgs", ) -use_repo(nix_packages, "nix", "nix_eval_jobs", "nix_unit", "python3") +# The host-integration lane's emulator, taken from the same pinned nix +# package set the guest image is realized from, so the emulator and the +# guest stay at one nixpkgs revision. +nix_packages.attr( + name = "qemu_kvm", + attr = "qemu_kvm", + repo = "@nixpkgs", +) +use_repo(nix_packages, "nix", "nix_eval_jobs", "nix_unit", "python3", "qemu_kvm") buildbuddy = use_extension("@toolchains_buildbuddy//:extensions.bzl", "buildbuddy") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 2a69a6fbc..70e917c13 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -727,7 +727,7 @@ "@@rules_nixpkgs_core+//extensions:package.bzl%nix_pkg": { "general": { "bzlTransitiveDigest": "2alEQC7q9PgrArc8ZeI662VmDHLGfuswqnzBJvVDnmg=", - "usagesDigest": "6f3lRgiMjLBysWeJGXZcKvEj4vtUKWpW+sjuFii+FBs=", + "usagesDigest": "PGNwR252ac/ou5hUxvwUPK/xMBbtgVRTpETRma9B5bU=", "recordedInputs": [ "REPO_MAPPING:,nixpkgs rules_nixpkgs_core++nix_repo+nixpkgs", "REPO_MAPPING:rules_nixpkgs_core+,bazel_skylib bazel_skylib+" @@ -792,6 +792,21 @@ "quiet": false, "fail_not_supported": true } + }, + "qemu_kvm": { + "repoRuleId": "@@rules_nixpkgs_core+//:nixpkgs.bzl%_nixpkgs_package", + "attributes": { + "unmangled_name": "qemu_kvm", + "attribute_path": "qemu_kvm", + "nix_file_deps": {}, + "nix_file_content": "", + "repository": "@@rules_nixpkgs_core++nix_repo+nixpkgs//:nixpkgs", + "repositories": {}, + "build_file_content": "", + "nixopts": [], + "quiet": false, + "fail_not_supported": true + } } }, "moduleExtensionMetadata": { @@ -799,7 +814,8 @@ "nix", "nix_unit", "nix_eval_jobs", - "python3" + "python3", + "qemu_kvm" ], "explicitRootModuleDirectDevDeps": [], "useAllRepos": "NO", diff --git a/Makefile b/Makefile index 463c8bbbe..da71d998f 100644 --- a/Makefile +++ b/Makefile @@ -184,6 +184,12 @@ generate: ## NixOS host; TCG software emulation is the slow fallback when /dev/kvm is ## absent). x86_64-linux only (a same-system VM builder is required). ## Set D2B_VM_CHECK= to build one named vmChecks entry. +## The host tools are built under the committed `guest` profile from +## .bazelrc, so an exported Bazel profile cannot change the guest closure. +## The Attic cache preflight and closure upload below belong to this nix +## recipe: the Bazel-owned lane's guest-image action declares its own +## substituters and preflights them itself, and the recipe retires with +## the nix lane. test-host-integration: @set -eu; \ system="$$(nix eval --raw --impure --expr builtins.currentSystem)"; \ @@ -256,8 +262,8 @@ test-host-integration: fi; \ echo "test-host-integration: Attic cache preflight passed"; \ fi; \ - echo "test-host-integration: building host tools with local Bazel"; \ - '$(BAZEL_BIN)' build --config=local \ + echo "test-host-integration: building host tools under the committed guest profile"; \ + '$(BAZEL_BIN)' build --config=guest \ //packages/d2b:d2b \ //packages/d2bd:d2bd \ //packages/d2b-broker-composition:d2b-broker \ diff --git a/bazel/checks/vm/BUILD.bazel b/bazel/checks/vm/BUILD.bazel new file mode 100644 index 000000000..50dbc4123 --- /dev/null +++ b/bazel/checks/vm/BUILD.bazel @@ -0,0 +1,51 @@ +load(":defs.bzl", "guest_image") + +package(default_visibility = ["//visibility:public"]) + +# Everything the guest evaluation reads. A guest image rebuilds when one +# of these changes, so the set names what the guest is made of - the +# flake, the d2b module tree, the host-tool sources, and the node +# configuration - rather than the whole workspace. +_GUEST_SOURCES = [ + "//:Cargo.lock", + "//:Cargo.toml", + "//:d2b_resource_schemas_v3", + "//:generated_cli_shell_artifacts", + "//:flake.lock", + "//:flake.nix", + "//:nix_workspace_sources", + "//:nixos_modules_workspace_sources", + "//:packages_workspace_sources", + "//packages/d2b-broker:src/ops/state-posture-contract.json", + "//:rust-toolchain.toml", + "//:tests/host-integration/lib.nix", + "//tests/fixtures:fixture_sources", +] + +guest_image( + name = "guest_image", + srcs = _GUEST_SOURCES, + cloud_hypervisor_controller = "//packages/d2b-provider-guest-cloud-hypervisor:d2b-cloud-hypervisor-controller", + flake = "//:flake.nix", + flake_lock = "//:flake.lock", + host_tools = [ + "//packages/d2b:d2b", + "//packages/d2bd:d2bd", + "//packages/d2b-broker-composition:d2b-broker", + "//packages/d2b-host:d2b-activation-helper", + "//packages/d2b-host-activation-helper:d2b-host-activation-helper", + "//packages/d2b-unsafe-local-helper:d2b-unsafe-local-helper", + "//packages/d2b-resource-compiler:d2b-resource-compiler", + "//packages/d2b-provider-display-wayland:d2b-wayland-proxy", + "//packages/d2b-provider-test-controller:d2b-provider-test-controller", + ], + nix = "@nix//:bin/nix", + substituters = "https://cache.nixos.org/", + tags = [ + "exclusive", + "local", + "no-remote-cache", + "no-remote-exec", + "no-sandbox", + ], +) diff --git a/bazel/checks/vm/defs.bzl b/bazel/checks/vm/defs.bzl new file mode 100644 index 000000000..18ba0d2a2 --- /dev/null +++ b/bazel/checks/vm/defs.bzl @@ -0,0 +1,214 @@ +"""Guest image for the Bazel-owned host-integration lane. + +The lane builds its guest as a graph output rather than as a side effect of +a shell recipe: the flake and its lock, the guest module sources, and the +d2b host binaries all arrive as declared label inputs, so the image is +keyed on the source and rebuilds when any of them changes. + +Modeled on `bazel/checks/fixtures/defs.bzl`, the repository's one existing +cacheable nix build action. + +The action runs unsandboxed and local. Nix cannot build inside a Bazel +sandbox - its own sandbox needs root, and the fallback degrades silently +rather than failing - so hermeticity comes from the action's own nix +configuration: the store it builds into, the substituters it may fetch +from, and the build-users setting are all set here, and the flake arrives +as a copied input rather than as a working-tree reference. +""" + +_GUEST_IMAGE_COMMAND = """\ +set -eu +label="%s" + +# Resolved before the fixed PATH is installed: on NixOS the setuid sudo is +# the wrapper, and the profile entry is not. +sudo_bin="" +for candidate in /run/wrappers/bin/sudo "$(command -v sudo 2>/dev/null || true)"; do + if [ -n "$candidate" ] && [ -x "$candidate" ]; then + sudo_bin="$candidate" + break + fi +done + +# Fixed PATH, a scratch HOME, and an explicit feature set: none of the +# action's nix configuration may come from the developer's shell. +export PATH=/run/current-system/sw/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export NIX_CONFIG="experimental-features = nix-command flakes" + +nix_bin="$1" +flake="$2" +lock="$3" +out="$4" +source_manifest="$5" +substituters="$6" +controller="$7" +shift 7 + +# Bazel hands over execroot-relative paths. Nix resolves its configuration +# against the working directory, so the output tree is anchored here. +case "$out" in + /*) ;; + *) out="$(pwd)/$out" ;; +esac + +store_dir=/nix/store +staging="$out.staging" +# The staged sources and binaries are scratch, on the failure path as much +# as on the success path. +trap 'rm -rf "$staging"' EXIT +fail() { + echo "guest-image $label: $1" >&2 + exit 1 +} +nix_run() { + if [ -w "$store_dir" ]; then + "$@" + elif [ -n "$sudo_bin" ]; then + "$sudo_bin" -E "$@" + else + fail "the nix store is not writable and no sudo is available, so the guest closure cannot be realized on this host" + fi +} + +rm -rf "$staging" +mkdir -p "$staging/home" +export HOME="$staging/home" +export XDG_CONFIG_HOME="$staging/home/config" +export XDG_CACHE_HOME="$staging/home/cache" + +# The flake arrives as declared label inputs, so the action key reflects +# the source rather than a working-tree reference. +source="$staging/source" +mkdir -p "$source" +while IFS= read -r input; do + [ -n "$input" ] || continue + destination="$source/$input" + mkdir -p "$(dirname "$destination")" + cp -L "$input" "$destination" +done < "$source_manifest" +root="$(CDPATH= cd -- "$source" && pwd -P)" +[ -f "$source/$flake" ] || fail "the declared flake ($flake) is not among the copied sources" +[ -f "$source/$lock" ] || fail "the declared flake lock ($lock) is not among the copied sources" + +# The d2b host binaries, staged by output name into the bundle the guest +# closure reads. The guest-side package refuses any other inventory, so a +# renamed or missing binary fails the build rather than producing a guest +# with a tool missing from it. +bundle="$staging/bundle" +mkdir -p "$bundle" +for tool in "$@"; do + install -m 755 "$tool" "$bundle/$(basename "$tool")" +done +controller_bundle="" +if [ -n "$controller" ]; then + controller_bundle="$staging/controller" + mkdir -p "$controller_bundle" + install -m 755 "$controller" "$controller_bundle/$(basename "$controller")" + controller_argument="\\"$controller_bundle\\"" +else + controller_argument="null" +fi + +# Preflight the declared substituters before building: an unreachable cache +# fails the action here rather than silently yielding an image from a +# partial closure. +reachable=0 +for cache in $substituters; do + if "$nix_bin" store info --store "$cache" >/dev/null 2>&1; then + reachable=1 + else + echo "guest-image $label: substituter $cache is unreachable" >&2 + fi +done +[ "$reachable" -eq 1 ] || fail "none of the declared substituters ($substituters) is reachable" + +system="$("$nix_bin" eval --raw --impure --expr builtins.currentSystem)" || fail "could not read the nix system" +expr="(builtins.getFlake \\"path:$root\\").guestImage.\\"$system\\" { rawBundle = \\"$bundle\\"; rawCloudHypervisorController = $controller_argument; }" +echo "guest-image $label: realizing the guest from declared inputs" >&2 +image="$(nix_run "$nix_bin" build \\ + --option store "local?store=$store_dir" \\ + --option substituters "$substituters" \\ + --option build-users-group "" \\ + --option sandbox true \\ + --option sandbox-fallback false \\ + --impure \\ + --no-write-lock-file \\ + --no-link \\ + --print-out-paths \\ + --expr "$expr")" || fail "the guest evaluation or build failed; see the nix output above" +[ -n "$image" ] && [ -d "$image" ] || fail "nix reported no guest image output ($image)" + +rm -rf "$staging" +mkdir -p "$out" +cp -a "$image/." "$out/" +[ -f "$out/manifest.json" ] || fail "the realized guest image carries no manifest" +""" + +def _guest_image_impl(ctx): + output = ctx.actions.declare_directory(ctx.label.name) + source_manifest = ctx.actions.declare_file(ctx.label.name + ".sources") + ctx.actions.write( + output = source_manifest, + content = "\n".join([source.path for source in ctx.files.srcs]) + "\n", + ) + controller = ctx.file.cloud_hypervisor_controller + ctx.actions.run_shell( + inputs = depset( + ctx.files.srcs + ctx.files.host_tools + ctx.files.cloud_hypervisor_controller + + [ctx.file.flake, ctx.file.flake_lock, source_manifest], + ), + tools = [ctx.executable.nix] + ctx.files.host_tools + ctx.files.cloud_hypervisor_controller, + outputs = [output], + arguments = [ + ctx.executable.nix.path, + ctx.file.flake.short_path, + ctx.file.flake_lock.short_path, + output.path, + source_manifest.path, + ctx.attr.substituters, + controller.path if controller else "", + ] + [tool.path for tool in ctx.files.host_tools], + command = _GUEST_IMAGE_COMMAND % {"label": str(ctx.label)}, + mnemonic = "GuestImage", + progress_message = "Building d2b guest image %s" % ctx.label.name, + ) + return [DefaultInfo(files = depset([output]))] + +guest_image = rule( + implementation = _guest_image_impl, + doc = "Realize the host-integration guest image from declared inputs.", + attrs = { + "cloud_hypervisor_controller": attr.label( + allow_single_file = True, + ), + "flake": attr.label( + allow_single_file = True, + mandatory = True, + ), + "flake_lock": attr.label( + allow_single_file = True, + mandatory = True, + ), + # The guest's own binaries, taken in the target configuration: the + # guest runs the binaries this build produces, not a second copy + # built for the action's own execution platform. + "host_tools": attr.label_list( + allow_files = True, + mandatory = True, + ), + "nix": attr.label( + allow_single_file = True, + cfg = "exec", + executable = True, + mandatory = True, + ), + "srcs": attr.label_list(allow_files = True, mandatory = True), + # The caches this action may fetch the guest closure from. They are + # declared rather than read from the host so an unreachable cache + # fails the action, and so a developer's nix configuration cannot + # change what the guest is built from. + "substituters": attr.string( + default = "https://cache.nixos.org/", + ), + }, +) diff --git a/changelog.d/bazel-owned-guest-image.md b/changelog.d/bazel-owned-guest-image.md new file mode 100644 index 000000000..31bbe32a6 --- /dev/null +++ b/changelog.d/bazel-owned-guest-image.md @@ -0,0 +1,14 @@ +### Added + +- Added the host-integration guest image as a Bazel build action. `bazel build //bazel/checks/vm:guest_image` realizes the guest's NixOS closure from declared label inputs - the flake, its lock, the d2b module sources, and the nine d2b host binaries plus the Cloud Hypervisor controller - so the image is a graph output that rebuilds when a guest module or a host binary changes, and a second build of an unchanged tree reuses it. The action configures its own nix store, substituters, and build-users setting, and preflights the substituters before building, so the image no longer depends on the developer's shell configuration. The guest's `d2bHostToolOverrides` now come from that declared label set rather than from an environment variable. +- The action's output is the bootable guest image, not a copy of the system closure's symlink tree: a kernel, an initrd, and a qcow2 root disk, built the way NixOS's own VM module builds them, alongside a manifest naming those artifacts, the host-tool package the closure was built against, and the per-check invocation shape. A toplevel symlink tree is not an image a lane can boot, and copying it into a build output only produces symlinks that resolve outside it. The disk's filesystem UUID and build clock are pinned, so the same inputs produce the same bytes and the image stays cacheable. +- Added the emulator to the pinned nix package extension in `MODULE.bazel`, so the lane's emulator comes from the same nixpkgs revision the guest image is realized from. + +### Changed + +- Factored the flake's Bazel host-tool wiring into one helper shared by both guest entry points, so a guest realized from the declared-input action and a guest realized from the legacy `D2B_HOST_TOOL_BUNDLE` environment handoff are built the same way. Both environment reads remain until the Bazel lane replaces the nix recipe. +- `make test-host-integration` now builds the guest's host tools under the committed `guest` profile from `.bazelrc`, so an exported Bazel profile cannot change the guest closure. + +### Removed + +- Retired the recipe's Attic closure upload from the new guest-image path: a network side effect would make the image uncacheable, and the build cache the image lands in replaces it. The upload stays with the nix recipe and retires with the lane, which is recorded here rather than left for the next reader to rediscover. diff --git a/flake.nix b/flake.nix index fb197e7e4..894378a41 100644 --- a/flake.nix +++ b/flake.nix @@ -84,6 +84,61 @@ in found; providerElfShim = import ./nix/provider-elf-shim.nix; + + # Wire the Bazel-built d2b host binaries into a guest: one nix + # package for the bundle, one self override that hands the package to + # the guest's modules. Both guest entry points use it, so a guest + # realized from the declared-input action and a guest realized from + # the legacy environment handoff are built the same way. + mkBazelHostTools = system: rawBundle: rawCloudHypervisorController: + let + bundlePath = builtins.path { + path = /. + rawBundle; + name = "d2b-bazel-host-tools"; + }; + cloudHypervisorControllerPath = + if rawCloudHypervisorController == null then + null + else + builtins.path { + path = /. + rawCloudHypervisorController; + name = "d2b-bazel-cloud-hypervisor-controller"; + }; + tools = import ./nix/test-support/bazel-host-tools.nix { + pkgs = nixpkgsFor.${system}; + rawBundle = bundlePath; + rawCloudHypervisorController = cloudHypervisorControllerPath; + }; + in + { + inherit tools; + hostSelf = self // { + lib = self.lib // { + d2bHostToolOverrides = tools.d2bHostToolOverrides; + d2bHostToolBundle = tools.package; + evalGuest = args: self.lib.evalGuest (args // { + d2bHostToolOverrides = tools.d2bHostToolOverrides; + }); + }; + nixosModules = self.nixosModules // { + default = { + imports = [ self.nixosModules.default ]; + _module.args.d2bHostToolOverrides = tools.d2bHostToolOverrides; + }; + }; + packages = self.packages // { + ${system} = self.packages.${system} // { + d2b-wayland-proxy = tools.package; + } // nixpkgs.lib.optionalAttrs + (tools.cloudHypervisorControllerPackage != null) + { + d2b-cloud-hypervisor-controller = + tools.cloudHypervisorControllerPackage; + }; + }; + }; + }; + # The Guest static workspace mirrors the shared daemon/broker dependency # closure. Guest packaging contains only the shared daemon, broker, # and signed Provider workspace inputs. @@ -608,60 +663,26 @@ if system == "x86_64-linux" then let pkgs = nixpkgsFor.${system}; + # The two environment reads are the legacy handoff. The + # Bazel-owned lane reaches the same package from the guest + # image action's declared label inputs; these reads stay until + # that lane replaces the recipe. hostToolBundleEnv = builtins.getEnv "D2B_HOST_TOOL_BUNDLE"; cloudHypervisorControllerBundleEnv = builtins.getEnv "D2B_CH_CONTROLLER_BUNDLE"; - bazelHostTools = + handoff = if hostToolBundleEnv == "" then null else - import ./nix/test-support/bazel-host-tools.nix { - inherit pkgs; - rawBundle = builtins.path { - path = /. + hostToolBundleEnv; - name = "d2b-bazel-host-tools"; - }; - rawCloudHypervisorController = - if cloudHypervisorControllerBundleEnv == "" then null else - builtins.path { - path = /. + cloudHypervisorControllerBundleEnv; - name = "d2b-bazel-cloud-hypervisor-controller"; - }; - }; + mkBazelHostTools system hostToolBundleEnv + (if cloudHypervisorControllerBundleEnv == "" then + null + else + cloudHypervisorControllerBundleEnv); testSelf = - if bazelHostTools == null then - self - else - self // { - lib = self.lib // { - d2bHostToolOverrides = - bazelHostTools.d2bHostToolOverrides; - d2bHostToolBundle = bazelHostTools.package; - evalGuest = args: self.lib.evalGuest (args // { - d2bHostToolOverrides = - bazelHostTools.d2bHostToolOverrides; - }); - }; - nixosModules = self.nixosModules // { - default = { - imports = [ self.nixosModules.default ]; - _module.args.d2bHostToolOverrides = - bazelHostTools.d2bHostToolOverrides; - }; - }; - packages = self.packages // { - ${system} = self.packages.${system} - // { - d2b-wayland-proxy = bazelHostTools.package; - } - // nixpkgs.lib.optionalAttrs - (bazelHostTools.cloudHypervisorControllerPackage != null) - { - d2b-cloud-hypervisor-controller = - bazelHostTools.cloudHypervisorControllerPackage; - }; - }; - }; + if handoff == null then self else handoff.hostSelf; + bazelHostTools = + if handoff == null then null else handoff.tools; testDir = ./tests/host-integration; testFiles = if builtins.pathExists testDir then builtins.attrNames (nixpkgs.lib.filterAttrs @@ -681,6 +702,29 @@ in builtins.listToAttrs (map mkTest testFiles) else { }); + # The guest image for the Bazel-owned host-integration lane. It is a + # function, not a package: the lane's guest-image action calls it + # with the d2b host binaries it received as declared label inputs, so + # the guest closure is keyed on the Bazel graph rather than on a + # developer's shell. The staged directories are the same bundle the + # legacy environment handoff passes, and the same host-tool package + # consumes them, so both paths realize the same guest. + # + # `rawBundle` and `rawCloudHypervisorController` are paths to + # directories holding the binaries by output name. `extraModules` + # carries a check's own guest contributions. + guestImage = forAllSystems (system: + { rawBundle, rawCloudHypervisorController ? null, extraModules ? [ ] }: + let + handoff = mkBazelHostTools system rawBundle rawCloudHypervisorController; + in + import ./nix/test-support/guest-image.nix { + inherit extraModules rawBundle; + pkgs = nixpkgsFor.${system}; + bazelHostTools = handoff.tools; + self = handoff.hostSelf; + }); + templates.default = { path = ./templates/default; description = "Minimal d2b host scaffold - one Zone"; diff --git a/nix/test-support/bazel-host-tools.nix b/nix/test-support/bazel-host-tools.nix index 496666f02..40c317f10 100644 --- a/nix/test-support/bazel-host-tools.nix +++ b/nix/test-support/bazel-host-tools.nix @@ -3,6 +3,10 @@ let inherit (pkgs) lib; + # The inventory is the handoff contract between the Bazel targets that + # build these binaries and the guest closure that consumes them. It is + # exported so a producer can be checked against it before the guest is + # evaluated, rather than only once its closure has been built. inventory = [ "d2b" "d2bd" @@ -14,6 +18,7 @@ let "d2b-wayland-proxy" "d2b-provider-test-controller" ]; + controllerName = "d2b-cloud-hypervisor-controller"; inventoryShell = lib.escapeShellArgs inventory; overrideKeys = [ "d2b" @@ -194,22 +199,22 @@ let installPhase = '' runHook preInstall actual="$(find -P "$src" -mindepth 1 -maxdepth 1 -printf '%f\n')" - if [ "$actual" != "d2b-cloud-hypervisor-controller" ]; then + if [ "$actual" != "${controllerName}" ]; then echo "d2b-bazel-cloud-hypervisor-controller: raw bundle inventory mismatch" >&2 exit 1 fi - source="$src/d2b-cloud-hypervisor-controller" + source="$src/${controllerName}" if [ ! -f "$source" ] || [ -L "$source" ] || [ ! -x "$source" ]; then echo "d2b-bazel-cloud-hypervisor-controller: expected a regular executable" >&2 exit 1 fi - install -Dm755 "$source" "$out/bin/d2b-cloud-hypervisor-controller" + install -Dm755 "$source" "$out/bin/${controllerName}" runHook postInstall ''; doInstallCheck = true; installCheckPhase = '' runHook preInstallCheck - bin="$out/bin/d2b-cloud-hypervisor-controller" + bin="$out/bin/${controllerName}" header="$(${pkgs.binutils}/bin/readelf -h "$bin")" grep -Eq 'Class:[[:space:]]+ELF64' <<< "$header" grep -Eq 'Machine:[[:space:]]+(Advanced Micro Devices X86-64|x86-64)' <<< "$header" @@ -229,5 +234,8 @@ let in { inherit package cloudHypervisorControllerPackage; + # The handoff contract, exported so a caller can refuse an incomplete + # binary set before it evaluates a guest closure. + inherit inventory controllerName; d2bHostToolOverrides = lib.genAttrs overrideKeys (_: package); } diff --git a/nix/test-support/guest-image.nix b/nix/test-support/guest-image.nix new file mode 100644 index 000000000..7e36a3b34 --- /dev/null +++ b/nix/test-support/guest-image.nix @@ -0,0 +1,121 @@ +# Guest image for the Bazel-owned host-integration lane. +# +# The lane's guest-image action (`bazel/checks/vm/defs.bzl`) calls this +# through the flake's `guestImage` output, passing the d2b host binaries it +# received as declared Bazel label inputs. The guest closure is therefore +# keyed on the Bazel graph rather than on a developer's shell: `rawBundle` +# and `rawCloudHypervisorController` are the staged binary directories, by +# the same contract the legacy `D2B_HOST_TOOL_BUNDLE` handoff passes, and +# the caller content-addresses them. +# +# The guest is the node the current `vmChecks` fixtures boot, taken from +# the same `d2bDaemonNode` configuration and the same Bazel-built host-tool +# package, so a guest realized through this entry point and a guest +# realized through the legacy handoff are the same guest. Per-check module +# contributions arrive through `extraModules`. +# +# The output is one store path holding the guest's system closure and a +# manifest of what a launcher needs to boot it: the toplevel, the kernel +# and initrd entry points, the declared invocation shape, and the host-tool +# package the closure was built against. +{ pkgs, self, bazelHostTools, rawBundle, extraModules ? [ ] }: + +let + inherit (pkgs) lib; + + # Refuse an incomplete handoff before a guest closure is evaluated. The + # host-tool package repeats this check when it is built, but that is the + # wrong place to learn a binary is missing: by then the guest closure has + # been realized. + stagedEntries = builtins.filter (name: name != "." && name != "..") + (lib.attrNames (builtins.readDir (/. + rawBundle))); + missing = lib.filter (name: !(builtins.elem name stagedEntries)) bazelHostTools.inventory; + unexpected = lib.filter (name: !(builtins.elem name bazelHostTools.inventory)) stagedEntries; + + d2bLib = import ../../tests/host-integration/lib.nix { + self = self; + inherit (pkgs) lib; + hostToolBundle = bazelHostTools.package; + }; + + # `d2bDaemonNode` declares `virtualisation.*`, so the guest is evaluated + # with the same QEMU VM module the runNixOSTest nodes carry. Evaluating + # the node module directly, rather than through the test driver, is what + # makes the result a bootable system closure the lane's own launcher can + # use. + evaluated = import (pkgs.path + "/nixos/lib/eval-config.nix") { + system = pkgs.stdenv.hostPlatform.system; + modules = [ + (pkgs.path + "/nixos/modules/virtualisation/qemu-vm.nix") + (d2bLib.d2bDaemonNode { extra = { imports = extraModules; }; }) + { + virtualisation.host.pkgs = pkgs; + } + ]; + }; + guest = evaluated.config; + toplevel = guest.system.build.toplevel; + + # The artifacts a launcher boots, the way NixOS's own VM module produces + # them: the kernel and initrd that carry the system closure, and a root + # disk in the format the module's run script builds. Real files, not a + # copy of the toplevel symlink tree, so the image is what the lane boots. + diskSizeMib = guest.virtualisation.diskSize; + manifest = { + system = pkgs.stdenv.hostPlatform.system; + kernel = "kernel"; + initrd = "initrd"; + disk = "disk.qcow2"; + diskFormat = "qcow2"; + init = "${toplevel}/init"; + toplevel = toplevel; + hostToolBundle = bazelHostTools.package; + hostToolInventory = bazelHostTools.inventory; + cloudHypervisorController = bazelHostTools.cloudHypervisorControllerPackage; + inherit (guest.virtualisation) cores diskSize memorySize; + qemuOptions = guest.virtualisation.qemu.options; + }; +in +if missing != [ ] || unexpected != [ ] then + throw '' + d2b guest image: the staged Bazel host-tool bundle does not match the + declared inventory. + missing: ${lib.concatStringsSep " " (if missing == [ ] then [ "(none)" ] else missing)} + unexpected: ${lib.concatStringsSep " " (if unexpected == [ ] then [ "(none)" ] else unexpected)} + '' +else + # The filesystem UUID and the build clock are pinned so the same inputs + # produce the same bytes: an image that changed hash on every build + # would not be a cacheable graph output. + let + fakeTime = "1"; + in + pkgs.runCommand "d2b-vm-guest-image" { + nativeBuildInputs = [ pkgs.jq pkgs.e2fsprogs pkgs.qemu ]; + } '' + mkdir -p "$out" + + # The initrd carries the system closure, so the kernel and initrd are + # the guest's real system. The toplevel's entries are store symlinks; + # -L resolves them to files. + cp -L ${toplevel}/kernel "$out/kernel" + cp -L ${toplevel}/initrd "$out/initrd" + + # The root disk, built the way the VM module's own run script builds + # it: an ext4 filesystem, converted to qcow2 so the lane can snapshot + # and restore it. + export E2FSPROGS_FAKE_TIME=${fakeTime} + ${pkgs.qemu}/bin/qemu-img create -f raw "$TMPDIR/root.raw" ${toString diskSizeMib}M + ${pkgs.e2fsprogs}/bin/mkfs.ext4 -q -F -L nixos -U 00000000-0000-0000-0000-000000000001 "$TMPDIR/root.raw" + ${pkgs.qemu}/bin/qemu-img convert -f raw -O qcow2 "$TMPDIR/root.raw" "$out/disk.qcow2" + rm -f "$TMPDIR/root.raw" + + # The manifest names the artifacts relative to the image root, the + # exact host-tool package the closure was built against, and the + # invocation shape the lane reproduces per check. + cat >"$out/manifest.json" <<'JSON' + ${builtins.toJSON manifest} + JSON + jq --sort-keys . "$out/manifest.json" >"$out/manifest.sorted" + mv "$out/manifest.sorted" "$out/manifest.json" + '' diff --git a/packages/d2b-provider-display-wayland/BUILD.bazel b/packages/d2b-provider-display-wayland/BUILD.bazel index 49dd5594d..92c501f43 100644 --- a/packages/d2b-provider-display-wayland/BUILD.bazel +++ b/packages/d2b-provider-display-wayland/BUILD.bazel @@ -9,7 +9,7 @@ load( # Provider crates link only from the daemon composition root and the # enumerated consumers; broker/shared crates cannot depend on them (U14). -package(default_visibility = ["//bazel/checks:__pkg__", "//packages/d2b-provider-wayland-policy:__pkg__", "//packages/d2b-provider-wayland-session:__pkg__", "//packages/d2bd:__pkg__"]) +package(default_visibility = ["//bazel/checks:__pkg__", "//bazel/checks/vm:__pkg__", "//packages/d2b-provider-wayland-policy:__pkg__", "//packages/d2b-provider-wayland-session:__pkg__", "//packages/d2bd:__pkg__"]) exports_files( [ diff --git a/packages/d2b-provider-guest-cloud-hypervisor/BUILD.bazel b/packages/d2b-provider-guest-cloud-hypervisor/BUILD.bazel index 4fed183ae..8bbc772a6 100644 --- a/packages/d2b-provider-guest-cloud-hypervisor/BUILD.bazel +++ b/packages/d2b-provider-guest-cloud-hypervisor/BUILD.bazel @@ -9,7 +9,7 @@ load( # Provider crates link only from the daemon composition root and the # enumerated consumers; broker/shared crates cannot depend on them (U14). -package(default_visibility = ["//bazel/checks:__pkg__", "//packages/d2b-provider-endpoint:__pkg__", "//packages/d2b-provider-guest:__pkg__", "//packages/d2b-provider-network-local:__pkg__", "//packages/d2b-provider-process:__pkg__", "//packages/d2bd:__pkg__"]) +package(default_visibility = ["//bazel/checks:__pkg__", "//bazel/checks/vm:__pkg__", "//packages/d2b-provider-endpoint:__pkg__", "//packages/d2b-provider-guest:__pkg__", "//packages/d2b-provider-network-local:__pkg__", "//packages/d2b-provider-process:__pkg__", "//packages/d2bd:__pkg__"]) exports_files( [ diff --git a/packages/d2b-provider-test-controller/BUILD.bazel b/packages/d2b-provider-test-controller/BUILD.bazel index 07fa3b20d..514f5a75f 100644 --- a/packages/d2b-provider-test-controller/BUILD.bazel +++ b/packages/d2b-provider-test-controller/BUILD.bazel @@ -9,7 +9,7 @@ load( # Provider crates link only from the daemon composition root and the # enumerated consumers; broker/shared crates cannot depend on them (U14). -package(default_visibility = []) +package(default_visibility = ["//bazel/checks/vm:__pkg__"]) exports_files( [ From 3beb7a27ac1a83a152d417d39586ba1dcdeaf1eb Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sat, 26 Sep 2026 09:51:05 -0700 Subject: [PATCH 03/51] refactor(vm): re-home per-check guest configuration from the fixtures The reusable NixOS node configuration - the daemon host, the nested cloud-hypervisor host, the shared base config and the acceptance-unit list - moves byte-identically out of the runNixOSTest fixture library into a module the lane evaluates, so a fixture can be deleted when its check ports without taking its guest declaration with it. The library keeps only the driver-coupled half, which dies with the fixtures. The eight fixtures that used the shared node are repointed at the re-homed module with no re-export shim: each diff is the import binding plus the node-constructor qualifier, and no testScript line changes. The other four never used it and are untouched. Per-check invocation is read off the evaluated config rather than restated, so the image action and the harness cannot drift from what the guest is actually configured to be. Carried here so the history is not broken: the image action now imports the re-homed module, and the stale fixture-library entry is dropped from its declared inputs so editing a driver diagnostic no longer invalidates a cacheable image the guest does not depend on. Both edits are behaviour-preserving - the rebuilt image carries the identical toplevel, host-tool bundle and guest sizing. --- bazel/checks/vm/BUILD.bazel | 1 - .../rehome-host-integration-guest-config.md | 25 +++ nix/test-support/guest-image.nix | 8 +- nix/test-support/host-integration-node.nix | 203 ++++++++++++++++++ tests/host-integration/daemon-smoke.nix | 8 +- .../deferred/host-zone-gateway-isolation.nix | 6 +- .../host-integration/device-worker-launch.nix | 8 +- tests/host-integration/lib.nix | 191 ++-------------- tests/host-integration/privilege-oracle.nix | 8 +- .../resource-operator-activation.nix | 8 +- ...ntime-cloud-hypervisor-guest-preflight.nix | 8 +- .../state-posture-contract.nix | 8 +- .../virtiofsd-volume-runtime.nix | 8 +- 13 files changed, 302 insertions(+), 188 deletions(-) create mode 100644 changelog.d/rehome-host-integration-guest-config.md create mode 100644 nix/test-support/host-integration-node.nix diff --git a/bazel/checks/vm/BUILD.bazel b/bazel/checks/vm/BUILD.bazel index 50dbc4123..9ae1093ba 100644 --- a/bazel/checks/vm/BUILD.bazel +++ b/bazel/checks/vm/BUILD.bazel @@ -18,7 +18,6 @@ _GUEST_SOURCES = [ "//:packages_workspace_sources", "//packages/d2b-broker:src/ops/state-posture-contract.json", "//:rust-toolchain.toml", - "//:tests/host-integration/lib.nix", "//tests/fixtures:fixture_sources", ] diff --git a/changelog.d/rehome-host-integration-guest-config.md b/changelog.d/rehome-host-integration-guest-config.md new file mode 100644 index 000000000..a0f6c9f5d --- /dev/null +++ b/changelog.d/rehome-host-integration-guest-config.md @@ -0,0 +1,25 @@ +### Added + +- Added `nix/test-support/host-integration-node.nix`, the reusable d2b + host-integration guest configuration, so a check's guest declaration + outlives the `runNixOSTest` fixture that used to carry it. The module + names both guest shapes the lane has to reproduce rather than collapsing + them: `d2bDaemonNode` attaches a dedicated `/var/lib/d2b` state disk and + keeps the VM module's writeback root cache, while + `d2bCloudHypervisorNode` drops that disk, replaces the root drive's cache + with `unsafe`, and boots through a bootloader so `/nix/store` and + `/var/lib/d2b` share one filesystem for the hardlink farm. A per-check + module passed as `extra` merges through the same import the fixture used, + so its own memory, vCPU, disk, drive, and device declarations reach the + same options list the shape contributes to. The lane reads each check's + invocation - memory, vCPU count, disk size, `useBootLoader`, + `virtualisation.qemu.drives`, and `virtualisation.qemu.options` - back off + the evaluated configuration instead of booting one uniform guest. + +### Changed + +- Split `tests/host-integration/lib.nix`. What stays is the part bound to + the test driver: the diagnostics prelude, the nested guest systems only + these fixtures boot, and the provider artifacts only these fixtures + install. The eight fixtures that boot a d2b daemon host now import the + re-homed module directly. No fixture's assertions changed. diff --git a/nix/test-support/guest-image.nix b/nix/test-support/guest-image.nix index 7e36a3b34..6cca3f209 100644 --- a/nix/test-support/guest-image.nix +++ b/nix/test-support/guest-image.nix @@ -32,10 +32,12 @@ let missing = lib.filter (name: !(builtins.elem name stagedEntries)) bazelHostTools.inventory; unexpected = lib.filter (name: !(builtins.elem name bazelHostTools.inventory)) stagedEntries; - d2bLib = import ../../tests/host-integration/lib.nix { + # The shared node configuration, re-homed so the guest outlives the + # fixtures that used to declare it. The Bazel host-tool package reaches + # the guest through the self override, not through this module. + d2bNode = import ./host-integration-node.nix { self = self; inherit (pkgs) lib; - hostToolBundle = bazelHostTools.package; }; # `d2bDaemonNode` declares `virtualisation.*`, so the guest is evaluated @@ -47,7 +49,7 @@ let system = pkgs.stdenv.hostPlatform.system; modules = [ (pkgs.path + "/nixos/modules/virtualisation/qemu-vm.nix") - (d2bLib.d2bDaemonNode { extra = { imports = extraModules; }; }) + (d2bNode.d2bDaemonNode { extra = { imports = extraModules; }; }) { virtualisation.host.pkgs = pkgs; } diff --git a/nix/test-support/host-integration-node.nix b/nix/test-support/host-integration-node.nix new file mode 100644 index 000000000..ed196b9f9 --- /dev/null +++ b/nix/test-support/host-integration-node.nix @@ -0,0 +1,203 @@ +# Guest node configuration for the d2b host-integration lane. +# +# This is the reusable half of the VM fixtures' shared node: the module +# every d2b daemon-host check boots, plus the per-check module contribution +# that rides on top of it. It lives here rather than under +# `tests/host-integration/` because that directory is removed check by check +# as each one ports; the guest a check needs has to outlive the fixture that +# used to declare it. +# +# The lane evaluates this module directly, carrying the same QEMU VM module +# the runNixOSTest nodes carry, and reads each check's invocation back off +# the evaluated configuration: `virtualisation.{memorySize,cores,diskSize}`, +# `virtualisation.useBootLoader`, `virtualisation.qemu.drives`, and +# `virtualisation.qemu.options`. Every one of those fields is declared here +# exactly once, so a guest is reproduced from its own declaration rather than +# from one uniform shape. A per-check module passed as `extra` merges through +# the same import the fixture used, so its own memory, drive, and device +# declarations - a vsock device, say - reach the same options list the shape +# contributes to. +# +# Two shapes, both named here and neither collapsed into the other: +# +# d2bDaemonNode the default node. /var/lib/d2b rides a dedicated +# state disk attached through `qemu.options`, and the +# root image keeps the VM module's own writeback +# cache. +# d2bCloudHypervisorNode the writable-store node. /nix/store and +# /var/lib/d2b must share one filesystem for the +# hardlink farm, so the state disk is dropped, the +# root drive replaces it with an unsafe cache, and +# the node boots through a bootloader. +{ self, lib }: + +let + # The minimal, hermetic d2b site declaration every daemon-host node shares. + # Zone and Guest resources belong to the acceptance fixture that exercises + # them; keeping this base free of legacy VM/env authoring prevents unrelated + # host checks from silently materializing a second lifecycle graph. + daemonAcceptanceUnits = [ + "d2bd.service" + "d2b-broker.socket" + "d2b-broker.service" + ]; + + baseD2bConfig = { + d2b.site = { + waylandUser = "alice"; + launcherUsers = [ "alice" ]; + yubikey.enable = false; + usePrebuiltHostTools = false; + }; + # The daemon's v3 bundle always carries the local-root storage row. Keep + # the corresponding root Zone in these minimal host fixtures so the + # emitted topology is sealed and the daemon can enter Ready. + d2b.zones.local-root = { }; + # The full daemon + broker systemd surface under test. + d2b.daemonExperimental.enable = true; + }; +in +rec { + # A NixOS module for a runNixOSTest node that boots the d2b daemon host. + # `extra` is merged as an additional module so individual tests can add + # per-test Zone/Guest resources, tampering helpers, or a larger disk. The + # node provisions the `alice` operator user the base config references. + # + # Structured as an attrset-module with everything in `imports` (an attrset is + # a valid module): `imports` must be top-level, NOT wrapped in `lib.mkMerge`, + # or the module system rejects it ("option nodes.machine.imports does not + # exist"). + d2bDaemonNode = + { extra ? { }, writableStore ? false }: + { config, pkgs, ... }: + let + # Dedicated state disk: the redb Zone store pays an fsync per commit + # on the emulated root disk, and bring-up write bursts stall the + # daemon writer thread ~700-900ms per write. Attaching /var/lib/d2b + # as its own virtio drive with cache=unsafe makes guest fsync a host + # page-cache no-op; the fixture VM is ephemeral, so the durability + # semantics that unsafe drops are irrelevant here. + stateDisk = pkgs.runCommand "d2b-state.img" + { + nativeBuildInputs = [ pkgs.e2fsprogs ]; + } + '' + truncate -s 4G "$out" + mkfs.ext4 -q -F "$out" + ''; + in + { + imports = [ + self.nixosModules.default + baseD2bConfig + extra + { + # Headroom for building/activating the bundle + daemon closure inside + # the VM; the default 1024 MiB is tight once the broker spawns + # runners. + virtualisation.memorySize = 3072; + virtualisation.diskSize = 8192; + # The daemon's redb writer thread, the daemon async runtime, the + # broker, and three controllers all need real CPU. The 1-vCPU + # default serializes them and starves every 250ms handshake. + virtualisation.cores = 3; + boot.kernelModules = [ "br_netfilter" "tun" "vhost_net" ]; + + users.users.alice = { + isNormalUser = true; + uid = 1000; + }; + + environment.etc."d2b/daemon-acceptance-units".text = + lib.concatStringsSep "\n" daemonAcceptanceUnits + "\n"; + + # Fail VM checks promptly when daemon startup is deterministically + # broken instead of spending the lane timeout in a restart loop. + systemd.services.d2bd.unitConfig = { + StartLimitIntervalSec = "30s"; + StartLimitBurst = 3; + }; + + # runNixOSTest runs first-boot activation before systemd-tmpfiles has + # materialized the d2b state tree. Pre-create the state directory so + # daemon-owned startup can rely on the same path ordering. + system.activationScripts.d2bTestStateDirs = { + deps = [ "users" ]; + text = '' + install -d -m 0750 -o root -g d2bd /var/lib/d2b + install -d -m 0710 -o root -g d2b /var/lib/d2b/keys + : > /var/lib/d2b/keys/.lock + chown root:root /var/lib/d2b/keys/.lock + chmod 0600 /var/lib/d2b/keys/.lock + ''; + }; + system.stateVersion = "25.11"; + } + # Opt-in writable same-fs store. ONLY needed by tests that drive the + # per-VM /nix/store hardlink farm (which requires /var/lib/d2b and + # /nix/store on the SAME filesystem - hardlinks can't cross FS - and the + # default runNixOSTest read-only store image splits them). It is OFF by + # default: `virtualisation.writableStore = true` copies the entire guest + # closure into a writable overlay at boot, which adds many minutes to + # (and can hang) VM startup. The daemon/broker activation + host-posture + # tests (daemon-smoke, bridge-isolation, privilege-oracle) + # never boot a microVM, so they never touch the farm - keep this off for + # a fast, reliable boot. + (lib.mkIf writableStore { + virtualisation.useBootLoader = true; + # The guest store-view hardlinks /nix/store into /var/lib/d2b, so + # both must stay on one filesystem - a separate state disk would + # break the hardlink farm with EXDEV. Instead, drop the root + # drive's cache to unsafe: every redb commit's fsync becomes a + # host page-cache no-op instead of a ~700-900ms stall, and the + # fixture VM is ephemeral, so the lost durability is irrelevant. + virtualisation.qemu.drives = lib.mkForce [ + { + name = "root"; + file = ''"$NIX_DISK_IMAGE"''; + driveExtraOpts.cache = "unsafe"; + driveExtraOpts.werror = "report"; + deviceExtraOpts.bootindex = "1"; + deviceExtraOpts.serial = "root"; + } + ]; + }) + # The state disk keeps /var/lib/d2b off the emulated root disk: + # cache=unsafe (host fsync no-op), noatime + nobarrier mounts. + # The writableStore hardlink-farm tests stay on the default + # same-fs layout, so this is opt-out for them. + (lib.mkIf (! writableStore) { + # The image is a `pkgs.runCommand` output, so QEMU must not need + # write access to it: the lane builds these checks inside the Nix + # sandbox, where /nix/store is mounted read-only, and a writable + # drive on a store path makes QEMU abort at machine start - the + # test driver surfaces that as a bare "Connection reset by peer". + # `snapshot=on` opens the backing file read-only and keeps every + # guest write in an ephemeral per-VM overlay under TMPDIR, which + # matches the fixture's ephemeral state disk either way. + virtualisation.qemu.options = [ + "-drive" + "file=${stateDisk},format=raw,if=virtio,cache=unsafe,aio=threads,snapshot=on" + ]; + fileSystems."/var/lib/d2b" = { + device = "/dev/vdb"; + fsType = "ext4"; + options = [ "noatime" "nobarrier" ]; + # Up before activation so d2bTestStateDirs lands inside the + # mounted filesystem, not under the covered root mountpoint. + neededForBoot = true; + }; + }) + ]; + }; + + # Shared host posture for every fixture that boots a Cloud Hypervisor Guest. + # The hardlink-backed Guest store view requires a writable host store on the + # same filesystem as /var/lib/d2b. + d2bCloudHypervisorNode = + { extra ? { } }: + d2bDaemonNode { + inherit extra; + writableStore = true; + }; +} diff --git a/tests/host-integration/daemon-smoke.nix b/tests/host-integration/daemon-smoke.nix index 1cca32cfd..e3280e070 100644 --- a/tests/host-integration/daemon-smoke.nix +++ b/tests/host-integration/daemon-smoke.nix @@ -16,11 +16,17 @@ let inherit self; inherit (pkgs) lib; }; + # The reusable guest configuration lives outside this directory so it + # survives the fixture (see `nix/test-support/host-integration-node.nix`). + d2bNode = import ../../nix/test-support/host-integration-node.nix { + inherit self; + inherit (pkgs) lib; + }; in pkgs.testers.runNixOSTest { name = "d2b-daemon-smoke"; - nodes.machine = d2bLib.d2bDaemonNode { + nodes.machine = d2bNode.d2bDaemonNode { extra = { pkgs, ... }: { environment.systemPackages = [ pkgs.jq ]; }; diff --git a/tests/host-integration/deferred/host-zone-gateway-isolation.nix b/tests/host-integration/deferred/host-zone-gateway-isolation.nix index 4f7a33497..6a83381d5 100644 --- a/tests/host-integration/deferred/host-zone-gateway-isolation.nix +++ b/tests/host-integration/deferred/host-zone-gateway-isolation.nix @@ -9,6 +9,10 @@ let hostToolBundle = if self.lib ? d2bHostToolBundle then self.lib.d2bHostToolBundle else null; }; + d2bNode = import ../../../nix/test-support/host-integration-node.nix { + inherit self; + inherit (pkgs) lib; + }; cloudHypervisorArtifact = d2bLib.mkRuntimeCloudHypervisorArtifact pkgs; volumeProviderArtifact = d2bLib.mkVolumeProviderArtifact pkgs; @@ -126,7 +130,7 @@ in pkgs.testers.runNixOSTest { name = "d2b-host-zone-gateway-isolation"; - nodes.machine = d2bLib.d2bCloudHypervisorNode { + nodes.machine = d2bNode.d2bCloudHypervisorNode { extra = { ... }: { environment.systemPackages = [ pkgs.iproute2 diff --git a/tests/host-integration/device-worker-launch.nix b/tests/host-integration/device-worker-launch.nix index 1641518b8..377ccc7fe 100644 --- a/tests/host-integration/device-worker-launch.nix +++ b/tests/host-integration/device-worker-launch.nix @@ -33,6 +33,12 @@ let inherit lib; inherit hostToolBundle; }; + # The reusable guest configuration lives outside this directory so it + # survives the fixture (see `nix/test-support/host-integration-node.nix`). + d2bNode = import ../../nix/test-support/host-integration-node.nix { + inherit self; + inherit lib; + }; cloudHypervisorArtifact = d2bLib.mkRuntimeCloudHypervisorArtifact pkgs; volumeProviderArtifact = d2bLib.mkVolumeProviderArtifact pkgs; @@ -279,7 +285,7 @@ in pkgs.testers.runNixOSTest { name = "d2b-device-worker-launch"; - nodes.machine = d2bLib.d2bDaemonNode { + nodes.machine = d2bNode.d2bDaemonNode { extra = { ... }: { d2b.site.adminUsers = [ "alice" ]; environment.systemPackages = with pkgs; [ diff --git a/tests/host-integration/lib.nix b/tests/host-integration/lib.nix index d60727d3a..ae9f426d8 100644 --- a/tests/host-integration/lib.nix +++ b/tests/host-integration/lib.nix @@ -1,10 +1,17 @@ -# Shared node configuration for d2b runNixOSTest (type-G) integration -# tests. These are the additive, real-kernel coverage layer: a runNixOSTest VM -# boots a real NixOS system with the d2b daemon surface -# (`d2b.daemonExperimental.enable`) and the test script asserts live broker -# / daemon behaviour (socket activation, SO_PEERCRED, the public.sock wire -# surface, audited host mutations) that the PR-tier fake-backed Rust canaries -# and pure-eval gates cannot exercise. +# Shared helpers for d2b runNixOSTest (type-G) integration tests. These are +# the additive, real-kernel coverage layer: a runNixOSTest VM boots a real +# NixOS system with the d2b daemon surface (`d2b.daemonExperimental.enable`) +# and the test script asserts live broker / daemon behaviour (socket +# activation, SO_PEERCRED, the public.sock wire surface, audited host +# mutations) that the PR-tier fake-backed Rust canaries and pure-eval gates +# cannot exercise. +# +# What stays here is the part bound to the test driver: the driver-coupled +# diagnostics prelude, the nested guest systems only these fixtures boot, and +# the provider artifacts only these fixtures install. The reusable guest +# configuration the lane also needs moved to +# `nix/test-support/host-integration-node.nix`, so a fixture can be removed +# when its check ports without taking its guest declaration with it. # # This file is NOT a flake check: the VM tests live under the `vmChecks` flake # output (selected explicitly by `make test-host-integration`), so the Layer-1 @@ -12,31 +19,6 @@ { self, lib, hostToolBundle ? null }: let - # The minimal, hermetic d2b site declaration every daemon-host node shares. - # Zone and Guest resources belong to the acceptance fixture that exercises - # them; keeping this base free of legacy VM/env authoring prevents unrelated - # host checks from silently materializing a second lifecycle graph. - daemonAcceptanceUnits = [ - "d2bd.service" - "d2b-broker.socket" - "d2b-broker.service" - ]; - - baseD2bConfig = { - d2b.site = { - waylandUser = "alice"; - launcherUsers = [ "alice" ]; - yubikey.enable = false; - usePrebuiltHostTools = false; - }; - # The daemon's v3 bundle always carries the local-root storage row. Keep - # the corresponding root Zone in these minimal host fixtures so the - # emitted topology is sealed and the daemon can enter Ready. - d2b.zones.local-root = { }; - # The full daemon + broker systemd surface under test. - d2b.daemonExperimental.enable = true; - }; - mkGuestSystem = { pkgs, name, zone ? "work", modules ? [ ] }: self.lib.evalGuest { @@ -507,149 +489,6 @@ let }; in rec { - # A NixOS module for a runNixOSTest node that boots the d2b daemon host. - # `extra` is merged as an additional module so individual tests can add - # per-test Zone/Guest resources, tampering helpers, or a larger disk. The - # node provisions the `alice` operator user the base config references. - # - # Structured as an attrset-module with everything in `imports` (an attrset is - # a valid module): `imports` must be top-level, NOT wrapped in `lib.mkMerge`, - # or the module system rejects it ("option nodes.machine.imports does not - # exist"). - d2bDaemonNode = - { extra ? { }, writableStore ? false }: - { config, pkgs, ... }: - let - # Dedicated state disk: the redb Zone store pays an fsync per commit - # on the emulated root disk, and bring-up write bursts stall the - # daemon writer thread ~700-900ms per write. Attaching /var/lib/d2b - # as its own virtio drive with cache=unsafe makes guest fsync a host - # page-cache no-op; the fixture VM is ephemeral, so the durability - # semantics that unsafe drops are irrelevant here. - stateDisk = pkgs.runCommand "d2b-state.img" - { - nativeBuildInputs = [ pkgs.e2fsprogs ]; - } - '' - truncate -s 4G "$out" - mkfs.ext4 -q -F "$out" - ''; - in - { - imports = [ - self.nixosModules.default - baseD2bConfig - extra - { - # Headroom for building/activating the bundle + daemon closure inside - # the VM; the default 1024 MiB is tight once the broker spawns - # runners. - virtualisation.memorySize = 3072; - virtualisation.diskSize = 8192; - # The daemon's redb writer thread, the daemon async runtime, the - # broker, and three controllers all need real CPU. The 1-vCPU - # default serializes them and starves every 250ms handshake. - virtualisation.cores = 3; - boot.kernelModules = [ "br_netfilter" "tun" "vhost_net" ]; - - users.users.alice = { - isNormalUser = true; - uid = 1000; - }; - - environment.etc."d2b/daemon-acceptance-units".text = - lib.concatStringsSep "\n" daemonAcceptanceUnits + "\n"; - - # Fail VM checks promptly when daemon startup is deterministically - # broken instead of spending the lane timeout in a restart loop. - systemd.services.d2bd.unitConfig = { - StartLimitIntervalSec = "30s"; - StartLimitBurst = 3; - }; - - # runNixOSTest runs first-boot activation before systemd-tmpfiles has - # materialized the d2b state tree. Pre-create the state directory so - # daemon-owned startup can rely on the same path ordering. - system.activationScripts.d2bTestStateDirs = { - deps = [ "users" ]; - text = '' - install -d -m 0750 -o root -g d2bd /var/lib/d2b - install -d -m 0710 -o root -g d2b /var/lib/d2b/keys - : > /var/lib/d2b/keys/.lock - chown root:root /var/lib/d2b/keys/.lock - chmod 0600 /var/lib/d2b/keys/.lock - ''; - }; - system.stateVersion = "25.11"; - } - # Opt-in writable same-fs store. ONLY needed by tests that drive the - # per-VM /nix/store hardlink farm (which requires /var/lib/d2b and - # /nix/store on the SAME filesystem - hardlinks can't cross FS - and the - # default runNixOSTest read-only store image splits them). It is OFF by - # default: `virtualisation.writableStore = true` copies the entire guest - # closure into a writable overlay at boot, which adds many minutes to - # (and can hang) VM startup. The daemon/broker activation + host-posture - # tests (daemon-smoke, bridge-isolation, privilege-oracle) - # never boot a microVM, so they never touch the farm - keep this off for - # a fast, reliable boot. - (lib.mkIf writableStore { - virtualisation.useBootLoader = true; - # The guest store-view hardlinks /nix/store into /var/lib/d2b, so - # both must stay on one filesystem - a separate state disk would - # break the hardlink farm with EXDEV. Instead, drop the root - # drive's cache to unsafe: every redb commit's fsync becomes a - # host page-cache no-op instead of a ~700-900ms stall, and the - # fixture VM is ephemeral, so the lost durability is irrelevant. - virtualisation.qemu.drives = lib.mkForce [ - { - name = "root"; - file = ''"$NIX_DISK_IMAGE"''; - driveExtraOpts.cache = "unsafe"; - driveExtraOpts.werror = "report"; - deviceExtraOpts.bootindex = "1"; - deviceExtraOpts.serial = "root"; - } - ]; - }) - # The state disk keeps /var/lib/d2b off the emulated root disk: - # cache=unsafe (host fsync no-op), noatime + nobarrier mounts. - # The writableStore hardlink-farm tests stay on the default - # same-fs layout, so this is opt-out for them. - (lib.mkIf (! writableStore) { - # The image is a `pkgs.runCommand` output, so QEMU must not need - # write access to it: the lane builds these checks inside the Nix - # sandbox, where /nix/store is mounted read-only, and a writable - # drive on a store path makes QEMU abort at machine start - the - # test driver surfaces that as a bare "Connection reset by peer". - # `snapshot=on` opens the backing file read-only and keeps every - # guest write in an ephemeral per-VM overlay under TMPDIR, which - # matches the fixture's ephemeral state disk either way. - virtualisation.qemu.options = [ - "-drive" - "file=${stateDisk},format=raw,if=virtio,cache=unsafe,aio=threads,snapshot=on" - ]; - fileSystems."/var/lib/d2b" = { - device = "/dev/vdb"; - fsType = "ext4"; - options = [ "noatime" "nobarrier" ]; - # Up before activation so d2bTestStateDirs lands inside the - # mounted filesystem, not under the covered root mountpoint. - neededForBoot = true; - }; - }) - ]; - }; - - # Shared host posture for every fixture that boots a Cloud Hypervisor Guest. - # The hardlink-backed Guest store view requires a writable host store on the - # same filesystem as /var/lib/d2b. - d2bCloudHypervisorNode = - { extra ? { } }: - d2bDaemonNode { - inherit extra; - writableStore = true; - }; - # Fixture diagnostics prelude, interpolated at the top of each VM fixture's # `testScript` (issue #513). The runNixOSTest driver discards # `machine.execute` output and never re-prints what a timed-out @@ -798,6 +637,6 @@ rec { ''; # Re-exported so tests can assert against the shared declaration. - inherit baseD2bConfig mkGuestSystem mkRuntimeCloudHypervisorArtifact + inherit mkGuestSystem mkRuntimeCloudHypervisorArtifact mkAcceptanceProviderArtifact mkVolumeProviderArtifact; } diff --git a/tests/host-integration/privilege-oracle.nix b/tests/host-integration/privilege-oracle.nix index 979205b6e..643b218d3 100644 --- a/tests/host-integration/privilege-oracle.nix +++ b/tests/host-integration/privilege-oracle.nix @@ -11,11 +11,17 @@ let inherit self; inherit (pkgs) lib; }; + # The reusable guest configuration lives outside this directory so it + # survives the fixture (see `nix/test-support/host-integration-node.nix`). + d2bNode = import ../../nix/test-support/host-integration-node.nix { + inherit self; + inherit (pkgs) lib; + }; in pkgs.testers.runNixOSTest { name = "d2b-privilege-oracle"; - nodes.machine = d2bLib.d2bDaemonNode { }; + nodes.machine = d2bNode.d2bDaemonNode { }; testScript = '' ${d2bLib.fixtureDiagnostics} diff --git a/tests/host-integration/resource-operator-activation.nix b/tests/host-integration/resource-operator-activation.nix index 7190aec3a..3c2f9f69b 100644 --- a/tests/host-integration/resource-operator-activation.nix +++ b/tests/host-integration/resource-operator-activation.nix @@ -15,6 +15,12 @@ let hostToolBundle = if self.lib ? d2bHostToolBundle then self.lib.d2bHostToolBundle else null; }; + # The reusable guest configuration lives outside this directory so it + # survives the fixture (see `nix/test-support/host-integration-node.nix`). + d2bNode = import ../../nix/test-support/host-integration-node.nix { + inherit self; + inherit lib; + }; providerArtifact = d2bLib.mkAcceptanceProviderArtifact pkgs; acceptancePublisherKey = providerArtifact.trustedPublisher.signingKey; artifacts = { @@ -33,7 +39,7 @@ in pkgs.testers.runNixOSTest { name = "d2b-resource-operator-activation"; - nodes.machine = d2bLib.d2bDaemonNode { + nodes.machine = d2bNode.d2bDaemonNode { extra = { ... }: { networking.nftables.enable = true; networking.nftables.ruleset = lib.mkAfter '' diff --git a/tests/host-integration/runtime-cloud-hypervisor-guest-preflight.nix b/tests/host-integration/runtime-cloud-hypervisor-guest-preflight.nix index 4a2d3e1f1..72fa82726 100644 --- a/tests/host-integration/runtime-cloud-hypervisor-guest-preflight.nix +++ b/tests/host-integration/runtime-cloud-hypervisor-guest-preflight.nix @@ -13,6 +13,12 @@ let hostToolBundle = if self.lib ? d2bHostToolBundle then self.lib.d2bHostToolBundle else null; }; + # The reusable guest configuration lives outside this directory so it + # survives the fixture (see `nix/test-support/host-integration-node.nix`). + d2bNode = import ../../nix/test-support/host-integration-node.nix { + inherit self; + inherit lib; + }; cloudHypervisorArtifact = d2bLib.mkRuntimeCloudHypervisorArtifact pkgs; volumeProviderArtifact = d2bLib.mkVolumeProviderArtifact pkgs; @@ -187,7 +193,7 @@ in pkgs.testers.runNixOSTest { name = "d2b-runtime-cloud-hypervisor-guest-preflight"; - nodes.machine = d2bLib.d2bCloudHypervisorNode { + nodes.machine = d2bNode.d2bCloudHypervisorNode { extra = { ... }: { d2b.site.adminUsers = [ "alice" ]; environment.systemPackages = with pkgs; [ diff --git a/tests/host-integration/state-posture-contract.nix b/tests/host-integration/state-posture-contract.nix index bce65960d..4d4abe7ca 100644 --- a/tests/host-integration/state-posture-contract.nix +++ b/tests/host-integration/state-posture-contract.nix @@ -19,6 +19,12 @@ let hostToolBundle = if self.lib ? d2bHostToolBundle then self.lib.d2bHostToolBundle else null; }; + # The reusable guest configuration lives outside this directory so it + # survives the fixture (see `nix/test-support/host-integration-node.nix`). + d2bNode = import ../../nix/test-support/host-integration-node.nix { + inherit self; + inherit lib; + }; cloudHypervisorArtifact = d2bLib.mkRuntimeCloudHypervisorArtifact pkgs; volumeProviderArtifact = d2bLib.mkVolumeProviderArtifact pkgs; @@ -193,7 +199,7 @@ in pkgs.testers.runNixOSTest { name = "d2b-state-posture-contract"; - nodes.machine = d2bLib.d2bCloudHypervisorNode { + nodes.machine = d2bNode.d2bCloudHypervisorNode { extra = { ... }: { d2b.site.adminUsers = [ "alice" ]; environment.systemPackages = with pkgs; [ diff --git a/tests/host-integration/virtiofsd-volume-runtime.nix b/tests/host-integration/virtiofsd-volume-runtime.nix index 20a89a93b..c188d9da5 100644 --- a/tests/host-integration/virtiofsd-volume-runtime.nix +++ b/tests/host-integration/virtiofsd-volume-runtime.nix @@ -20,6 +20,12 @@ let hostToolBundle = if self.lib ? d2bHostToolBundle then self.lib.d2bHostToolBundle else null; }; + # The reusable guest configuration lives outside this directory so it + # survives the fixture (see `nix/test-support/host-integration-node.nix`). + d2bNode = import ../../nix/test-support/host-integration-node.nix { + inherit self; + inherit lib; + }; volumeProviderArtifact = d2bLib.mkVolumeProviderArtifact pkgs; artifacts = { volume-acceptance-provider = { @@ -37,7 +43,7 @@ in pkgs.testers.runNixOSTest { name = "d2b-virtiofsd-volume-runtime"; - nodes.machine = d2bLib.d2bDaemonNode { + nodes.machine = d2bNode.d2bDaemonNode { extra = { ... }: { networking.nftables.enable = true; networking.nftables.ruleset = lib.mkAfter '' From dbaca897b6f51bb89a4bc2d6262f112ec90c213b Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sat, 26 Sep 2026 12:33:18 -0700 Subject: [PATCH 04/51] feat(vm): boot the host-integration guest from a Bazel-owned harness Adds a harness crate that reproduces each check's emulator invocation from the evaluated node configuration, enforces the host virtualization preconditions before anything is copied or spawned, and refuses at boot any writable device that cannot carry an internal snapshot. The lane suite is its own, and the crate deliberately holds no test aggregate so the repository's test census does not force it into the main package suite. The image action now evaluates both node shapes and emits a fully resolved manifest, so the invocation is read off the evaluated config rather than restated anywhere. Five defects were found by running the guest, not by reading it. The node's device-option list is shell text that only the module's run script word-splits; the monitor socket path exceeded the kernel's sun_path limit on the long runfiles path; the copied root disk inherited read-only mode from the read-only graph output; the image stopped producing the kernel and initrd its own manifest names, which an already-passing unit test had been asserting; and the guest activation script used $$ expansions, which a Nix indented string leaves as literal double dollars, so the unit polled a service named after its own pid. The activation stall itself was the lane dropping the node's declared networking, leaving the guest without virtio_net, which the daemon refuses to start without. Monitor and spawn failures now carry the emulator's stderr and the assembled command line, and an activation timeout carries the guest's systemd ordering state. Each of those is what made the next defect findable. --- BUILD.bazel | 1 + Cargo.lock | 8 + Cargo.toml | 1 + bazel/checks/vm/BUILD.bazel | 108 +- bazel/checks/vm/defs.bzl | 124 +- changelog.d/bazel-owned-guest-image.md | 4 + changelog.d/bazel-owned-vm-harness.md | 24 + flake.nix | 10 +- nix/test-support/guest-image.nix | 442 +++++- packages/d2b-vm-harness/BUILD.bazel | 61 + packages/d2b-vm-harness/Cargo.toml | 32 + .../d2b-vm-harness/src/bin/d2b-vm-harness.rs | 255 ++++ packages/d2b-vm-harness/src/error.rs | 158 +++ packages/d2b-vm-harness/src/guest.rs | 1264 +++++++++++++++++ packages/d2b-vm-harness/src/host.rs | 296 ++++ packages/d2b-vm-harness/src/lib.rs | 28 + packages/d2b-vm-harness/src/manifest.rs | 291 ++++ packages/d2b-vm-harness/src/monitor.rs | 315 ++++ packages/xtask/src/provider_crate_policy.rs | 2 + 19 files changed, 3366 insertions(+), 58 deletions(-) create mode 100644 changelog.d/bazel-owned-vm-harness.md create mode 100644 packages/d2b-vm-harness/BUILD.bazel create mode 100644 packages/d2b-vm-harness/Cargo.toml create mode 100644 packages/d2b-vm-harness/src/bin/d2b-vm-harness.rs create mode 100644 packages/d2b-vm-harness/src/error.rs create mode 100644 packages/d2b-vm-harness/src/guest.rs create mode 100644 packages/d2b-vm-harness/src/host.rs create mode 100644 packages/d2b-vm-harness/src/lib.rs create mode 100644 packages/d2b-vm-harness/src/manifest.rs create mode 100644 packages/d2b-vm-harness/src/monitor.rs diff --git a/BUILD.bazel b/BUILD.bazel index 8c236739b..1171e2051 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -343,6 +343,7 @@ filegroup( "//packages/d2b-provider-host:cargo_workspace_sources", "//packages/d2b-provider-user:cargo_workspace_sources", "//packages/d2b-provider-test-controller:cargo_workspace_sources", + "//packages/d2b-vm-harness:cargo_workspace_sources", "//packages/d2b-resource-runtime:cargo_workspace_sources", "//packages/d2b-audit:BUILD.bazel", "//packages/d2b-broker-composition:BUILD.bazel", diff --git a/Cargo.lock b/Cargo.lock index dc85b6e18..6900ab9e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2173,6 +2173,14 @@ dependencies = [ "zbus", ] +[[package]] +name = "d2b-vm-harness" +version = "0.0.0-bootstrap" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "d2b-zone-routing" version = "0.0.0-bootstrap" diff --git a/Cargo.toml b/Cargo.toml index ddb693873..0ca9e6e6d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -95,6 +95,7 @@ members = [ "packages/d2b-provider-command", "packages/d2b-provider-operation", "packages/d2b-provider-seccomp-profile", + "packages/d2b-vm-harness", ] [workspace.package] diff --git a/bazel/checks/vm/BUILD.bazel b/bazel/checks/vm/BUILD.bazel index 9ae1093ba..ee3d02a05 100644 --- a/bazel/checks/vm/BUILD.bazel +++ b/bazel/checks/vm/BUILD.bazel @@ -1,4 +1,4 @@ -load(":defs.bzl", "guest_image") +load(":defs.bzl", "guest_boot_test", "guest_image") package(default_visibility = ["//visibility:public"]) @@ -21,30 +21,100 @@ _GUEST_SOURCES = [ "//tests/fixtures:fixture_sources", ] +# The d2b host binaries every guest image is built against. +_HOST_TOOLS = [ + "//packages/d2b:d2b", + "//packages/d2bd:d2bd", + "//packages/d2b-broker-composition:d2b-broker", + "//packages/d2b-host:d2b-activation-helper", + "//packages/d2b-host-activation-helper:d2b-host-activation-helper", + "//packages/d2b-unsafe-local-helper:d2b-unsafe-local-helper", + "//packages/d2b-resource-compiler:d2b-resource-compiler", + "//packages/d2b-provider-display-wayland:d2b-wayland-proxy", + "//packages/d2b-provider-test-controller:d2b-provider-test-controller", +] + +# The Cloud Hypervisor controller the nested-guest checks enroll. +_CONTROLLER = "//packages/d2b-provider-guest-cloud-hypervisor:d2b-cloud-hypervisor-controller" + +# The image action realizes a guest closure through nix, which cannot build +# inside a sandboxed action, so the action is unsandboxed, local, and +# uncacheable on any remote cache. +_IMAGE_TAGS = [ + "exclusive", + "local", + "no-remote-cache", + "no-remote-exec", + "no-sandbox", +] + +# The default guest shape: the node the daemon and broker host checks boot. guest_image( name = "guest_image", srcs = _GUEST_SOURCES, - cloud_hypervisor_controller = "//packages/d2b-provider-guest-cloud-hypervisor:d2b-cloud-hypervisor-controller", + cloud_hypervisor_controller = _CONTROLLER, flake = "//:flake.nix", flake_lock = "//:flake.lock", - host_tools = [ - "//packages/d2b:d2b", - "//packages/d2bd:d2bd", - "//packages/d2b-broker-composition:d2b-broker", - "//packages/d2b-host:d2b-activation-helper", - "//packages/d2b-host-activation-helper:d2b-host-activation-helper", - "//packages/d2b-unsafe-local-helper:d2b-unsafe-local-helper", - "//packages/d2b-resource-compiler:d2b-resource-compiler", - "//packages/d2b-provider-display-wayland:d2b-wayland-proxy", - "//packages/d2b-provider-test-controller:d2b-provider-test-controller", - ], + host_tools = _HOST_TOOLS, + nix = "@nix//:bin/nix", + node_shape = "daemon", + substituters = "https://cache.nixos.org/", + tags = _IMAGE_TAGS, +) + +# The writable-store shape: the node the two nested-guest checks boot, which +# replaces the root drive with a writable overlay on an installed system +# image and boots through a bootloader. It is a second graph output rather +# than a runtime switch, so a guest is always built from the shape its +# configuration declared. +guest_image( + name = "guest_image_writable_store", + srcs = _GUEST_SOURCES, + cloud_hypervisor_controller = _CONTROLLER, + flake = "//:flake.nix", + flake_lock = "//:flake.lock", + host_tools = _HOST_TOOLS, nix = "@nix//:bin/nix", + node_shape = "writable-store", substituters = "https://cache.nixos.org/", - tags = [ - "exclusive", - "local", - "no-remote-cache", - "no-remote-exec", - "no-sandbox", + tags = _IMAGE_TAGS, +) + +# The emulator every lane guest boots, taken from the same pinned nix +# package set the guest closure is realized from, so the emulator and the +# guest are at one nixpkgs revision and a snapshot taken by one version is +# never assumed restorable by another. +_EMULATOR = "@qemu_kvm//:bin/qemu-kvm" + +# The lane's harness, which is the thing that spawns a guest. +_HARNESS = "//packages/d2b-vm-harness:d2b-vm-harness" + +guest_boot_test( + name = "guest_boot_daemon", + emulator = _EMULATOR, + harness = _HARNESS, + image = ":guest_image", +) + +guest_boot_test( + name = "guest_boot_writable_store", + emulator = _EMULATOR, + harness = _HARNESS, + image = ":guest_image_writable_store", +) + +# The lane's own suite. The harness crate deliberately carries no test +# aggregate - the repository's test census would force-register such a crate +# into the main package suite - so the targets that lint and exercise it are +# named here instead, and the crate is still covered by the repository's +# Rust gates. +test_suite( + name = "host_integration_lane", + tests = [ + "//packages/d2b-vm-harness:d2b-vm-harness_clippy", + "//packages/d2b-vm-harness:d2b_vm_harness_clippy", + "//packages/d2b-vm-harness:d2b_vm_harness_test", + ":guest_boot_daemon", + ":guest_boot_writable_store", ], ) diff --git a/bazel/checks/vm/defs.bzl b/bazel/checks/vm/defs.bzl index 18ba0d2a2..da4de5652 100644 --- a/bazel/checks/vm/defs.bzl +++ b/bazel/checks/vm/defs.bzl @@ -16,9 +16,12 @@ from, and the build-users setting are all set here, and the flake arrives as a copied input rather than as a working-tree reference. """ +load("@bazel_skylib//rules:native_binary.bzl", "native_test") + _GUEST_IMAGE_COMMAND = """\ set -eu label="%s" +node_shape="%s" # Resolved before the fixed PATH is installed: on NixOS the setuid sudo is # the wrapper, and the profile entry is not. @@ -123,7 +126,7 @@ done [ "$reachable" -eq 1 ] || fail "none of the declared substituters ($substituters) is reachable" system="$("$nix_bin" eval --raw --impure --expr builtins.currentSystem)" || fail "could not read the nix system" -expr="(builtins.getFlake \\"path:$root\\").guestImage.\\"$system\\" { rawBundle = \\"$bundle\\"; rawCloudHypervisorController = $controller_argument; }" +expr="(builtins.getFlake \\"path:$root\\").guestImage.\\"$system\\" { rawBundle = \\"$bundle\\"; rawCloudHypervisorController = $controller_argument; nodeShape = \\"$node_shape\\"; }" echo "guest-image $label: realizing the guest from declared inputs" >&2 image="$(nix_run "$nix_bin" build \\ --option store "local?store=$store_dir" \\ @@ -168,7 +171,10 @@ def _guest_image_impl(ctx): ctx.attr.substituters, controller.path if controller else "", ] + [tool.path for tool in ctx.files.host_tools], - command = _GUEST_IMAGE_COMMAND % {"label": str(ctx.label)}, + command = _GUEST_IMAGE_COMMAND % ( + str(ctx.label), + ctx.attr.node_shape, + ), mnemonic = "GuestImage", progress_message = "Building d2b guest image %s" % ctx.label.name, ) @@ -189,6 +195,16 @@ guest_image = rule( allow_single_file = True, mandatory = True, ), + # Which of the re-homed node's two guest shapes this image evaluates: + # `daemon` for the daemon/broker host checks, `writable-store` for + # the checks that boot a nested guest, which replaces the root drive + # and boots through a bootloader. It is a declared attribute rather + # than a lane constant, so the shape a guest is built from is part of + # the action's key rather than an ambient fact. + "node_shape": attr.string( + default = "daemon", + values = ["daemon", "writable-store"], + ), # The guest's own binaries, taken in the target configuration: the # guest runs the binaries this build produces, not a second copy # built for the action's own execution platform. @@ -212,3 +228,107 @@ guest_image = rule( ), }, ) + + + +_LANE_RUNNER_SCRIPT = """\ +#!/bin/sh +set -eu + +# A test runs with its working directory inside the runfiles tree, not at its +# root, so the root is derived from this script's own location rather than +# assumed - a wrong guess here is a guest that never boots, with a path error +# instead of a boot error. +runfiles="$(CDPATH= cd -- "$(dirname -- "$0")/../../../.." && pwd -P)" + +export D2B_VM_HARNESS_CYCLES="{cycles}" +export D2B_VM_HARNESS_EMULATOR="$runfiles/{emulator}" +export D2B_VM_HARNESS_IMAGE="$runfiles/{image}" +# A lane-scoped working directory that outlives each individual guest, and is +# this test's own rather than the sandboxed temporary directory the current +# Bazel release does not expose to a sandboxed action. The harness resolves a +# relative one against its own working directory. +export D2B_VM_HARNESS_WORK_ROOT="{work_root}" + +exec "$runfiles/{harness}" "$@" +""" + +_LANE_TAGS = [ + "exclusive", + "local", + "no-remote-cache", + "no-remote-exec", + "no-sandbox", +] + +def guest_boot_test(name, image, emulator, harness, timeout = "eternal"): + """Boot one guest shape, wait for its activation contract, and tear it down. + + The guest is booted by the lane's own harness, against the image the + guest-image action produced from the re-homed guest node. The emulator + arrives as a declared runfile from the pinned nix package set, so the + emulator and the guest closure are at one nixpkgs revision. + + The image, the emulator, and the lane's working directory are handed to + the harness through a generated runner rather than through the test + rule's `env`: a runfile location is only expanded where a rule expands + it, and the harness is a binary, not a shell script that could resolve + its own runfiles. + + The target runs unsandboxed and local: a guest opens `/dev/kvm`, may run + a nested guest, and owns a working directory that outlives an individual + check, none of which a sandboxed action can provide. The tags keep it off + remote execution and out of any aggregate that would replay a guest's + verdict. + """ + runner = name + "_runner.sh" + script = _LANE_RUNNER_SCRIPT.format( + cycles = "2", + emulator = "$(rlocationpath %s)" % emulator, + harness = "$(rlocationpath %s)" % harness, + image = "$(rlocationpath %s)" % image, + work_root = "d2b-vm-lane-work/%s" % name, + ) + + # The runner is written through a genrule rather than handed to the test + # rule's `env`, following `nix_native_test`: a runfile location is + # expanded only where a rule expands it, and a binary cannot resolve its + # own runfiles. Each location becomes a placeholder first so the `$` + # escaping genrule's own expansion needs does not touch the shell script + # around it. + make_vars = { + "$(rlocationpath %s)" % harness: "__HARNESS__", + "$(rlocationpath %s)" % emulator: "__EMULATOR__", + "$(rlocationpath %s)" % image: "__IMAGE__", + } + for make_var, placeholder in make_vars.items(): + script = script.replace(make_var, placeholder) + script = script.replace("$", "$$") + for make_var, placeholder in make_vars.items(): + script = script.replace(placeholder, make_var) + + native.genrule( + name = runner, + srcs = [ + emulator, + harness, + image, + ], + outs = [name + "_runner"], + cmd = "\"$(execpath @python3//:bin/python3)\" -c 'import pathlib,sys; p=pathlib.Path(sys.argv[1]); p.write_text(sys.stdin.read()); p.chmod(0o755)' \"$(OUTS)\" <<'EOF'\n%s\nEOF" % script, + tags = _LANE_TAGS, + tools = ["@python3//:bin/python3"], + ) + native_test( + name = name, + src = ":" + name + "_runner", + data = [ + ":" + name + "_runner", + harness, + image, + emulator, + ], + size = "large", + tags = _LANE_TAGS, + timeout = timeout, + ) diff --git a/changelog.d/bazel-owned-guest-image.md b/changelog.d/bazel-owned-guest-image.md index 31bbe32a6..7efd53d6f 100644 --- a/changelog.d/bazel-owned-guest-image.md +++ b/changelog.d/bazel-owned-guest-image.md @@ -12,3 +12,7 @@ ### Removed - Retired the recipe's Attic closure upload from the new guest-image path: a network side effect would make the image uncacheable, and the build cache the image lands in replaces it. The upload stays with the nix recipe and retires with the lane, which is recorded here rather than left for the next reader to rediscover. + +### Fixed + +- The image carries the kernel and initrd its manifest names. The rewrite that split the action across the two node shapes dropped the two copies, so the manifest declared artifacts relative to the image root that the image did not contain, and the emulator refused the launch on the missing kernel - which surfaces as `Connection reset by peer` on the monitor, because the monitor socket is created before the kernel is opened. The kernel and initrd are copied in from the paths the NixOS VM module's own run script names: the toplevel's `kernel` link, and `virtualisation.directBoot.initrd`, so a node that redirects the payload redirects it here too. An image whose kernel or initrd cannot be copied now fails to build rather than producing an image that cannot be booted. diff --git a/changelog.d/bazel-owned-vm-harness.md b/changelog.d/bazel-owned-vm-harness.md new file mode 100644 index 000000000..97c8fb9aa --- /dev/null +++ b/changelog.d/bazel-owned-vm-harness.md @@ -0,0 +1,24 @@ +### Added + +- Added `packages/d2b-vm-harness`, the Bazel-owned host-integration lane's guest launcher. It reproduces the emulator invocation a check's guest configuration declared - memory, vCPU count, the drive layout including the writable-store root drive, and per-check device options such as a vsock device - reads that invocation off the guest image's manifest rather than off a uniform shape of its own, boots the guest with hardware virtualization selected explicitly, waits for the guest's own activation contract on its serial console, and tears it down. A guest that never activates fails the lane inside a bound with the console tail attached, rather than hanging. +- The lane asserts the host's virtualization capabilities before it boots anything, and names every one that is missing: `/dev/kvm`, nested virtualization, and nested-state save support. The lane does not fall back to emulation, so a host that cannot run it stops with a message naming the knob to turn rather than running the suite several times slower without saying so. +- The lane verifies at boot that every writable device the guest attached can carry an internal snapshot, and refuses the guest when one cannot - before any check runs, rather than at the first restore after a suite has already executed against a guest that could not be rolled back. +- Every reusable-pool guest carries a machine identity device and a hardware random source, and the guest's activation marker is written only once its random pool is initialised, so a snapshot can never be taken of a guest whose `getrandom()` would block after a restore. +- The lane's guest targets take the emulator from the pinned nix package set as a declared runfile, so the emulator and the guest closure are at one nixpkgs revision. +- The guest image action evaluates both guest shapes the re-homed node declares. `//bazel/checks/vm:guest_image` builds the daemon shape; `//bazel/checks/vm:guest_image_writable_store` builds the writable-store shape the two nested-guest checks boot, which replaces the root drive with a writable overlay on an installed system image and boots through a bootloader. The shape is a declared attribute of the action rather than a lane constant, and each image's manifest now carries the resolved boot method, drive layout, shared directories, and activation contract the launcher reads. +- Added `//bazel/checks/vm:host_integration_lane`, the lane's own suite, which names the harness's clippy and test targets directly. The harness crate deliberately carries no test aggregate, so the repository's test census does not force-register it into the main package suite while the repository's Rust gates still cover it. +- A guest that stalls now describes the ordering state it stalled in, on its own console and in the launcher's activation failure. A unit ordered after one that never started never enters `failed` at all, so the boot's failed set can be empty while the unit the activation contract is waiting for sits in `activating` behind an ordering dependency nobody has named. The guest therefore reports the d2b slice of its unit list, the boot's failed set, and every unit the contract waits on together with each unit it is ordered against, required by, or waiting on - and the report is written on a stall timer that fires well inside the launcher's own bound, because a report written after the launcher has stopped reading the console is not a report. The launcher lifts that report out of the console and prints it ahead of the console tail, so reading a failed lane no longer takes a twenty-minute run to reproduce. + +### Changed + + +- The lane's launcher renders the guest node's device options the way the NixOS VM module's own run script consumes them: the option list is shell text, so a single list entry that names a flag and its value (`-device virtio-keyboard`) is word-split into two arguments before it reaches the emulator. Handing the list to `execvp` element by element makes the emulator reject the entry with `-device virtio-keyboard: invalid option`, and the failure names the device rather than the launcher. This was diagnosed by bisecting eleven argument groups against the pinned emulator and finding none of them at fault, including `-device virtio-keyboard` on its own, which the same emulator accepts. Recorded here so that bisect is not repeated. +- The guest image's manifest is now the single record of a guest's invocation. The VM module's direct-boot directives are resolved into it - the kernel command line the module would have assembled from a shell substitution is written out resolved - and the per-check device options are carried through verbatim, so a check's own contribution rides to the emulator exactly as its node declared it. +- A failure to read the emulator's monitor now carries the emulator's own stderr and the assembled command line, exactly as a launch that could not be spawned does. The emulator creates its monitor socket before it has finished acting on the rest of the command line, so a guest it dies on later accepts the connection and then resets it: a reset that names the socket and nothing about the option that caused it. The reason is only on the emulator's stderr, so the lane reads it there rather than leaving a contributor to bisect the command line. +- A block device the emulator names by its qdev path rather than by a drive id is named by that path in a snapshot-capability refusal. Such an entry carries an empty `device` key rather than omitting it, so reading the key literally named the refused device with the empty string. + +### Fixed + +- A guest booted by the lane now gets the networking its configuration declared. The NixOS VM module renders `virtualisation.qemu.networkingOptions` in its own run script as a group separate from `virtualisation.qemu.options`, and the launcher carried only the option list - so every lane guest booted with no network device at all. The kernel never loaded the module that device needs, and the daemon refused to start over exactly that: `host-kernel-modules-missing: daemon refused to start: required kernel modules not loaded: virtio_net`, after which `d2bd.service` hit its start limit and the guest sat at a login prompt that looked healthy. The networking options are now resolved into the manifest the same way the kernel command line's own shell substitution is, and the launcher renders them where the module's run script does. +- The guest's activation script aborted on its first statement. Its deadline was written `$(( $(date +%s) + 1800s ))`: the systemd time span above it carries an `s` because that is a duration, and shell arithmetic does not, so the shell read `1800s` as a base it could not parse and - under `set -e` - the script died before announcing anything. The deadline is plain seconds now, and the comment that got it wrong is gone with it. +- Activation is no longer lost to a login prompt sharing the serial console. A getty on the same tty emits terminal queries - cursor position, erase display - and lands them mid-line with no newline of its own, so the guest's activation marker could end up in the middle of a line whose first byte was an escape sequence. A launcher that looks for the marker only at the start of a line then waits out its whole bound on a guest that activated within a minute, and the failure it reports contradicts the console it attaches. The guest now opens every line it writes, the marker included, with a newline, and the launcher matches the marker anywhere in the console rather than depending on line structure a shared byte stream does not guarantee. diff --git a/flake.nix b/flake.nix index 894378a41..a5c0c12bd 100644 --- a/flake.nix +++ b/flake.nix @@ -712,14 +712,18 @@ # # `rawBundle` and `rawCloudHypervisorController` are paths to # directories holding the binaries by output name. `extraModules` - # carries a check's own guest contributions. + # carries a check's own guest contributions. `nodeShape` names which + # of the re-homed node's two guest shapes to evaluate - `daemon` for + # the daemon/broker host checks, `writable-store` for the two that + # boot a nested guest - so the shape is a declared input of the image + # rather than a second record of the numbers behind it. guestImage = forAllSystems (system: - { rawBundle, rawCloudHypervisorController ? null, extraModules ? [ ] }: + { rawBundle, rawCloudHypervisorController ? null, extraModules ? [ ], nodeShape ? "daemon" }: let handoff = mkBazelHostTools system rawBundle rawCloudHypervisorController; in import ./nix/test-support/guest-image.nix { - inherit extraModules rawBundle; + inherit extraModules nodeShape rawBundle; pkgs = nixpkgsFor.${system}; bazelHostTools = handoff.tools; self = handoff.hostSelf; diff --git a/nix/test-support/guest-image.nix b/nix/test-support/guest-image.nix index 6cca3f209..3dcee49d5 100644 --- a/nix/test-support/guest-image.nix +++ b/nix/test-support/guest-image.nix @@ -8,21 +8,49 @@ # the same contract the legacy `D2B_HOST_TOOL_BUNDLE` handoff passes, and # the caller content-addresses them. # -# The guest is the node the current `vmChecks` fixtures boot, taken from -# the same `d2bDaemonNode` configuration and the same Bazel-built host-tool -# package, so a guest realized through this entry point and a guest -# realized through the legacy handoff are the same guest. Per-check module +# Two guest shapes are declared here, and the shape is a declared input +# rather than a second copy of the numbers: `daemon` is the node the current +# `vmChecks` fixtures boot for the daemon/broker host checks, and +# `writable-store` is the node the two nested-guest checks boot, which +# replaces the root drive and boots through a bootloader. Per-check module # contributions arrive through `extraModules`. # -# The output is one store path holding the guest's system closure and a -# manifest of what a launcher needs to boot it: the toplevel, the kernel -# and initrd entry points, the declared invocation shape, and the host-tool -# package the closure was built against. -{ pkgs, self, bazelHostTools, rawBundle, extraModules ? [ ] }: +# The output is one store path holding the guest's system closure, the root +# disk in the shape's own format, and a manifest of what a launcher needs to +# boot it: the machine size, the drive layout, the boot method with its +# kernel command line resolved, the per-check device options, and the +# activation contract the launcher waits for on the guest's serial console. +# The manifest is the only record of the invocation shape: the launcher +# renders it and never restates a number the re-homed node declared. +{ pkgs, self, bazelHostTools, rawBundle, extraModules ? [ ], nodeShape ? "daemon" }: let inherit (pkgs) lib; + # The lane's activation contract, in the repository's own field names: + # `nixos-modules/lib.nix` describes every service capability with a + # readiness signal and a contract that signal buys. The lane's guest + # declares the same two fields - `readiness = "console-marker"` and + # `contract = "d2b-daemon-acceptance"` - and the launcher waits for the + # marker the contract names, so "the guest activated" means here what it + # means everywhere else in the tree. + activationMarker = "D2B_LANE_READY"; + + # The guest-side bound on that wait. It is the backstop behind the + # launcher's own bounded wait, and is deliberately the longer of the two: + # the launcher fails the lane with the guest's console attached, which is + # a better failure than a unit that dies silently first. + activationTimeoutSeconds = 1800; + # How long a polled unit may sit not-active before the guest describes the + # ordering state it is sitting in, and how long after that it repeats + # itself. Both are well inside the launcher's own bound, which is the whole + # point: a report written after the launcher has stopped reading the + # console is not a report, and the launcher's bound is the shorter of the + # two. The repeat is deliberately wide, because a second copy of an + # unchanged ordering state is console noise rather than evidence. + activationStallSeconds = 90; + activationStallRepeatSeconds = 600; + # Refuse an incomplete handoff before a guest closure is evaluated. The # host-tool package repeats this check when it is built, but that is the # wrong place to learn a binary is missing: by then the guest closure has @@ -49,10 +77,17 @@ let system = pkgs.stdenv.hostPlatform.system; modules = [ (pkgs.path + "/nixos/modules/virtualisation/qemu-vm.nix") - (d2bNode.d2bDaemonNode { extra = { imports = extraModules; }; }) + # `d2bCloudHypervisorNode` is `d2bDaemonNode` with the writable store + # opted into, so the two shapes are one declaration read two ways and + # cannot drift apart. + (d2bNode.d2bDaemonNode { + extra = { imports = extraModules; }; + writableStore = nodeShape == "writable-store"; + }) { virtualisation.host.pkgs = pkgs; } + laneGuestModule ]; }; guest = evaluated.config; @@ -63,22 +98,338 @@ let # disk in the format the module's run script builds. Real files, not a # copy of the toplevel symlink tree, so the image is what the lane boots. diskSizeMib = guest.virtualisation.diskSize; + useBootLoader = guest.virtualisation.useBootLoader; + + # The two files the direct-boot shape is handed, named exactly as the VM + # module's own run script names them: the kernel through the toplevel's + # `kernel` link, and the initrd through `virtualisation.directBoot.initrd`, + # so a node that redirects the payload redirects it here too. The build + # below copies both into the image, because the manifest names them + # relative to the image root and the image is what the lane boots. + directBootKernel = "${toplevel}/kernel"; + directBootInitrd = guest.virtualisation.directBoot.initrd; + + # The registration file the guest's activation loads into its Nix + # database, and the console list the VM module appends to a direct-boot + # kernel command line. Both are read off the evaluated configuration + # rather than restated, so the manifest carries what the guest declared. + regInfo = guest.virtualisation.host.pkgs.closureInfo { + rootPaths = guest.virtualisation.additionalPaths; + }; + consoles = map (console: "console=${console}") guest.virtualisation.qemu.consoles; + + # The serial device the activation marker is written to, taken from the + # console the guest was configured to log to. + serialDevice = lib.head (lib.splitString "," (lib.head guest.virtualisation.qemu.consoles)); + + # The bootloader shape boots from an installed system image rather than + # from the empty ext4 image the direct-boot shape boots with, so its root + # drive is a writable overlay on that image. It is built the way the VM + # module builds its own: one MBR partition, a BIOS bootloader, no EFI + # variables - the layout `selectPartitionTableLayout` picks for a node + # that asks for a bootloader without asking for EFI. + bootableSystemImage = + import (pkgs.path + "/nixos/lib/make-disk-image.nix") { + inherit pkgs; + config = guest; + inherit lib; + additionalPaths = [ regInfo ]; + additionalSpace = "0M"; + copyChannel = false; + diskSize = "auto"; + format = "qcow2"; + installBootLoader = true; + label = "nixos"; + onlyNixStore = false; + partitionTableType = "legacy"; + touchEFIVars = false; + }; + + # The direct-boot directives the VM module contributes to + # `virtualisation.qemu.options` are re-declared under `boot` below with + # their shell substitutions resolved, so they are removed from the option + # list the launcher passes through verbatim. Everything a check + # contributes - the state disk, a vsock device - stays in that list, in + # its declared order. + directBootFlags = [ "-kernel" "-initrd" "-append" ]; + isDirectBootFlag = option: lib.any (flag: lib.hasPrefix "${flag} " option) directBootFlags; + extraOptions = builtins.filter (option: !(isDirectBootFlag option)) guest.virtualisation.qemu.options; + + # The networking the node declared. The VM module renders + # `virtualisation.qemu.networkingOptions` in its own run script as a + # separate group, alongside `qemu.options` rather than inside it, and the + # launcher has to carry them for the same reason it carries the option + # list: a guest booted without them has no network device at all, so the + # kernel never loads the module that device needs and the daemon refuses + # to start over exactly that. The one shell substitution in the default - + # `"$QEMU_NET_OPTS"`, which the module leaves for an outer wrapper to fill + # - is resolved to nothing here, the same way the kernel command line's + # substitution is, because the launcher does not run a shell around it. + networkingOptions = map + (option: builtins.replaceStrings [ ''"$QEMU_NET_OPTS"'' ] [ "" ] option) + guest.virtualisation.qemu.networkingOptions; + + # The root drive the launcher attaches. `virtualisation.qemu.drives` + # entries are module submodules, so each is projected onto the fields the + # launcher renders; `$NIX_DISK_IMAGE` is the VM module's placeholder for + # the disk this image builds, and becomes the image's own `disk.qcow2`. + driveFile = file: if file == ''"$NIX_DISK_IMAGE"'' then "disk.qcow2" else file; + drives = map + (drive: { + name = drive.name or null; + file = driveFile drive.file; + format = drive.driveExtraOpts.format or null; + cache = drive.driveExtraOpts.cache or "writeback"; + werror = drive.driveExtraOpts.werror or "report"; + bootIndex = drive.deviceExtraOpts.bootindex or null; + serial = drive.deviceExtraOpts.serial or null; + interface = guest.virtualisation.qemu.diskInterface; + }) + guest.virtualisation.qemu.drives; + + # The host directories the guest mounts over 9p. A `$TMPDIR`-relative + # source stays relative in the manifest and the launcher resolves it + # against the working directory it owns, exactly as the VM module's run + # script does. + sharedDirectories = lib.mapAttrsToList + (mountTag: share: { + inherit mountTag; + securityModel = share.securityModel; + target = share.target; + source = share.source; + }) + (lib.filterAttrs (_: share: share ? source) guest.virtualisation.sharedDirectories); + manifest = { + schemaVersion = 1; system = pkgs.stdenv.hostPlatform.system; - kernel = "kernel"; - initrd = "initrd"; - disk = "disk.qcow2"; - diskFormat = "qcow2"; + nodeShape = nodeShape; + image = { + disk = "disk.qcow2"; + diskFormat = "qcow2"; + inherit diskSizeMib; + # Only the bootloader shape carries a backing system image; the + # direct shape's root drive is self-contained. + systemImage = if useBootLoader then "${bootableSystemImage}/nixos.qcow2" else null; + }; + machine = { + cores = guest.virtualisation.cores; + memorySizeMib = guest.virtualisation.memorySize; + }; + boot = { + method = if useBootLoader then "bootloader" else "direct"; + kernel = if useBootLoader then null else "kernel"; + initrd = if useBootLoader then null else "initrd"; + append = if useBootLoader then + null + else + "${builtins.readFile "${toplevel}/kernel-params"} init=${toplevel}/init regInfo=${regInfo}/registration ${lib.concatStringsSep " " consoles}"; + }; + inherit drives extraOptions networkingOptions; + inherit sharedDirectories; + activation = { + readiness = "console-marker"; + contract = "d2b-daemon-acceptance"; + marker = activationMarker; + inherit serialDevice; + acceptanceUnitsFile = "/etc/d2b/daemon-acceptance-units"; + shape = nodeShape; + }; init = "${toplevel}/init"; - toplevel = toplevel; + inherit toplevel; hostToolBundle = bazelHostTools.package; hostToolInventory = bazelHostTools.inventory; cloudHypervisorController = bazelHostTools.cloudHypervisorControllerPackage; - inherit (guest.virtualisation) cores diskSize memorySize; - qemuOptions = guest.virtualisation.qemu.options; }; + + # The lane's own guest surface, layered on the re-homed node rather than + # merged into it: the legacy fixtures boot these same nodes through the + # nix test driver, which brings its own readiness, and a marker wired into + # the shared module would reach their guests too. + laneGuestModule = + { config, lib, pkgs, ... }: + { + # The direct-boot shape gets its serial console from the `-append` the + # VM module builds. The bootloader shape reads its command line off + # the disk instead, so the same console list is declared as kernel + # parameters for it - same consoles, same order, same primary. + boot.kernelParams = lib.mkIf config.virtualisation.useBootLoader ( + map (console: "console=${console}") config.virtualisation.qemu.consoles + ); + + systemd.services.d2b-lane-activation = { + description = "Report d2b host-integration lane guest activation"; + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + unitConfig = { + Type = "oneshot"; + RemainAfterExit = true; + }; + serviceConfig.TimeoutStartSec = "${toString activationTimeoutSeconds}s"; + path = [ + pkgs.coreutils + pkgs.gnugrep + pkgs.systemd + ]; + # The script is materialised by `writeShellScriptBin`, which embeds + # this text verbatim: a Nix indented string escapes `${` and `''`, + # and nothing else. A `$$` here is two literal dollar signs to the + # shell, so `$$unit` is the script's own process id followed by + # `unit` - a unit name that does not exist, polled forever. The + # expansions below are therefore single-dollar. + # + # Every step of the contract is announced on the serial console, not + # only the marker. The launcher's console is the only view it has of + # the guest, so a guest that never activates is described by what it + # last said it was waiting for - a unit name, or the unit's own + # journal - rather than by the absence of a marker. Every line this + # unit writes, the marker included, opens with a newline: the serial + # console is a byte stream a login prompt shares, and a getty on the + # same tty emits terminal queries - cursor position, erase display - + # with no newline of its own, which otherwise lands them on the front + # of this unit's next line. + script = '' + set -eu + say() { + printf '\nd2b-lane-activation: %s\n' "$1" >/dev/${serialDevice} + } + # A failed unit is reported at once, whoever it is, with its own + # journal and the boot's failed set, rather than at the deadline. + # The boot's failed set and not the polled unit's own state, because + # a unit ordered after one that has not started never enters `failed` + # at all: polling only its own state waits out the whole bound on a + # guest whose failure is at a unit the poll has not reached. + first_failed() { + systemctl list-units --failed --no-legend --plain --no-pager \ + | head -n 1 | cut -d' ' -f1 + } + report_failure() { + say "$1 failed" + journalctl -b --no-pager -o cat -u "$1" -n 40 >/dev/${serialDevice} 2>&1 || true + systemctl --failed --no-pager --plain >/dev/${serialDevice} 2>&1 || true + exit 1 + } + # The units one unit is ordered against, requires, wants, and that + # trigger it, read out of systemd's own view of the unit rather than + # out of a list restated here: a guest that stalls is described by + # the graph it stalls in, and that graph belongs to the node module. + edges_of() { + systemctl show "$1" --property=After --property=Requires \ + --property=Wants --property=Triggers --property=TriggeredBy \ + --value 2>/dev/null | tr ' ' '\n' | grep -v '^$' || true + } + unit_state() { + echo "--- unit: $1" + systemctl show "$1" --property=Id --property=LoadState \ + --property=ActiveState --property=SubState --property=Result \ + --property=UnitFileState --property=Description --property=After \ + --property=Requires --property=Wants --property=WantedBy \ + --property=Triggers --property=TriggeredBy 2>&1 || true + } + # What a reader of a failed lane needs to see a stall without + # reproducing the run: the d2b slice of the unit list, the boot's + # failed set, and every unit the contract is waiting on together + # with each unit it is ordered against, required by, or waiting on. + # A unit that never started and a unit ordered behind one that never + # started are different failures, and only the graph distinguishes + # them. Announced between two lines the launcher recognises, so the + # state travels with the launcher's own activation failure instead of + # being buried in the console's boot output. + report_ordering() { + stalled="$1" + reason="$2" + say "ordering state for $stalled ($reason)" + { + echo "=== d2b units on this boot ===" + systemctl list-units --all --no-legend --plain --no-pager 'd2b*' 2>&1 || true + echo "=== every unit this boot failed ===" + systemctl list-units --failed --all --no-legend --plain --no-pager 2>&1 || true + frontier="$stalled" + seen="" + while [ -n "$frontier" ]; do + following="" + for candidate in $frontier; do + case " $seen " in + *" $candidate "*) continue ;; + esac + seen="$seen $candidate" + unit_state "$candidate" + for edge in $(edges_of "$candidate"); do + case " $seen $following " in + *" $edge "*) ;; + *) following="$following $edge" ;; + esac + done + done + frontier="$following" + done + } >/dev/${serialDevice} 2>&1 || true + say "end ordering state" + } + # Plain seconds. The systemd time span above carries an `s` because + # that is a duration; shell arithmetic does not, and `1800s` is not + # a number - it is a base the shell cannot read - so a deadline + # written that way aborts this script on its first statement under + # `set -e`, and the guest reports nothing at all. + deadline=$(( $(date +%s) + ${toString activationTimeoutSeconds} )) + units="" + while read -r unit; do + [ -n "$unit" ] || continue + units="$units $unit" + say "waiting for $unit" + # A unit that has not activated for this long is described before + # the launcher's own bound, not at it, because a report written + # after the launcher has stopped reading the console is not a + # report. The first report is unconditional; later ones repeat + # only once the state may have moved on since. + waiting_since=$(date +%s) + reported_at="" + until systemctl is-active --quiet "$unit"; do + if failed=$(first_failed); [ -n "$failed" ]; then + report_failure "$failed" + fi + now=$(date +%s) + if [ $(( now - waiting_since )) -ge ${toString activationStallSeconds} ] && + { [ -z "$reported_at" ] || [ $(( now - reported_at )) -ge ${toString activationStallRepeatSeconds} ]; }; then + report_ordering "$unit" "not active after $(( now - waiting_since ))s" + reported_at="$now" + fi + if [ "$now" -ge "$deadline" ]; then + say "$unit did not activate" + journalctl -b --no-pager -o cat -u "$unit" -n 40 >/dev/${serialDevice} 2>&1 || true + report_ordering "$unit" "activation deadline reached" + exit 1 + fi + sleep 1 + done + say "$unit is active" + done /dev/${serialDevice} + ''; + }; + }; in -if missing != [ ] || unexpected != [ ] then +if !(lib.elem nodeShape [ "daemon" "writable-store" ]) then + throw '' + d2b guest image: unknown node shape '${nodeShape}'. + known shapes: daemon writable-store + '' +else if missing != [ ] || unexpected != [ ] then throw '' d2b guest image: the staged Bazel host-tool bundle does not match the declared inventory. @@ -92,29 +443,52 @@ else let fakeTime = "1"; in - pkgs.runCommand "d2b-vm-guest-image" { + pkgs.runCommand "d2b-vm-guest-image-${nodeShape}" { nativeBuildInputs = [ pkgs.jq pkgs.e2fsprogs pkgs.qemu ]; } '' mkdir -p "$out" - # The initrd carries the system closure, so the kernel and initrd are - # the guest's real system. The toplevel's entries are store symlinks; - # -L resolves them to files. - cp -L ${toplevel}/kernel "$out/kernel" - cp -L ${toplevel}/initrd "$out/initrd" - - # The root disk, built the way the VM module's own run script builds - # it: an ext4 filesystem, converted to qcow2 so the lane can snapshot - # and restore it. export E2FSPROGS_FAKE_TIME=${fakeTime} - ${pkgs.qemu}/bin/qemu-img create -f raw "$TMPDIR/root.raw" ${toString diskSizeMib}M - ${pkgs.e2fsprogs}/bin/mkfs.ext4 -q -F -L nixos -U 00000000-0000-0000-0000-000000000001 "$TMPDIR/root.raw" - ${pkgs.qemu}/bin/qemu-img convert -f raw -O qcow2 "$TMPDIR/root.raw" "$out/disk.qcow2" - rm -f "$TMPDIR/root.raw" + qemu_img=${pkgs.qemu}/bin/qemu-img + + if ${if useBootLoader then "true" else "false"}; then + # The bootloader shape: a writable qcow2 overlay on the installed + # system image, sized so the overlay is at least as large as the disk + # the node asked for, exactly as the VM module's run script sizes it. + backing_mib=$(( $("$qemu_img" info ${bootableSystemImage}/nixos.qcow2 --output=json | ${pkgs.jq}/bin/jq -r '."virtual-size"') / 1024 / 1024 )) + disk_mib=${toString diskSizeMib} + if [ "$disk_mib" -gt "$backing_mib" ]; then + overlay_mib="$disk_mib" + else + overlay_mib="$backing_mib" + fi + "$qemu_img" create -f qcow2 -F qcow2 \ + -b ${bootableSystemImage}/nixos.qcow2 \ + "$TMPDIR/disk.qcow2" "''${overlay_mib}M" + else + # The direct-boot shape's root disk, built the way the VM module's run + # script builds it: an ext4 filesystem, converted to qcow2 so the lane + # can snapshot and restore it. The system closure itself rides in the + # initrd. + "$qemu_img" create -f raw "$TMPDIR/root.raw" ${toString diskSizeMib}M + ${pkgs.e2fsprogs}/bin/mkfs.ext4 -q -F -L nixos -U 00000000-0000-0000-0000-000000000001 "$TMPDIR/root.raw" + "$qemu_img" convert -f raw -O qcow2 "$TMPDIR/root.raw" "$TMPDIR/disk.qcow2" + rm -f "$TMPDIR/root.raw" + + # The direct-boot shape's kernel and initrd, copied in as real files + # rather than named by store path. The manifest names them relative to + # the image root, so an image that only pointed at the host's store + # would declare files it does not carry, and the launcher would find + # them missing at boot. A copy that fails fails the image here. + cp -L ${directBootKernel} "$out/kernel" + cp -L ${directBootInitrd} "$out/initrd" + fi + mv "$TMPDIR/disk.qcow2" "$out/disk.qcow2" # The manifest names the artifacts relative to the image root, the - # exact host-tool package the closure was built against, and the - # invocation shape the lane reproduces per check. + # exact host-tool package the closure was built against, the invocation + # shape the lane reproduces per check, and the activation contract the + # launcher waits for. cat >"$out/manifest.json" <<'JSON' ${builtins.toJSON manifest} JSON diff --git a/packages/d2b-vm-harness/BUILD.bazel b/packages/d2b-vm-harness/BUILD.bazel new file mode 100644 index 000000000..226ccc55a --- /dev/null +++ b/packages/d2b-vm-harness/BUILD.bazel @@ -0,0 +1,61 @@ +load("@crates//:defs.bzl", "all_crate_deps") +load( + "//bazel/checks/rust:d2b_rust_rules.bzl", + "d2b_rust_binary", + "d2b_rust_library", + "d2b_rust_test", +) + +# The lane's guest launcher is not a main-package crate: the repository's +# test census force-registers any crate that carries a test aggregate into the +# main package suite, and this crate's tests run as part of the lane, not as +# part of `make check`. The lane's own suite in `bazel/checks/vm` names the +# targets below directly, so they are still built, linted, and run. +package(default_visibility = ["//bazel/checks:__pkg__", "//bazel/checks/vm:__pkg__"]) + +exports_files( + ["BUILD.bazel", "Cargo.toml"], + visibility = ["//visibility:public"], +) + +d2b_rust_library( + name = "d2b_vm_harness", + srcs = glob( + ["src/**/*.rs"], + exclude = ["src/bin/**/*.rs"], + allow_empty = True, + ), + compile_data = ["Cargo.toml"], + deps = all_crate_deps(normal = True, cargo_only = True), +) + +d2b_rust_binary( + name = "d2b-vm-harness", + srcs = ["src/bin/d2b-vm-harness.rs"], + compile_data = ["Cargo.toml"], + deps = [ + ":d2b_vm_harness", + ] + all_crate_deps(normal = True, cargo_only = True), +) + +d2b_rust_test( + name = "d2b_vm_harness_test", + crate = ":d2b_vm_harness", + compile_data = ["Cargo.toml"], + deps = all_crate_deps(normal = True, normal_dev = True, cargo_only = True), +) + +# keep +filegroup( + name = "cargo_workspace_sources", + srcs = glob( + [ + "Cargo.lock", + "Cargo.toml", + "src/**/*.rs", + "tests/**/*.rs", + ], + allow_empty = True, + ), + visibility = ["//visibility:public"], +) diff --git a/packages/d2b-vm-harness/Cargo.toml b/packages/d2b-vm-harness/Cargo.toml new file mode 100644 index 000000000..7dc12d890 --- /dev/null +++ b/packages/d2b-vm-harness/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "d2b-vm-harness" +version = "0.0.0-bootstrap" +edition = "2024" +publish = false +license.workspace = true + +# The lane's guest launcher. It reproduces the emulator invocation a check's +# guest configuration declares, boots it, waits for the guest's activation +# contract, and tears it down - the boot responsibility the nix test driver +# used to own. +# +# The crate carries no test aggregate on purpose: the repository's test census +# force-registers any crate that has one into the main package suite, and this +# crate is not a main-package test. The lane's own suite names its clippy +# targets and its guest-boot targets directly, so it is still linted and still +# run by the repository's gates. +[lints.rust] +unsafe_code = "forbid" + +[lints.clippy] +disallowed_methods = "deny" +await_holding_lock = "deny" +await_holding_refcell_ref = "deny" + +[dependencies] +serde = { workspace = true } +serde_json = { workspace = true } + +[[bin]] +name = "d2b-vm-harness" +path = "src/bin/d2b-vm-harness.rs" diff --git a/packages/d2b-vm-harness/src/bin/d2b-vm-harness.rs b/packages/d2b-vm-harness/src/bin/d2b-vm-harness.rs new file mode 100644 index 000000000..095775774 --- /dev/null +++ b/packages/d2b-vm-harness/src/bin/d2b-vm-harness.rs @@ -0,0 +1,255 @@ +//! The lane's guest self-check. +//! +//! One invocation proves the three things the lane's own harness owns, +//! against a real guest built by the guest-image action from the re-homed +//! guest node: +//! +//! 1. the host satisfies every virtualization precondition, or the lane +//! stops before a guest starts; +//! 2. the guest boots with exactly the invocation its configuration +//! declares, reaches its activation contract, and attaches only +//! devices that can carry a snapshot; +//! 3. tearing it down leaves no emulator process and no working directory +//! behind, and doing that repeatedly changes nothing. +//! +//! Everything it needs arrives as runfile paths in the environment, because +//! that is how the Bazel test target hands it the emulator from the pinned +//! nix package set and the guest image from the guest-image action. + +use std::{env, fs, path::{Path, PathBuf}, process::ExitCode, time::Duration}; + +use d2b_vm_harness::{ + GuestSpec, HarnessError, boot, host, manifest::GuestManifest, report, +}; +use serde_json::json; + +/// The image the lane was pointed at. Supplied by the Bazel test target as a +/// runfile path. +const IMAGE: &str = "D2B_VM_HARNESS_IMAGE"; +/// The emulator binary, from the pinned nix package set the guest closure was +/// realized from. +const EMULATOR: &str = "D2B_VM_HARNESS_EMULATOR"; +/// The lane's working directory, which outlives each individual guest. +const WORK_ROOT: &str = "D2B_VM_HARNESS_WORK_ROOT"; +/// How long the guest has to activate. +const ACTIVATION_TIMEOUT: &str = "D2B_VM_HARNESS_ACTIVATION_TIMEOUT_SECS"; +/// How many boot and teardown cycles to run. +const CYCLES: &str = "D2B_VM_HARNESS_CYCLES"; + +/// The block-graph node name the boot-time snapshot-capability proof attaches +/// its unsnapshottable device under. +const REFUSAL_NODE: &str = "lane_refusal"; + +/// The device id the proof attaches. The emulator reports a hot-attached +/// device by its qdev path rather than by this id, so the id exists to be +/// legible in the emulator's own diagnostics. +const REFUSAL_DEVICE: &str = "lane-refusal"; + +/// How large the proof's raw device is. The size is irrelevant to the +/// judgement - the format is what has no snapshot support - and only has to +/// be big enough for the guest to accept the disk. +const REFUSAL_DEVICE_BYTES: u64 = 8 * 1024 * 1024; + +fn main() -> ExitCode { + match run() { + Ok(report) => { + for line in report { + report_line(&line); + } + ExitCode::SUCCESS + } + Err(error) => { + report_line(&format!("FAIL {error}")); + ExitCode::FAILURE + } + } +} + +/// The lane's working directory, resolved against the caller's own directory +/// when the caller named a relative one. The lane's directory has to outlive +/// each individual guest, so it cannot be a per-action temporary directory +/// the current Bazel release does not expose to a sandboxed action. +fn work_root(caller_dir: std::path::PathBuf) -> Result { + let declared = required_path(WORK_ROOT)?; + Ok(if declared.is_absolute() { + declared + } else { + caller_dir.join(declared) + }) +} + +fn report_line(line: &str) { + report(line); +} + +fn run() -> Result, HarnessError> { + let image_dir = required_path(IMAGE)?; + let emulator = required_path(EMULATOR)?; + let work_root = work_root(env::current_dir().map_err(|error| { + HarnessError::io("reading the lane's working directory", error) + })?)?; + let activation_timeout = Duration::from_secs(optional_u64(ACTIVATION_TIMEOUT, 1200)?); + let cycles = optional_u64(CYCLES, 1)?; + + // The host preconditions come first and gate everything after them: a + // host that cannot run the lane must stop before a guest is built or + // booted, not degrade into a slower run. + let facts = host::require_this_host()?; + let mut lines = vec![format!( + "host: /dev/kvm usable, nested virtualization and nested-state save present ({})", + facts + .module_parameters + .iter() + .map(|(name, value)| format!("{name}={}", value.as_deref().unwrap_or("absent"))) + .collect::>() + .join(" ") + )]; + + let manifest = GuestManifest::load(&image_dir)?; + let contract = manifest.activation.contract.clone(); + lines.push(format!( + "image: shape {} booted {} with {} vCPU and {} MiB of memory on a {} MiB {} disk", + manifest.node_shape, + manifest.boot.method, + manifest.machine.cores, + manifest.machine.memory_size_mib, + manifest.image.disk_size_mib, + manifest.image.disk_format, + )); + + let mut spec = GuestSpec::new( + manifest, + image_dir, + emulator, + work_root, + "lane-harness", + ); + spec.activation_timeout = activation_timeout; + let work_dir = spec.work_dir(); + + for cycle in 0..cycles { + lines.push(format!("cycle {}: booting", cycle + 1)); + let mut guest = boot(&spec)?; + guest.require_snapshottable()?; + lines.push(format!( + "cycle {}: activated ({contract}), every attached writable device carries a snapshot", + cycle + 1, + )); + guest.shutdown()?; + require_removed(&work_dir)?; + lines.push(format!( + "cycle {}: torn down, no emulator process and no working directory left", + cycle + 1 + )); + } + + lines.push(refuse_unsnapshottable_device(&spec)?); + Ok(lines) +} + +/// A guest that left its working directory behind has not been torn down, and +/// a second cycle would inherit whatever the first one left. +fn require_removed(work_dir: &Path) -> Result<(), HarnessError> { + if work_dir.exists() { + return Err(HarnessError::Configuration(format!( + "{} survived teardown", + work_dir.display() + ))); + } + Ok(()) +} + +/// Prove, against a real booted guest, that a writable device the lane +/// cannot snapshot stops the lane at boot. +/// +/// The device is a raw image inside the guest's own working directory, +/// attached through the monitor the way any other device is attached, and it +/// is the case the refusal exists for: a writable node whose format has no +/// internal snapshot support, which a later `snapshot-save` would refuse for +/// the whole guest rather than for that one node. The guest is judged once +/// before the device is attached and once after, so the refusal is +/// attributable to the device rather than to the guest. +#[allow(clippy::disallowed_methods, reason = "synchronous path")] +fn refuse_unsnapshottable_device(spec: &GuestSpec) -> Result { + let work_dir = spec.work_dir(); + let mut guest = boot(spec)?; + guest.require_snapshottable()?; + + let image = work_dir.join("refusal.img"); + let backing = fs::File::create(&image) + .map_err(|error| HarnessError::io(format!("creating {}", image.display()), error))?; + backing + .set_len(REFUSAL_DEVICE_BYTES) + .map_err(|error| HarnessError::io(format!("sizing {}", image.display()), error))?; + drop(backing); + + let monitor = guest + .monitor() + .ok_or_else(|| HarnessError::Configuration("the guest has no monitor".to_owned()))?; + monitor.execute_with( + "blockdev-add", + json!({ + "node-name": REFUSAL_NODE, + "driver": "raw", + "file": { "driver": "file", "filename": image.to_string_lossy() }, + }), + )?; + monitor.execute_with( + "device_add", + json!({ + "driver": "virtio-blk-pci", + "drive": REFUSAL_NODE, + "id": REFUSAL_DEVICE, + }), + )?; + + let error = guest + .require_snapshottable() + .expect_err("a writable raw device stops the lane at boot"); + let HarnessError::NotSnapshottable { devices } = error else { + return Err(HarnessError::Configuration(format!( + "a writable raw device was refused for the wrong reason: {error}" + ))); + }; + let named = devices + .iter() + .map(|device| device.to_string()) + .collect::>() + .join(", "); + let refused = devices + .iter() + .find(|device| device.file == image.to_string_lossy()) + .ok_or_else(|| { + HarnessError::Configuration(format!( + "the refusal did not name the device the lane attached, but named: {named}" + )) + })? + .clone(); + guest.shutdown()?; + require_removed(&work_dir)?; + Ok(format!( + "boot-time snapshot capability: a writable raw device stops the lane before any check runs \ + ({} bytes at {}, refused as {} on {})", + REFUSAL_DEVICE_BYTES, + refused.file, + refused.format, + refused.device, + )) +} + +fn required_path(name: &str) -> Result { + env::var_os(name) + .filter(|value| !value.is_empty()) + .map(PathBuf::from) + .ok_or_else(|| HarnessError::Configuration(format!("{name} is not set"))) +} + +fn optional_u64(name: &str, default: u64) -> Result { + match env::var(name) { + Err(_) => Ok(default), + Ok(value) if value.is_empty() => Ok(default), + Ok(value) => value.parse().map_err(|error| { + HarnessError::Configuration(format!("{name} is not a number: {value} ({error})")) + }), + } +} diff --git a/packages/d2b-vm-harness/src/error.rs b/packages/d2b-vm-harness/src/error.rs new file mode 100644 index 000000000..031dece01 --- /dev/null +++ b/packages/d2b-vm-harness/src/error.rs @@ -0,0 +1,158 @@ +//! Failure vocabulary for the lane's guest launcher. +//! +//! Every failure a caller can act on names the thing that was missing or the +//! thing that broke: a host that cannot run the lane, a guest whose attached +//! devices cannot be snapshotted, a guest that never activated. A launcher +//! that reports "error" leaves a contributor guessing which of the three it +//! was. + +use std::{fmt, io, path::PathBuf, time::Duration}; + +use crate::host::Capability; + +/// A writable block device the guest attached that cannot carry an internal +/// snapshot. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UnsnapshottableDevice { + /// The device's id as QEMU names it on the monitor. + pub device: String, + /// The backing file QEMU resolved for it. + pub file: String, + /// The image format that device reports. + pub format: String, +} + +impl fmt::Display for UnsnapshottableDevice { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + formatter, + "{} (file {}, format {})", + self.device, self.file, self.format + ) + } +} + +/// Everything that can stop the lane before, during, or after a boot. +#[derive(Debug)] +pub enum HarnessError { + /// The guest image carries no readable manifest, or one the launcher + /// cannot use. + Manifest { path: PathBuf, detail: String }, + /// The host cannot run the lane. Every missing capability is named at + /// once so a contributor fixes them in one pass rather than one boot at + /// a time. + HostUnsupported { missing: Vec }, + /// A guest attached a writable device with no internal-snapshot support. + /// The lane refuses to run without restore rather than silently + /// degrading. + NotSnapshottable { devices: Vec }, + /// The guest never reached its activation contract inside the bound the + /// caller gave it. The console tail travels with the failure, because a + /// bounded wait with no diagnostics is the hang the bound replaced. + NotActivated { + bound: Duration, + marker: String, + console_tail: String, + }, + /// The guest reported activation for a different guest shape than the one + /// the launcher was asked to boot. + WrongShape { + expected: String, + reported: String, + }, + /// The monitor refused a command. + Monitor { command: String, detail: String }, + /// Filesystem or process work that failed underneath the launcher. + Io { action: String, source: io::Error }, + /// The emulator could not be started at all. + Spawn { detail: String }, + /// The launcher's own configuration is unusable. + Configuration(String), +} + +impl fmt::Display for HarnessError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Manifest { path, detail } => { + write!(formatter, "guest manifest {} is unusable: {detail}", path.display()) + } + Self::HostUnsupported { missing } => { + write!(formatter, "this host cannot run the host-integration lane:")?; + for capability in missing { + write!(formatter, "\n - {capability}")?; + } + write!( + formatter, + "\nthe lane requires hardware virtualization; it does not fall back to emulation" + ) + } + Self::NotSnapshottable { devices } => { + write!( + formatter, + "the guest attached writable devices that do not support internal snapshots, so it cannot be restored between checks:" + )?; + for device in devices { + write!(formatter, "\n - {device}")?; + } + write!( + formatter, + "\nmaterialize each of them in qcow2, or attach them with an ephemeral overlay, and the lane will run it" + ) + } + Self::NotActivated { + bound, + marker, + console_tail, + } => { + write!( + formatter, + "the guest did not reach activation within {}s: no {marker} marker on its console", + bound.as_secs() + )?; + if !console_tail.is_empty() { + write!(formatter, "\n--- guest console tail ---\n{console_tail}")?; + } + Ok(()) + } + Self::WrongShape { + expected, + reported, + } => { + write!( + formatter, + "the guest reported activation for the {reported} shape, but the lane asked it to boot the {expected} shape" + ) + } + Self::Monitor { command, detail } => { + write!(formatter, "the emulator monitor refused {command}: {detail}") + } + Self::Io { action, source } => write!(formatter, "{action}: {source}"), + Self::Spawn { detail } => write!(formatter, "the emulator could not be started: {detail}"), + Self::Configuration(detail) => write!(formatter, "lane configuration is unusable: {detail}"), + } + } +} + +impl std::error::Error for HarnessError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::Io { source, .. } => Some(source), + _ => None, + } + } +} + +impl HarnessError { + /// Wrap filesystem or process work in the action that attempted it, so + /// the message says what the launcher was doing rather than only what the + /// operating system said. + pub fn io(action: impl Into, source: io::Error) -> Self { + Self::Io { + action: action.into(), + source, + } + } +} + +/// The launcher's result type. +pub type Result = std::result::Result; diff --git a/packages/d2b-vm-harness/src/guest.rs b/packages/d2b-vm-harness/src/guest.rs new file mode 100644 index 000000000..63883fe14 --- /dev/null +++ b/packages/d2b-vm-harness/src/guest.rs @@ -0,0 +1,1264 @@ +//! Booting one check's guest, and taking it down again. +//! +//! The launcher is the half of the lane that replaces the nix test driver's +//! boot responsibility. It reproduces the emulator invocation the check's +//! guest configuration declared - memory, vCPU count, the drive layout +//! including the writable-store root drive, and whatever per-check device +//! options the node contributed - boots it, waits for the guest's own +//! activation contract, and tears it down. +//! +//! Three properties are worth naming, because each is a decision rather than +//! a detail: +//! +//! * The accelerator is selected explicitly. `-accel kvm` with no TCG +//! fallback is the difference between a host that runs the lane and a host +//! that runs it six times slower without saying so. +//! * The root drive is copied into a working directory the lane owns. The +//! image is a read-only graph output, the guest writes to its disk, and +//! `snapshot=on` state disks put their per-guest overlays under the +//! emulator's `TMPDIR` - which therefore has to be that same directory and +//! has to survive the run. +//! * Every reusable-pool guest carries a machine identity device and a +//! hardware random source. Without them a restored guest replays one RNG +//! stream, and the guest's own activation marker is only written once its +//! random pool is initialised, so a snapshot can never be taken cold. + +use std::{ + ffi::OsString, + fs, + io::Write, + net::TcpListener, + os::unix::fs::PermissionsExt, + path::{Path, PathBuf}, + process::{Child, Command, Stdio}, + thread::sleep, + time::{Duration, Instant}, +}; + +use crate::{ + error::{HarnessError, Result, UnsnapshottableDevice}, + manifest::GuestManifest, + monitor::Monitor, +}; + +/// The longest a guest's own directory may be while still carrying a Unix +/// socket. The limit is 108 bytes; the headroom covers the socket name the +/// emulator appends. +const WORK_DIR_PATH_BUDGET: usize = 88; + +/// The short, lane-scoped root a guest falls back to when its own directory +/// cannot carry a socket. +const LANE_TEMP_DIR: &str = "d2b-vm-lane"; + +/// How often the launcher re-reads the guest's console while waiting for +/// activation. +const CONSOLE_POLL: Duration = Duration::from_millis(500); + +/// How much of the guest's console travels with an activation failure. A +/// bounded wait that reports nothing is the hang the bound replaced. +const CONSOLE_TAIL_BYTES: usize = 64 * 1024; +/// The lines the guest announces its ordering report between. A guest that +/// never activates has not necessarily failed: a unit ordered after one that +/// never started never enters `failed` at all, so the boot's failed set can +/// be empty while the unit the contract is waiting for sits in `activating` +/// behind an ordering dependency nobody has named. The guest therefore +/// reports the graph it is sitting in, and these delimiters are what let the +/// launcher lift that report out of the console rather than leaving the next +/// reader to reproduce a twenty-minute run to see it. +const ORDERING_REPORT_OPENS: &str = "d2b-lane-activation: ordering state for "; +const ORDERING_REPORT_CLOSES: &str = "d2b-lane-activation: end ordering state"; + +/// Everything one guest needs in order to be booted exactly as its +/// configuration declared. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct GuestSpec { + /// The invocation the guest image declared. + pub manifest: GuestManifest, + /// The built image the manifest came from. + pub image_dir: PathBuf, + /// The emulator binary, taken from the pinned nix package set the guest + /// closure was realized from. + pub emulator: PathBuf, + /// The lane's working directory. Each guest gets a subdirectory of it + /// that it owns for its whole life and removes on teardown. + pub work_root: PathBuf, + /// How long the guest has to report activation before the lane fails. + pub activation_timeout: Duration, + /// The guest's name on the monitor and in diagnostics. + pub name: String, +} + +impl GuestSpec { + /// Build a spec for an image, with the lane's own defaults. + pub fn new( + manifest: GuestManifest, + image_dir: impl Into, + emulator: impl Into, + work_root: impl Into, + name: impl Into, + ) -> Self { + Self { + manifest, + image_dir: image_dir.into(), + emulator: emulator.into(), + work_root: work_root.into(), + activation_timeout: Duration::from_secs(1200), + name: name.into(), + } + } + + /// The directory this guest owns. + /// + /// The monitor socket the launcher talks to the emulator over lives in + /// this directory, and a Unix socket path is limited to 108 bytes. Under + /// Bazel the runfiles path is far longer than that, so a guest whose + /// directory would be too long to carry a socket is given a short + /// lane-scoped directory under the host's temporary directory instead. + /// It is created, owned, and removed exactly like the other one; only its + /// location differs, and `work_root` is a caller choice precisely so it + /// can be. + pub fn work_dir(&self) -> PathBuf { + let directory = self.work_root.join(&self.name); + if directory.as_os_str().len() < WORK_DIR_PATH_BUDGET { + return directory; + } + std::env::temp_dir() + .join(LANE_TEMP_DIR) + .join(&self.name) + } + + /// The emulator's command line, exactly as the guest's configuration + /// declared it. + /// + /// This is a pure function of the manifest and the guest's own working + /// directory, which is what makes the per-check invocation assertable + /// without booting anything: a node that asked for a different memory + /// size, a different vCPU count, a different disk, or an extra device + /// produces a different command line, and that difference is the check. + pub fn command_line(&self) -> Result> { + let manifest = &self.manifest; + let work_dir = self.work_dir(); + let mut argv: Vec = Vec::new(); + let mut push = |argument: &str| argv.push(OsString::from(argument)); + + push(&self.emulator.to_string_lossy()); + push("-name"); + push(&self.name); + + // Hardware virtualization, explicitly, with no emulation fallback. + // `max` is the CPU model the VM module launches with, and it is what + // exposes virtualization extensions to the guest, which is how a + // guest can run the nested guest the Cloud Hypervisor checks boot. + push("-machine"); + push("accel=kvm"); + push("-cpu"); + push("max"); + + push("-m"); + push(&manifest.machine.memory_size_mib.to_string()); + push("-smp"); + push(&manifest.machine.cores.to_string()); + + // A machine identity device and a hardware random source, for every + // guest. The identity device changes on every restore and forces the + // guest to reseed; the random device is what the guest's activation + // marker waits for before the lane is allowed to snapshot it. + push("-device"); + push("vmgenid,guid=auto"); + push("-device"); + push("virtio-rng-pci"); + + // The networking the node declared, in the position the VM module's + // own run script gives it: after the random device, before the + // drives. These are not cosmetic. A guest booted without them has no + // network device at all, so the kernel never loads the module that + // device needs and the daemon refuses to start over exactly that - a + // guest that boots cleanly and then stops itself. + for option in render_device_options(&manifest.networking_options) { + push(&option.to_string_lossy()); + } + + // The emulator's runtime data - firmware blobs, keymaps, device + // ROMs - lives beside the binary in the nix package set it came + // from. The runfile is a symlink into that set, so the data + // directory is named explicitly rather than left to the emulator's + // own search, which a relocated runfiles tree would defeat. + if let Some(data_dir) = self.emulator_data_dir() { + push("-L"); + push(&data_dir); + } + + // The lane's own plumbing: no display, the guest's console captured + // for the activation wait and for diagnostics, and a monitor socket + // the launcher owns. + push("-display"); + push("none"); + push("-serial"); + push(&format!("file:{}", work_dir.join("console.log").display())); + push("-monitor"); + push("none"); + push("-qmp"); + push(&format!( + "unix:{},server=on,wait=off", + work_dir.join("qmp.sock").display() + )); + + // The guest's clock follows the emulator's virtual clock, which + // stops while the guest does. That is what keeps a restored guest's + // view of elapsed time continuous across a restore. + push("-rtc"); + push("base=utc,clock=vm"); + + for (index, drive) in manifest.drives.iter().enumerate() { + let file = self.drive_file(drive.file.as_str(), &work_dir); + let drive_id = format!("lane_drive_{index}"); + let mut drive_options = vec![ + ("index".to_owned(), index.to_string()), + ("id".to_owned(), drive_id.clone()), + ("if".to_owned(), "none".to_owned()), + ("file".to_owned(), file), + ]; + if let Some(format) = &drive.format { + drive_options.push(("format".to_owned(), format.clone())); + } + drive_options.push(("cache".to_owned(), drive.cache.clone())); + drive_options.push(("werror".to_owned(), drive.werror.clone())); + push("-drive"); + push(&render_options(&drive_options)); + + let mut device_options = vec![("drive".to_owned(), drive_id)]; + if let Some(boot_index) = &drive.boot_index { + device_options.push(("bootindex".to_owned(), boot_index.clone())); + } + if let Some(serial) = &drive.serial { + device_options.push(("serial".to_owned(), serial.clone())); + } + push("-device"); + push(&format!( + "{},{}", + device_model(&drive.interface), + render_options(&device_options) + )); + } + + for share in &manifest.shared_directories { + push("-virtfs"); + push(&format!( + "local,path={},security_model={},mount_tag={}", + resolve_share(&share.source, &work_dir)?, + share.security_model, + share.mount_tag + )); + } + + match manifest.boot.method.as_str() { + "direct" => { + let (Some(kernel), Some(initrd), Some(append)) = ( + manifest.boot.kernel.as_deref(), + manifest.boot.initrd.as_deref(), + manifest.boot.append.as_deref(), + ) else { + return Err(HarnessError::Configuration( + "the guest image declares the direct boot method without a kernel, an initrd, and a command line" + .to_owned(), + )); + }; + push("-kernel"); + push(&manifest.resolve(&self.image_dir, kernel).to_string_lossy()); + push("-initrd"); + push(&manifest.resolve(&self.image_dir, initrd).to_string_lossy()); + push("-append"); + push(append); + } + "bootloader" => { + if manifest.image.system_image.is_none() { + return Err(HarnessError::Configuration( + "the guest image declares the bootloader boot method without a system image to boot" + .to_owned(), + )); + } + } + other => { + return Err(HarnessError::Configuration(format!( + "the guest image declares the unknown boot method {other}" + ))); + } + } + + // The per-check device options come last, exactly where the node put + // them: a vsock device, an extra drive, anything a check's own + // configuration contributed rides through untouched. + argv.extend(render_device_options(&manifest.extra_options)); + Ok(argv) + } + + /// The emulator's runtime data directory, when the nix package set it + /// came from carries one. + /// + /// The runfile is a symlink into the store, so the data directory is + /// found beside the resolved binary rather than beside the runfile. A + /// build of the emulator that ships no data directory - which is what a + /// `qemu-kvm` wrapper built without the full data set looks like - gets + /// no `-L` at all rather than a path that does not exist. + #[allow(clippy::disallowed_methods, reason = "synchronous path")] + fn emulator_data_dir(&self) -> Option { + let resolved = fs::canonicalize(&self.emulator).unwrap_or_else(|_| self.emulator.clone()); + let data_dir = resolved.parent()?.join("share").join("qemu"); + data_dir + .is_dir() + .then(|| data_dir.to_string_lossy().into_owned()) + } + + /// Where a declared drive file actually is at boot. + /// + /// The root drive is the guest's own: the image is a read-only graph + /// output and the guest writes to it, so the launcher copies it into the + /// working directory it owns. Everything else - a state disk, a shared + /// image - is a store path the guest's node attached itself, and is + /// used where the node put it. + fn drive_file(&self, declared: &str, work_dir: &Path) -> String { + if Path::new(declared).is_absolute() { + declared.to_owned() + } else { + work_dir.join(declared).to_string_lossy().into_owned() + } + } + + /// Materialize the guest's working directory: the writable root drive, + /// the scratch directory the guest's 9p shares are rooted at, and the + /// emulator's `TMPDIR`. + #[allow(clippy::disallowed_methods, reason = "synchronous path")] + fn prepare_work_dir(&self) -> Result { + let work_dir = self.work_dir(); + if work_dir.exists() { + fs::remove_dir_all(&work_dir) + .map_err(|error| HarnessError::io(format!("clearing {}", work_dir.display()), error))?; + } + fs::create_dir_all(work_dir.join("xchg")) + .map_err(|error| HarnessError::io(format!("creating {}", work_dir.display()), error))?; + let disk = self.manifest.resolve(&self.image_dir, &self.manifest.image.disk); + let guest_disk = work_dir.join(&self.manifest.image.disk); + fs::copy(&disk, &guest_disk) + .map_err(|error| HarnessError::io(format!("copying {} to {}", disk.display(), guest_disk.display()), error))?; + // The image is a read-only graph output and the copy inherits that + // mode, but the guest writes to its root drive: the emulator refuses + // to open a drive it cannot write, and reports the permission rather + // than the shape that caused it. + fs::set_permissions(&guest_disk, std::fs::Permissions::from_mode(0o644)) + .map_err(|error| HarnessError::io(format!("making {} writable", guest_disk.display()), error))?; + Ok(work_dir) + } +} + +/// Render the node's device-option list into arguments. +/// +/// The list is shell text by construction, not a pre-split argument vector: +/// the NixOS VM module declares a whole flag and its value in one entry - +/// `"-device virtio-keyboard"` - and its own run script consumes the list +/// unquoted, so the shell word-splits it on the way to the emulator. The +/// launcher reproduces that split rather than handing the entries to +/// `execvp` as they stand: an entry that names a flag and its value is two +/// arguments to the emulator, and passing it as one makes the emulator reject +/// the entry with a message that names the device rather than the launcher. +/// An entry carrying its own properties, `-device usb-tablet,bus=usb-bus.0`, +/// has no whitespace and reaches the emulator whole. +fn render_device_options(options: &[String]) -> impl Iterator { + options + .iter() + .flat_map(|option| option.split_whitespace()) + .map(OsString::from) +} + +/// The block device model for a bus the node declared. +fn device_model(interface: &str) -> &'static str { + match interface { + "scsi" => "lsi53c895a", + "ide" => "ide-hd", + _ => "virtio-blk-pci", + } +} + +fn render_options(options: &[(String, String)]) -> String { + options + .iter() + .map(|(key, value)| format!("{key}={value}")) + .collect::>() + .join(",") +} + +/// A host directory the guest shares over 9p, with the VM module's +/// `TMPDIR`-relative sources resolved against the working directory the lane +/// owns. +/// +/// The module declares two scratch shares that way - one rooted directly at +/// `TMPDIR`, one the caller can redirect through `SHARED_DIR` - and both name +/// the guest's own scratch directory. A source the lane cannot resolve to a +/// path is an error rather than a literal string handed to the emulator, +/// because the emulator reads an unresolved `9p` source as a directory that +/// does not exist and stops at machine start. +fn resolve_share(source: &str, work_dir: &Path) -> Result { + let declared = source.replace('"', ""); + let inner = declared + .strip_prefix("${SHARED_DIR:-") + .map(|rest| { + rest.strip_suffix('}').ok_or_else(|| { + HarnessError::Configuration(format!( + "the guest declares an unterminated share source: {source}" + )) + }) + }) + .transpose()? + .unwrap_or(&declared); + let relative = if let Some(rest) = inner.strip_prefix("$TMPDIR") { + rest + } else if inner.contains('$') { + return Err(HarnessError::Configuration(format!( + "the guest declares a share source the lane cannot resolve: {source}" + ))); + } else { + return Ok(inner.to_owned()); + }; + Ok(work_dir + .join(relative.trim_start_matches('/')) + .to_string_lossy() + .into_owned()) +} + +/// A booted guest that has reported its activation contract. +pub struct ActiveGuest { + child: Child, + monitor: Option, + work_dir: PathBuf, + shut_down: bool, +} + +impl ActiveGuest { + /// The guest's monitor, for the pool's snapshot and restore work. + pub fn monitor(&mut self) -> Option<&mut Monitor> { + self.monitor.as_mut() + } + + /// Where the guest's console is being captured. + pub fn console_log(&self) -> PathBuf { + self.work_dir.join("console.log") + } + + /// The working directory this guest owns. + pub fn work_dir(&self) -> &Path { + &self.work_dir + } + + /// Refuse a guest whose attached writable devices cannot carry an + /// internal snapshot. + /// + /// This runs at boot, before any check does, because the alternative is + /// discovering it at the first restore - after a suite has already run + /// against a guest that could not be rolled back. + pub fn require_snapshottable(&mut self) -> Result<()> { + let devices = self + .monitor + .as_mut() + .ok_or_else(|| HarnessError::Configuration("the guest has no monitor".to_owned()))? + .block_devices()?; + let unsnapshottable: Vec = devices + .iter() + .filter(|device| !device.snapshottable) + .map(|device| UnsnapshottableDevice { + device: device.id.clone(), + file: device.file.clone(), + format: device.format.clone(), + }) + .collect(); + if unsnapshottable.is_empty() { + Ok(()) + } else { + Err(HarnessError::NotSnapshottable { + devices: unsnapshottable, + }) + } + } + + /// Ask the emulator to stop, wait for the process to go, and remove the + /// working directory. + pub fn shutdown(mut self) -> Result<()> { + let outcome = self.stop_emulator(); + self.shut_down = true; + self.remove_work_dir(); + outcome + } + + fn stop_emulator(&mut self) -> Result<()> { + if let Some(monitor) = self.monitor.as_mut() { + // A monitor that refuses `quit` is not a reason to leave a guest + // running: the process is signalled either way. + let _ = monitor.quit(); + } + wait_for_exit(&mut self.child, Duration::from_secs(30))?; + Ok(()) + } + + #[allow(clippy::disallowed_methods, reason = "synchronous path")] + fn remove_work_dir(&mut self) { + if self.work_dir.exists() { + let _ = fs::remove_dir_all(&self.work_dir); + } + } +} + +impl Drop for ActiveGuest { + /// A guest that reaches the end of its scope without a `shutdown` - a + /// failed check, a dropped future - still leaves nothing running. The + /// lane's own teardown is the clean path; this is the backstop that + /// keeps a red test from becoming a leaked emulator. + #[allow(clippy::disallowed_methods, reason = "synchronous path")] + fn drop(&mut self) { + if self.shut_down { + return; + } + let _ = self.child.kill(); + let _ = self.child.wait(); + self.remove_work_dir(); + } +} + +/// Boot a guest and wait for it to report activation. +#[allow(clippy::disallowed_methods, reason = "synchronous path")] +pub fn boot(spec: &GuestSpec) -> Result { + let work_dir = spec.prepare_work_dir()?; + let command_line = spec.command_line()?; + let program = command_line + .first() + .ok_or_else(|| HarnessError::Configuration("the emulator command line is empty".to_owned()))?; + let mut command = Command::new(program); + command + .args(&command_line[1..]) + // The guest's ephemeral block overlays belong to the guest that owns + // them, not to a shared temporary directory that a concurrent guest + // could fill. + .env("TMPDIR", &work_dir) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + // The emulator's own diagnostics are kept: an emulator that refuses + // its command line says why on stderr, and that reason is the whole + // difference between a wrong option and a broken host. + .stderr(Stdio::from(fs::File::create(work_dir.join("emulator.log")).map_err( + |error| HarnessError::io("opening the emulator log", error), + )?)); + let mut child = command.spawn().map_err(|error| HarnessError::Spawn { + detail: format!("{}: {error}\n{command_line:?}", program.to_string_lossy()), + })?; + + let outcome = connect_monitor(&mut child, &work_dir, &command_line) + .and_then(|monitor| wait_for_activation(&mut child, spec, &work_dir, monitor)); + match outcome { + Ok(monitor) => Ok(ActiveGuest { + child, + monitor: Some(monitor), + work_dir, + shut_down: false, + }), + Err(error) => { + // The guest never became usable; the working directory and the + // process go with it, and the guest's own account of the stall + // stays in the error. + let tail = activation_failure_tail(&read_console(&work_dir.join("console.log"))); + let _ = child.kill(); + let _ = child.wait(); + let _ = fs::remove_dir_all(&work_dir); + Err(decorate(error, tail)) + } + } +} + +/// Attach the guest's own account of the stall a launch failure is only +/// useful with. +fn decorate(error: HarnessError, console_tail: String) -> HarnessError { + match error { + HarnessError::NotActivated { + bound, + marker, + console_tail: _, + } => HarnessError::NotActivated { + bound, + marker, + console_tail, + }, + other => other, + } +} + +/// Open the monitor socket the emulator was told to create, bounded so an +/// emulator that never reaches it fails the lane instead of hanging on a +/// connect. +/// +/// Every failure out of here carries the emulator's own stderr. The monitor +/// socket is created before the rest of the command line is finished being +/// acted on, so an emulator that dies on a later argument has already +/// accepted the connection and then resets it - a reset that names the +/// socket and nothing about the option that caused it. The reason is only +/// in the emulator's stderr, so it is read here and travels with the error. +#[allow(clippy::disallowed_methods, reason = "synchronous path")] +fn connect_monitor( + child: &mut Child, + work_dir: &Path, + command_line: &[OsString], +) -> Result { + let socket = work_dir.join("qmp.sock"); + let deadline = Instant::now() + Duration::from_secs(30); + loop { + if let Some(status) = child.try_wait() + .map_err(|error| HarnessError::io("waiting for the emulator", error))? + { + return Err(HarnessError::Spawn { + detail: format!( + "the emulator exited with {status} before its monitor was reachable\n{}", + emulator_diagnostics(work_dir, command_line) + ), + }); + } + if socket.exists() { + return match Monitor::connect(&socket) { + Ok(monitor) => Ok(monitor), + Err(error @ (HarnessError::Monitor { .. } | HarnessError::Io { .. })) => { + Err(HarnessError::Spawn { + detail: format!( + "the emulator accepted its monitor connection and then stopped: {error}\n{}", + emulator_diagnostics(work_dir, command_line) + ), + }) + } + Err(other) => Err(other), + }; + } + if Instant::now() >= deadline { + return Err(HarnessError::Configuration( + "the emulator did not open its monitor socket within 30s".to_owned(), + )); + } + sleep(Duration::from_millis(50)); + } +} + +/// What the emulator was asked to do, and what it said about it. +fn emulator_diagnostics(work_dir: &Path, command_line: &[OsString]) -> String { + format!( + "command line: {command_line:?}\n\ + emulator said:\n{}", + console_tail(&work_dir.join("emulator.log")) + ) +} + +/// Wait for the guest's activation contract, bounded. +/// +/// The guest writes one line on its serial console once the units its own +/// configuration declares are active and its random pool is initialised. +/// Everything the lane needs to know about a boot that did not reach that +/// point is in the console it was reading: the guest reports the ordering +/// state it stalled in on the way, and that report travels with the failure +/// ahead of the console tail. +#[allow(clippy::disallowed_methods, reason = "synchronous path")] +fn wait_for_activation( + child: &mut Child, + spec: &GuestSpec, + work_dir: &Path, + mut monitor: Monitor, +) -> Result { + let console = work_dir.join("console.log"); + let marker = spec.manifest.activation.marker.as_str(); + let deadline = Instant::now() + spec.activation_timeout; + loop { + if let Some(line) = activation_line(&read_console(&console), marker) { + check_shape(&spec.manifest.activation.shape, &line)?; + return Ok(monitor); + } + if let Some(status) = child + .try_wait() + .map_err(|error| HarnessError::io("waiting for the emulator", error))? + { + // The monitor is drained before the failure is reported so a + // guest that stopped itself is described rather than guessed at. + let _ = monitor.take_events(); + return Err(HarnessError::NotActivated { + bound: spec.activation_timeout, + marker: marker.to_owned(), + console_tail: format!( + "the emulator exited with {status}\n{}", + activation_failure_tail(&read_console(&console)) + ), + }); + } + if Instant::now() >= deadline { + return Err(HarnessError::NotActivated { + bound: spec.activation_timeout, + marker: marker.to_owned(), + console_tail: activation_failure_tail(&read_console(&console)), + }); + } + sleep(CONSOLE_POLL); + } +} + +/// The guest's activation line, if the console has produced one. +/// +/// The marker is matched anywhere in the console, not only at the start of a +/// line, because a serial console is a byte stream several writers share and +/// line structure is not a property the launcher can rely on. A login prompt +/// on the same tty emits terminal queries - cursor position, erase display - +/// and lands them mid-line with no newline of its own, which puts the guest's +/// marker in the middle of a line whose first byte is an escape sequence. The +/// guest writes its marker onto a line of its own, and the launcher does not +/// depend on that having worked. +fn activation_line(console: &str, marker: &str) -> Option { + console + .lines() + .find(|line| line.contains(marker)) + .map(str::to_owned) +} + +/// A guest that reports activation for a shape it was not booted as is a +/// wrong image, not a slow guest, and says so. +fn check_shape(expected: &str, line: &str) -> Result<()> { + let reported = line + .split_whitespace() + .find_map(|field| field.strip_prefix("shape=")) + .unwrap_or("unknown"); + if reported == expected { + Ok(()) + } else { + Err(HarnessError::WrongShape { + expected: expected.to_owned(), + reported: reported.to_owned(), + }) + } +} + +#[allow(clippy::disallowed_methods, reason = "synchronous path")] +fn read_console(console: &Path) -> String { + fs::read_to_string(console).unwrap_or_default() +} + +/// The last [`CONSOLE_TAIL_BYTES`] of a console. +fn last_console_bytes(console: &str) -> String { + if console.len() <= CONSOLE_TAIL_BYTES { + return console.to_owned(); + } + let mut start = console.len() - CONSOLE_TAIL_BYTES; + while start < console.len() && !console.is_char_boundary(start) { + start += 1; + } + console[start..].to_owned() +} + +/// The last [`CONSOLE_TAIL_BYTES`] of the guest's console. +#[allow(clippy::disallowed_methods, reason = "synchronous path")] +fn console_tail(console: &Path) -> String { + last_console_bytes(&read_console(console)) +} + +/// The guest's own account of the ordering state it stalled in. +/// +/// The span runs from the first report the guest opened to the last one it +/// closed, so a guest that reported the stall more than once as its state +/// moved is reported whole rather than half. +fn ordering_report(console: &str) -> Option<&str> { + let opened = console.find(ORDERING_REPORT_OPENS)?; + let closed = console.rfind(ORDERING_REPORT_CLOSES)? + ORDERING_REPORT_CLOSES.len(); + (closed > opened).then_some(&console[opened..closed]) +} + +/// What an activation failure carries: the guest's ordering report first, +/// because that is the part a reader acts on, and the console tail behind +/// it, because that is the part that says where the guest got to. +fn activation_failure_tail(console: &str) -> String { + let tail = last_console_bytes(console); + match ordering_report(console) { + Some(report) => { + format!("--- guest ordering report ---\n{report}\n--- guest console tail ---\n{tail}") + } + None => tail, + } +} + +#[allow(clippy::disallowed_methods, reason = "synchronous path")] +fn wait_for_exit(child: &mut Child, bound: Duration) -> Result<()> { + let deadline = Instant::now() + bound; + loop { + if child + .try_wait() + .map_err(|error| HarnessError::io("waiting for the emulator", error))? + .is_some() + { + return Ok(()); + } + if Instant::now() >= deadline { + let _ = child.kill(); + let _ = child.wait(); + return Err(HarnessError::Configuration(format!( + "the emulator did not exit within {}s of being asked to", + bound.as_secs() + ))); + } + sleep(Duration::from_millis(50)); + } +} + +/// Reserve a loopback port for a forwarded guest port. The lane forwards the +/// guest's ssh port so a check's assertions have the same reach the nix test +/// driver gave them. +pub fn reserve_loopback_port() -> Result { + let listener = TcpListener::bind("127.0.0.1:0") + .map_err(|error| HarnessError::io("reserving a loopback port", error))?; + let port = listener + .local_addr() + .map_err(|error| HarnessError::io("reading the reserved port", error))? + .port(); + drop(listener); + Ok(port) +} + +/// Write a line to the lane's own report. +#[allow(clippy::disallowed_methods, reason = "synchronous path")] +pub fn report(line: &str) { + let mut stdout = std::io::stdout().lock(); + let _ = writeln!(stdout, "{line}"); + let _ = stdout.flush(); +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::manifest::{ + Activation, Boot, Drive, ImageFiles, Machine, SharedDirectory, + }; + + fn manifest(cores: u32, memory: u32, extra: &[&str]) -> GuestManifest { + GuestManifest { + schema_version: 1, + system: "x86_64-linux".to_owned(), + node_shape: "daemon".to_owned(), + image: ImageFiles { + disk: "disk.qcow2".to_owned(), + disk_format: "qcow2".to_owned(), + disk_size_mib: 8192, + system_image: None, + }, + machine: Machine { + cores, + memory_size_mib: memory, + }, + boot: Boot { + method: "direct".to_owned(), + kernel: Some("kernel".to_owned()), + initrd: Some("initrd".to_owned()), + append: Some("console=ttyS0,115200n8".to_owned()), + }, + drives: vec![Drive { + name: Some("root".to_owned()), + file: "disk.qcow2".to_owned(), + format: None, + cache: "writeback".to_owned(), + werror: "report".to_owned(), + boot_index: Some("1".to_owned()), + serial: Some("root".to_owned()), + interface: "virtio".to_owned(), + }], + extra_options: extra.iter().map(|value| (*value).to_owned()).collect(), + networking_options: vec![ + "-net nic,netdev=user.0,model=virtio".to_owned(), + "-netdev user,id=user.0,".to_owned(), + ], + shared_directories: vec![ + SharedDirectory { + mount_tag: "nix-store".to_owned(), + security_model: "none".to_owned(), + target: "/nix/.ro-store".to_owned(), + source: "/nix/store".to_owned(), + }, + SharedDirectory { + mount_tag: "xchg".to_owned(), + security_model: "none".to_owned(), + target: "/tmp/xchg".to_owned(), + source: "\"$TMPDIR\"/xchg".to_owned(), + }, + SharedDirectory { + mount_tag: "shared".to_owned(), + security_model: "none".to_owned(), + target: "/tmp/shared".to_owned(), + source: "\"${SHARED_DIR:-$TMPDIR/xchg}\"".to_owned(), + }, + ], + activation: Activation { + readiness: "console-marker".to_owned(), + contract: "d2b-daemon-acceptance".to_owned(), + marker: "D2B_LANE_READY".to_owned(), + serial_device: "ttyS0".to_owned(), + acceptance_units_file: "/etc/d2b/daemon-acceptance-units".to_owned(), + shape: "daemon".to_owned(), + }, + init: "/nix/store/x/init".to_owned(), + toplevel: "/nix/store/x".to_owned(), + host_tool_bundle: "/nix/store/y".to_owned(), + host_tool_inventory: vec!["d2b".to_owned()], + cloud_hypervisor_controller: None, + } + } + + fn spec(manifest: GuestManifest) -> GuestSpec { + GuestSpec::new( + manifest, + "/run/lane/image", + "/nix/store/qemu/bin/qemu-kvm", + "/run/lane/work", + "lane-member-0", + ) + } + + fn argv(spec: &GuestSpec) -> Vec { + spec.command_line() + .expect("the command line renders") + .iter() + .map(|argument| argument.to_string_lossy().into_owned()) + .collect() + } + + fn value_of<'a>(argv: &'a [String], flag: &str) -> &'a str { + let index = argv + .iter() + .position(|argument| argument == flag) + .unwrap_or_else(|| panic!("{flag} is on the command line: {argv:?}")); + &argv[index + 1] + } + + #[test] + fn the_declared_machine_reaches_the_command_line() { + // A check whose configuration asks for a different memory size, vCPU + // count, and disk gets exactly that, because the manifest is the only + // record of the invocation and the launcher restates none of it. + let bigger = spec(manifest(8, 16384, &[])); + let argv = argv(&bigger); + assert_eq!(value_of(&argv, "-m"), "16384"); + assert_eq!(value_of(&argv, "-smp"), "8"); + let drive = value_of(&argv, "-drive"); + assert!( + drive.contains("file=/run/lane/work/lane-member-0/disk.qcow2"), + "the root drive is the guest's own writable copy: {drive}" + ); + assert!(!drive.contains("format="), "the node pinned no format: {drive}"); + } + + #[test] + fn a_different_node_shape_reaches_the_command_line() { + let mut declared = manifest(3, 3072, &[]); + declared.machine.cores = 5; + declared.machine.memory_size_mib = 5120; + let argv = argv(&spec(declared)); + assert_eq!(value_of(&argv, "-smp"), "5"); + assert_eq!(value_of(&argv, "-m"), "5120"); + } + + #[test] + fn a_per_check_device_option_reaches_the_command_line_untouched() { + // The vsock device the guest-shell-service check declares is an + // ordinary entry in the node's own option list, and it rides through + // exactly as the node wrote it. + let declared = manifest( + 3, + 3072, + &["-drive", "file=/nix/store/z-state.img,format=raw,if=virtio", "-device", "vhost-vsock-pci,guest-cid=3"], + ); + let argv = argv(&spec(declared)); + assert!(argv.windows(2).any(|pair| pair + == ["-device", "vhost-vsock-pci,guest-cid=3"])); + assert!(argv + .iter() + .any(|argument| argument.contains("z-state.img"))); + } + + #[test] + fn the_networking_the_node_declared_reaches_the_command_line() { + // The regression this unit's activation failure was. The VM module + // renders `virtualisation.qemu.networkingOptions` in its own run + // script as a group separate from `virtualisation.qemu.options`, and + // a launcher that carries only the option list boots a guest with no + // network device at all. The guest boots cleanly, the kernel never + // loads the module that device needs, and the daemon refuses to + // start over exactly that - with a unit that failed and a guest that + // looks fine, which is the worst shape a boot failure can have. + let argv = argv(&spec(manifest(3, 3072, &[]))); + assert!( + argv.windows(2) + .any(|pair| pair == ["-net", "nic,netdev=user.0,model=virtio"]), + "the guest gets the network device the node declared: {argv:?}" + ); + assert!( + argv.windows(2) + .any(|pair| pair == ["-netdev", "user,id=user.0,"]), + "and the backend that device rides on: {argv:?}" + ); + } + + #[test] + fn the_networking_options_take_the_position_the_vm_module_gives_them() { + // After the random device and before the drives, which is where the + // VM module's own run script puts them, so a snapshot of the command + // line reads like the invocation the node declared. + let argv = argv(&spec(manifest(3, 3072, &[]))); + let position = |needle: &str| { + argv.iter() + .position(|argument| argument.starts_with(needle)) + .unwrap_or_else(|| panic!("{needle} is on the command line: {argv:?}")) + }; + assert!(position("virtio-rng-pci") < position("-net")); + assert!(position("-net") < position("-drive")); + } + + #[test] + fn one_entry_naming_a_flag_and_its_value_becomes_two_arguments() { + // The regression this unit's boot failure was: the node declares + // `-device virtio-keyboard` as a single list entry, the VM module's + // run script word-splits it, and handing the entry to `execwp` whole + // makes the emulator refuse it. + let declared = manifest( + 3, + 3072, + &["-device virtio-keyboard", "-usb", "-device virtio-keyboard"], + ); + let argv = argv(&spec(declared)); + assert!( + argv.windows(2).any(|pair| pair == ["-device", "virtio-keyboard"]), + "the flag and its value reach the emulator as two arguments: {argv:?}" + ); + assert!( + !argv.iter().any(|argument| argument.contains(' ')), + "no argument carries the node's internal whitespace: {argv:?}" + ); + } + + #[test] + fn an_entry_carrying_its_own_properties_reaches_the_emulator_whole() { + let declared = manifest( + 3, + 3072, + &["-device usb-tablet,bus=usb-bus.0", "-device vhost-vsock-pci,guest-cid=3"], + ); + let argv = argv(&spec(declared)); + assert!(argv + .windows(2) + .any(|pair| pair == ["-device", "usb-tablet,bus=usb-bus.0"])); + assert!(argv + .windows(2) + .any(|pair| pair == ["-device", "vhost-vsock-pci,guest-cid=3"])); + } + + #[test] + fn the_accelerator_is_selected_and_never_falls_back_to_emulation() { + let argv = argv(&spec(manifest(3, 3072, &[]))); + let machine = value_of(&argv, "-machine"); + assert_eq!(machine, "accel=kvm", "no tcg fallback on this path"); + assert!(!argv.iter().any(|argument| argument.contains("tcg"))); + assert_eq!(value_of(&argv, "-cpu"), "max"); + } + + #[test] + fn a_reusable_pool_guest_carries_an_identity_device_and_a_random_source() { + let argv = argv(&spec(manifest(3, 3072, &[]))); + assert!(argv.windows(2).any(|pair| pair == ["-device", "vmgenid,guid=auto"])); + assert!(argv.windows(2).any(|pair| pair == ["-device", "virtio-rng-pci"])); + } + + #[test] + fn the_direct_boot_shape_is_handed_its_kernel_initrd_and_command_line() { + let argv = argv(&spec(manifest(3, 3072, &[]))); + assert_eq!(value_of(&argv, "-kernel"), "/run/lane/image/kernel"); + assert_eq!(value_of(&argv, "-initrd"), "/run/lane/image/initrd"); + assert_eq!(value_of(&argv, "-append"), "console=ttyS0,115200n8"); + } + + #[test] + fn the_bootloader_shape_gets_no_kernel_and_keeps_its_root_drive() { + // The writable-store shape boots its own disk through the bootloader + // the image was built with, and its root drive is the one the node + // declared - an unsafe cache, a boot index, and the serial the guest + // mounts by - rather than the direct-boot shape's. + let mut declared = manifest(3, 3072, &[]); + declared.node_shape = "writable-store".to_owned(); + declared.boot = Boot { + method: "bootloader".to_owned(), + kernel: None, + initrd: None, + append: None, + }; + declared.image.system_image = Some("/nix/store/z/nixos.qcow2".to_owned()); + declared.drives[0].cache = "unsafe".to_owned(); + declared.drives[0].werror = "report".to_owned(); + declared.activation.shape = "writable-store".to_owned(); + let argv = argv(&spec(declared)); + assert!(!argv.iter().any(|argument| argument == "-kernel")); + assert!(!argv.iter().any(|argument| argument == "-append")); + let drive = value_of(&argv, "-drive"); + assert!(drive.contains("cache=unsafe"), "{drive}"); + let device = argv + .iter() + .find(|argument| argument.starts_with("virtio-blk-pci,")) + .expect("the root drive is attached"); + assert!(device.contains("bootindex=1"), "{device}"); + assert!(device.contains("serial=root"), "{device}"); + } + + #[test] + fn a_shared_directory_relative_to_tmpdir_lands_in_the_guests_own_work_dir() { + let argv = argv(&spec(manifest(3, 3072, &[]))); + let share = argv + .iter() + .find(|argument| argument.contains("mount_tag=xchg")) + .expect("the scratch share is exported"); + assert!( + share.contains("path=/run/lane/work/lane-member-0/xchg"), + "{share}" + ); + let store = argv + .iter() + .find(|argument| argument.contains("mount_tag=nix-store")) + .expect("the store share is exported"); + assert!(store.contains("path=/nix/store"), "{store}"); + let shared = argv + .iter() + .find(|argument| argument.contains("mount_tag=shared")) + .expect("the caller-redirectable share is exported"); + assert!( + shared.contains("path=/run/lane/work/lane-member-0/xchg"), + "a share the caller can redirect still names the guest's own scratch directory: {shared}" + ); + } + + #[test] + fn the_console_and_the_monitor_are_where_the_lane_can_reach_them() { + let argv = argv(&spec(manifest(3, 3072, &[]))); + assert_eq!( + value_of(&argv, "-serial"), + "file:/run/lane/work/lane-member-0/console.log" + ); + assert_eq!( + value_of(&argv, "-qmp"), + "unix:/run/lane/work/lane-member-0/qmp.sock,server=on,wait=off" + ); + } + + #[test] + fn a_console_carrying_the_marker_is_the_activation_signal() { + let console = "[ 2.113456] systemd: Reached target multi-user\n\ + D2B_LANE_READY shape=daemon units=d2bd.service d2b-broker.service\n\ + [ 9.000000] sshd[1]: started\n"; + assert_eq!( + activation_line(console, "D2B_LANE_READY").as_deref(), + Some("D2B_LANE_READY shape=daemon units=d2bd.service d2b-broker.service") + ); + assert_eq!(activation_line(console, "NOT_A_MARKER"), None); + } + + #[test] + fn a_login_prompt_sharing_the_console_does_not_hide_the_marker() { + // The regression this unit's flaky lane run was. A getty on the same + // serial tty emits terminal queries - cursor position, erase display - + // and lands them mid-line with no newline of its own, so the guest's + // marker ends up in the middle of a line that starts with an escape + // sequence. Matching the marker only at the start of a line misses + // that boot entirely, and the lane waits out its whole bound on a + // guest that activated within a minute. The bytes below are what the + // console actually held. + let console = "d2b-lane-activation: waiting for the random pool\n\ + \u{1b}[!p\u{5b}104\\\u{1b}[0m\u{1b}[?7h\u{1b}[1G\u{1b}[0J\u{1b}[6n\u{1b}[32766;32766H\u{1b}[6nD2B_LANE_READY shape=daemon units=d2bd.service\r\n\ + <<< Welcome to NixOS 26.05pre-git (x86_64) - ttyS0 >>>\n"; + let line = activation_line(console, "D2B_LANE_READY") + .expect("the marker is in the stream, wherever the stream put it"); + assert!(line.contains("D2B_LANE_READY"), "{line}"); + check_shape("daemon", &line).expect("and the shape still reads off the line"); + assert!(!line.starts_with("D2B_LANE_READY"), "{line}"); + } + + #[test] + fn a_guest_that_reports_the_wrong_shape_is_a_wrong_image() { + let error = check_shape("writable-store", "D2B_LANE_READY shape=daemon units=d2bd.service") + .expect_err("the shape is checked"); + let rendered = error.to_string(); + assert!(rendered.contains("daemon"), "{rendered}"); + assert!(rendered.contains("writable-store"), "{rendered}"); + check_shape("daemon", "D2B_LANE_READY shape=daemon units=d2bd.service") + .expect("the declared shape activates"); + } + + /// The console shape a guest that stalls produces: the launcher read + /// past the boot output, the guest announced it was waiting, and then + /// reported the ordering graph it was waiting in. + fn stalled_console() -> String { + "[ 2.113456] systemd: Reached target multi-user\n\ + d2b-lane-activation: waiting for d2bd.service\n\ + d2b-lane-activation: ordering state for d2bd.service (not active after 91s)\n\ + === d2b units on this boot ===\n\ + d2bd.service activating start d2b daemon\n\ + --- unit: d2bd.service\n\ + ActiveState=activating\n\ + SubState=start\n\ + After=systemd-tmpfiles-setup.service network.target d2b-broker.socket d2b.slice\n\ + Wants=d2b-broker.socket systemd-tmpfiles-setup.service\n\ + --- unit: systemd-tmpfiles-setup.service\n\ + ActiveState=inactive\n\ + SubState=dead\n\ + d2b-lane-activation: end ordering state\n\ + nixos login: \n" + .to_owned() + } + + #[test] + fn the_guests_own_ordering_report_travels_with_the_activation_failure() { + // The whole point of the report: a boot that reaches the console but + // not the marker is described by the graph the guest stalled in, and + // that description is in the failure without another twenty-minute + // run to reproduce it. + let tail = activation_failure_tail(&stalled_console()); + assert!(tail.contains("guest ordering report"), "{tail}"); + assert!(tail.contains("ordering state for d2bd.service"), "{tail}"); + assert!( + tail.contains("After=systemd-tmpfiles-setup.service"), + "{tail}" + ); + assert!(tail.contains("ActiveState=activating"), "{tail}"); + assert!(tail.contains("guest console tail"), "{tail}"); + assert!(tail.find("ordering state for") < tail.find("guest console tail")); + } + + #[test] + fn a_console_that_carries_no_ordering_report_is_just_a_console() { + let console = "[ 2.113456] systemd: Reached target multi-user\n\ + d2b-lane-activation: waiting for d2bd.service\n"; + assert_eq!(ordering_report(console), None); + assert_eq!(activation_failure_tail(console), console); + } + + #[test] + fn an_ordering_report_the_guest_opened_and_never_closed_is_not_a_report() { + // A guest killed mid-report leaves an opening delimiter and nothing + // else. Reading a report that was never finished would present a + // truncated ordering graph as a complete one. + let truncated = "d2b-lane-activation: ordering state for d2bd.service\n\ + --- unit: d2bd.service\n"; + assert_eq!(ordering_report(truncated), None); + } + + #[test] + fn a_guest_that_reported_the_stall_twice_is_reported_whole() { + let console = format!( + "{}{}", + stalled_console(), + stalled_console() + .strip_prefix("[ 2.113456] systemd: Reached target multi-user\n") + .expect("the console restarts at the report") + ); + let report = ordering_report(&console).expect("the guest reported the stall"); + assert_eq!( + report.matches("ordering state for d2bd.service").count(), + 2, + "{report}" + ); + } +} diff --git a/packages/d2b-vm-harness/src/host.rs b/packages/d2b-vm-harness/src/host.rs new file mode 100644 index 000000000..5d431f641 --- /dev/null +++ b/packages/d2b-vm-harness/src/host.rs @@ -0,0 +1,296 @@ +//! Host virtualization preconditions. +//! +//! The lane runs on a contributor's own machine, so a host that cannot run +//! it is a host where the checks are silently six times slower rather than a +//! host where they are unavailable. The lane therefore refuses to start +//! unless the host has hardware virtualization, nested virtualization, and +//! the host kernel's ability to save a nested guest's state - the third is +//! what snapshotting a guest that has run a nested check needs, and its +//! absence surfaces as a `-EINVAL` from the emulator at snapshot time rather +//! than as anything a contributor could act on. +//! +//! Every capability is read from a host fact, never inferred from the +//! emulator's own behaviour, and every missing one is reported by name in a +//! single message so a contributor fixes them in one pass. + +use std::{fmt, fs, path::Path}; + +use crate::error::{HarnessError, Result}; + +/// The KVM character device the lane's guests attach to. +const KVM_DEVICE: &str = "/dev/kvm"; + +/// The in-tree x86 KVM modules, with the module parameter that gates nested +/// virtualization and the one that gates saving a nested guest's state. +/// +/// `kvm_amd` separates the two: `nested` admits an L2 guest at all, and +/// `nested_svm` is what makes the nested SVM state extractable. `kvm_intel` +/// has no second knob - the nested VMCS save path is compiled in whenever +/// `nested` is on - so on that module the two capabilities are satisfied by +/// one read, and the state-save capability reports the module that decided +/// it. +const KVM_MODULES: &[KvmModule] = &[ + KvmModule { + name: "kvm_amd", + nested: "nested", + nested_state: Some("nested_svm"), + }, + KvmModule { + name: "kvm_intel", + nested: "nested", + nested_state: None, + }, +]; + +struct KvmModule { + name: &'static str, + nested: &'static str, + nested_state: Option<&'static str>, +} + +/// One precondition the lane states as a precondition rather than assuming. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum Capability { + /// `/dev/kvm` exists and this process can open it for both reading and + /// writing. + KvmDevice, + /// The KVM module in use has nested virtualization enabled, so a guest + /// can run the nested guest the Cloud Hypervisor checks boot. + NestedVirtualization, + /// The host kernel can save a nested guest's virtualization state, so a + /// guest that has run one can still be snapshotted. + NestedStateSave, +} + +impl Capability { + /// A short, stable name for the capability, used in the failure message. + pub fn as_str(self) -> &'static str { + match self { + Self::KvmDevice => "hardware virtualization (/dev/kvm)", + Self::NestedVirtualization => "nested virtualization", + Self::NestedStateSave => "nested-state save support", + } + } + + /// What a contributor does to provide the capability. + fn remedy(self) -> &'static str { + match self { + Self::KvmDevice => { + "load the kvm_intel or kvm_amd module, then add this user to the `kvm` group" + } + Self::NestedVirtualization => { + "boot the host kernel with the `kvm-intel.nested=1` or `kvm-amd.nested=1` module parameter" + } + Self::NestedStateSave => { + "boot the host kernel with the `kvm-amd.nested_svm=1` module parameter (on Intel the nested-state path needs only `nested=1`)" + } + } + } +} + +impl fmt::Display for Capability { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "{}: {}", self.as_str(), self.remedy()) + } +} + +/// The host facts the assessment reads. +/// +/// Gathering them is separated from judging them so the judgment is a pure +/// function: a host that is missing a capability is described by the same +/// facts a healthy host is, and the missing-capability path is exercised +/// without needing a second machine. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HostFacts { + /// `Some(())` when `/dev/kvm` opened read-write, `Some(reason)` when it + /// exists but did not, `None` when it is not there at all. + pub kvm_device: std::result::Result<(), String>, + /// The loaded KVM module and the value of each parameter the assessment + /// reads, keyed `"/"`. + pub module_parameters: Vec<(String, Option)>, +} + +impl HostFacts { + /// Read the facts off this host. + #[allow(clippy::disallowed_methods, reason = "synchronous path")] + pub fn probe() -> Self { + let kvm_device = match fs::OpenOptions::new() + .read(true) + .write(true) + .open(KVM_DEVICE) + { + Ok(_file) => Ok(()), + Err(error) => Err(error.to_string()), + }; + let mut module_parameters = Vec::new(); + for module in KVM_MODULES { + for parameter in [Some(module.nested), module.nested_state] + .into_iter() + .flatten() + { + let path = Path::new("/sys/module") + .join(module.name) + .join("parameters") + .join(parameter); + module_parameters.push(( + format!("{}/{}", module.name, parameter), + read_trimmed(&path), + )); + } + } + Self { + kvm_device, + module_parameters, + } + } + + fn parameter(&self, key: &str) -> Option { + self.module_parameters + .iter() + .find(|(name, _)| name == key) + .and_then(|(_, value)| value.as_deref()) + .map(is_enabled) + } +} + +#[allow(clippy::disallowed_methods, reason = "synchronous path")] +fn read_trimmed(path: &Path) -> Option { + fs::read_to_string(path).ok().map(|text| text.trim().to_owned()) +} + +/// A KVM module parameter is enabled when it reads `Y`, `y`, `1`, or `true`; +/// every other value - including a module that is not loaded at all, which +/// has no parameter node - is not. +fn is_enabled(value: &str) -> bool { + matches!(value, "Y" | "y" | "1" | "true" | "True") +} + +/// The capabilities this host is missing, in a stable order. +/// +/// A host with no loaded KVM module at all is missing both nested +/// capabilities rather than reporting a module it never found: the lane needs +/// the capability, and naming the knob is the actionable half of the message. +pub fn missing_capabilities(facts: &HostFacts) -> Vec { + let mut missing = Vec::new(); + if facts.kvm_device.is_err() { + missing.push(Capability::KvmDevice); + } + let nested = KVM_MODULES + .iter() + .any(|module| facts.parameter(&format!("{}/{}", module.name, module.nested)) == Some(true)); + if !nested { + missing.push(Capability::NestedVirtualization); + } + // Saved nested state is a superset of nested virtualization: on AMD it + // has its own knob, and on Intel the single `nested` read already covers + // it. A host with neither knob is reported as missing both, because + // neither is satisfied. + let state = KVM_MODULES.iter().any(|module| match module.nested_state { + Some(parameter) => { + facts.parameter(&format!("{}/{}", module.name, parameter)) == Some(true) + && facts.parameter(&format!("{}/{}", module.name, module.nested)) == Some(true) + } + None => facts.parameter(&format!("{}/{}", module.name, module.nested)) == Some(true), + }); + if !state { + missing.push(Capability::NestedStateSave); + } + missing +} + +/// Refuse to start the lane on a host that cannot run it. +pub fn require(facts: &HostFacts) -> Result<()> { + let missing = missing_capabilities(facts); + if missing.is_empty() { + Ok(()) + } else { + Err(HarnessError::HostUnsupported { missing }) + } +} + +/// Read this host's facts and require every capability. +pub fn require_this_host() -> Result { + let facts = HostFacts::probe(); + require(&facts)?; + Ok(facts) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn facts( + kvm_device: std::result::Result<(), String>, + parameters: &[(&str, &str)], + ) -> HostFacts { + HostFacts { + kvm_device, + module_parameters: parameters + .iter() + .map(|(name, value)| ((*name).to_owned(), Some((*value).to_owned()))) + .collect(), + } + } + + #[test] + fn a_healthy_intel_host_reports_nothing_missing() { + let healthy = facts(Ok(()), &[("kvm_intel/nested", "Y")]); + assert_eq!(missing_capabilities(&healthy), Vec::new()); + require(&healthy).expect("an Intel host with nested enabled runs the lane"); + } + + #[test] + fn a_healthy_amd_host_needs_both_of_its_knobs() { + let healthy = facts(Ok(()), &[("kvm_amd/nested", "Y"), ("kvm_amd/nested_svm", "Y")]); + assert_eq!(missing_capabilities(&healthy), Vec::new()); + } + + #[test] + fn an_amd_host_without_nested_state_save_names_that_capability() { + // The one the plan calls out: a host that admits a nested guest but + // cannot save its state stops the lane before any guest boots, and + // the message says which of the two knobs is wrong. + let amd = facts( + Ok(()), + &[("kvm_amd/nested", "Y"), ("kvm_amd/nested_svm", "N")], + ); + let missing = missing_capabilities(&amd); + assert_eq!(missing, vec![Capability::NestedStateSave]); + let error = require(&amd).expect_err("the lane refuses this host"); + let rendered = error.to_string(); + assert!(rendered.contains("nested-state save support"), "{rendered}"); + assert!(rendered.contains("nested_svm=1"), "{rendered}"); + } + + #[test] + fn a_host_without_kvm_names_every_capability_it_is_missing() { + let bare = facts(Err("No such file or directory".to_owned()), &[]); + let error = require(&bare).expect_err("the lane refuses a host with no KVM"); + let rendered = error.to_string(); + for expected in [ + "/dev/kvm", + "nested virtualization", + "nested-state save support", + "does not fall back to emulation", + ] { + assert!(rendered.contains(expected), "{rendered}"); + } + } + + #[test] + fn nested_disabled_is_reported_as_both_nested_capabilities() { + let host = facts(Ok(()), &[("kvm_intel/nested", "N")]); + assert_eq!( + missing_capabilities(&host), + vec![Capability::NestedVirtualization, Capability::NestedStateSave] + ); + } + + #[test] + fn an_unloaded_module_is_not_an_enabled_parameter() { + assert!(!is_enabled("N")); + assert!(!is_enabled("")); + assert!(is_enabled("Y")); + assert!(is_enabled("1")); + } +} diff --git a/packages/d2b-vm-harness/src/lib.rs b/packages/d2b-vm-harness/src/lib.rs new file mode 100644 index 000000000..b92e7093b --- /dev/null +++ b/packages/d2b-vm-harness/src/lib.rs @@ -0,0 +1,28 @@ +//! The d2b host-integration lane's guest launcher. +//! +//! The lane is Bazel-owned end to end: a build action produces each check's +//! guest image as a cacheable graph output, and this crate boots it, waits +//! for it to activate, and takes it down. What it deliberately does *not* do +//! is decide the invocation - memory, vCPU count, the drive layout, the +//! per-check device options - for a guest. Those are read off the re-homed +//! guest node's evaluated configuration and reach the launcher through the +//! image's manifest, so a check boots the guest it declared rather than a +//! uniform shape the lane made up. +//! +//! The crate holds no test aggregate: the repository's test census +//! force-registers any crate that has one into the main package suite, and +//! this crate is not a main-package test. The lane's own suite in +//! `bazel/checks/vm` names its clippy targets and its guest-boot targets +//! directly, so it is still linted and still run by the repository's gates. + +pub mod error; +pub mod guest; +pub mod host; +pub mod manifest; +pub mod monitor; + +pub use error::{HarnessError, Result, UnsnapshottableDevice}; +pub use guest::{ActiveGuest, GuestSpec, boot, report, reserve_loopback_port}; +pub use host::{Capability, HostFacts, require_this_host}; +pub use manifest::GuestManifest; +pub use monitor::{BlockDevice, Monitor}; diff --git a/packages/d2b-vm-harness/src/manifest.rs b/packages/d2b-vm-harness/src/manifest.rs new file mode 100644 index 000000000..3d3ff2469 --- /dev/null +++ b/packages/d2b-vm-harness/src/manifest.rs @@ -0,0 +1,291 @@ +//! The guest image's declared invocation. +//! +//! The image action evaluates the re-homed guest node and writes down what it +//! found: the machine size, the drive layout, the boot method with its kernel +//! command line already resolved, the per-check device options, and the +//! activation contract. That manifest is the only record of the invocation. +//! The launcher renders it and never restates a number the node declared, so +//! a check that asks for four vCPUs, eight gigabytes of memory, and a vsock +//! device gets exactly that, and a check that asks for something else gets +//! something else. + +use std::{fs, path::{Path, PathBuf}}; + +use serde::Deserialize; + +use crate::error::{HarnessError, Result}; + +/// The manifest schema this launcher reads. A manifest that declares a +/// different one is refused rather than guessed at. +pub const SUPPORTED_SCHEMA_VERSION: u32 = 1; + +/// The guest image's declared invocation, as the image action wrote it. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct GuestManifest { + /// The manifest schema version. + pub schema_version: u32, + /// The system the guest closure was realized for. + pub system: String, + /// Which of the re-homed node's two guest shapes this image evaluates. + pub node_shape: String, + /// The files the launcher boots, relative to the image root. + pub image: ImageFiles, + /// The machine size the node declared. + pub machine: Machine, + /// How the guest is booted, and with what. + pub boot: Boot, + /// The block devices the launcher attaches, in the node's order. + pub drives: Vec, + /// The per-check device options, in the node's order, with the VM + /// module's own direct-boot directives already resolved into [`Boot`]. + pub extra_options: Vec, + /// The networking options the node declared, with the VM module's own + /// shell substitution resolved out. + /// + /// A separate field from [`Self::extra_options`] because the VM module + /// renders them in a separate place in its own run script, and a guest + /// booted without them has no network device at all - which the daemon + /// refuses to start over, by name. + pub networking_options: Vec, + /// The host directories the guest mounts over 9p. + #[serde(default)] + pub shared_directories: Vec, + /// The activation contract the launcher waits for. + pub activation: Activation, + /// The guest's init, as the closure names it. + pub init: String, + /// The guest's system closure, as the closure names it. + pub toplevel: String, + /// The host-tool package the guest closure was built against. + pub host_tool_bundle: String, + /// The binaries that package carries. + pub host_tool_inventory: Vec, + /// The Cloud Hypervisor controller, when the image carried one. + pub cloud_hypervisor_controller: Option, +} + +/// The files the launcher boots, relative to the image root. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ImageFiles { + /// The guest's root disk. + pub disk: String, + /// The format that disk is in. + pub disk_format: String, + /// How large the node asked the disk to be. + pub disk_size_mib: u64, + /// The installed system image the root disk is a writable overlay on, for + /// the shape that boots through a bootloader. + pub system_image: Option, +} + +/// The machine size the node declared. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct Machine { + /// The vCPU count. + pub cores: u32, + /// The memory size, in MiB. + pub memory_size_mib: u32, +} + +/// How the guest is booted. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct Boot { + /// `direct` boots the kernel and initrd the image carries; `bootloader` + /// boots the root disk through the bootloader the image was built with. + pub method: String, + /// The kernel to direct-boot, relative to the image root. + pub kernel: Option, + /// The initrd to direct-boot, relative to the image root. + pub initrd: Option, + /// The kernel command line, already resolved: the guest's own kernel + /// parameters, its init, the store registration the activation reads, + /// and the console list the node declared. + pub append: Option, +} + +/// One attached block device, in the shape the node declared it. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct Drive { + /// The node's name for the drive, when it gave one. + pub name: Option, + /// The backing file: an image-relative name, or an absolute path into + /// the store for a device the node attaches itself. + pub file: String, + /// The image format, when the node pinned one. `None` leaves the + /// emulator to read it from the file, exactly as the node asked. + pub format: Option, + /// The cache mode. + pub cache: String, + /// Whether the emulator reports write errors. + pub werror: String, + /// The boot order index, when the node pinned one. + pub boot_index: Option, + /// The serial the guest identifies the device by. + pub serial: Option, + /// The bus the device is attached to. + pub interface: String, +} + +/// One host directory the guest mounts over 9p. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct SharedDirectory { + /// The mount tag the guest sees. + pub mount_tag: String, + /// The 9p security model. + pub security_model: String, + /// Where the guest mounts it. + pub target: String, + /// The host path, exactly as the node declared it. A `$TMPDIR`-relative + /// source stays relative; the launcher resolves it against the working + /// directory it owns. + pub source: String, +} + +/// The activation contract, in the repository's own field names. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct Activation { + /// The readiness signal the guest reports on. + pub readiness: String, + /// What that signal means. + pub contract: String, + /// The line the guest writes once the contract is met. + pub marker: String, + /// The serial device that line arrives on. + pub serial_device: String, + /// Where inside the guest the units the contract names are declared. + pub acceptance_units_file: String, + /// The shape the guest reports, so a launcher that booted the wrong + /// image finds out from the guest rather than from a later failure. + pub shape: String, +} + +impl GuestManifest { + /// Read the manifest out of a built guest image. + #[allow(clippy::disallowed_methods, reason = "synchronous path")] + pub fn load(image_dir: &Path) -> Result { + let path = image_dir.join("manifest.json"); + let text = fs::read_to_string(&path) + .map_err(|error| HarnessError::io(format!("reading {}", path.display()), error))?; + let manifest: Self = serde_json::from_str(&text).map_err(|error| { + HarnessError::Manifest { + path: PathBuf::from(&path), + detail: error.to_string(), + } + })?; + if manifest.schema_version != SUPPORTED_SCHEMA_VERSION { + return Err(HarnessError::Manifest { + path, + detail: format!( + "manifest schema {} is not the version this launcher reads ({SUPPORTED_SCHEMA_VERSION})", + manifest.schema_version + ), + }); + } + Ok(manifest) + } + + /// The path of a file the manifest names relative to the image root. + pub fn resolve(&self, image_dir: &Path, file: &str) -> PathBuf { + let declared = Path::new(file); + if declared.is_absolute() { + declared.to_path_buf() + } else { + image_dir.join(declared) + } + } + + /// Whether the guest boots through a bootloader rather than being handed + /// a kernel. + pub fn uses_bootloader(&self) -> bool { + self.boot.method == "bootloader" + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const MANIFEST: &str = r#"{ + "schemaVersion": 1, + "system": "x86_64-linux", + "nodeShape": "daemon", + "image": {"disk": "disk.qcow2", "diskFormat": "qcow2", "diskSizeMib": 8192, "systemImage": null}, + "machine": {"cores": 3, "memorySizeMib": 3072}, + "boot": {"method": "direct", "kernel": "kernel", "initrd": "initrd", "append": "console=ttyS0"}, + "drives": [ + {"name": "root", "file": "disk.qcow2", "format": null, "cache": "writeback", + "werror": "report", "bootIndex": "1", "serial": "root", "interface": "virtio"} + ], + "extraOptions": ["-device", "virtio-keyboard", "-usb"], + "networkingOptions": ["-net nic,netdev=user.0,model=virtio", "-netdev user,id=user.0,"], + "sharedDirectories": [ + {"mountTag": "nix-store", "securityModel": "none", "target": "/nix/.ro-store", "source": "/nix/store"} + ], + "activation": { + "readiness": "console-marker", + "contract": "d2b-daemon-acceptance", + "marker": "D2B_LANE_READY", + "serialDevice": "ttyS0", + "acceptanceUnitsFile": "/etc/d2b/daemon-acceptance-units", + "shape": "daemon" + }, + "init": "/nix/store/x-nixos-system/init", + "toplevel": "/nix/store/x-nixos-system", + "hostToolBundle": "/nix/store/y-d2b-bazel-host-tools-0.0.0", + "hostToolInventory": ["d2b", "d2bd"], + "cloudHypervisorController": null + }"#; + + #[test] + fn the_manifest_round_trips_into_the_declared_invocation() { + let manifest: GuestManifest = serde_json::from_str(MANIFEST).expect("the manifest parses"); + assert_eq!(manifest.machine.cores, 3); + assert_eq!(manifest.machine.memory_size_mib, 3072); + assert_eq!(manifest.image.disk_size_mib, 8192); + assert_eq!(manifest.drives[0].file, "disk.qcow2"); + assert!(!manifest.uses_bootloader()); + assert_eq!(manifest.activation.marker, "D2B_LANE_READY"); + assert_eq!( + manifest.networking_options, + vec![ + "-net nic,netdev=user.0,model=virtio".to_owned(), + "-netdev user,id=user.0,".to_owned() + ], + "the node's networking declaration survives into the manifest whole" + ); + } + + #[test] + fn a_manifest_field_the_launcher_does_not_know_is_refused() { + // A manifest that grew a field is a manifest whose meaning the + // launcher cannot honour; guessing is how a lane boots a guest + // shaped like the wrong check. + let drifted = MANIFEST.replace( + "\"cores\": 3", + "\"cores\": 3, \"machineIdentity\": \"whatever\"", + ); + let error = serde_json::from_str::(&drifted) + .expect_err("an unknown field is refused"); + assert!(error.to_string().contains("machineIdentity"), "{error}"); + } + + #[test] + fn an_image_relative_file_resolves_against_the_image_and_a_store_path_does_not() { + let manifest: GuestManifest = serde_json::from_str(MANIFEST).expect("the manifest parses"); + assert_eq!( + manifest.resolve(Path::new("/run/lane/image"), "disk.qcow2"), + Path::new("/run/lane/image/disk.qcow2") + ); + assert_eq!( + manifest.resolve(Path::new("/run/lane/image"), "/nix/store/z-state.img"), + Path::new("/nix/store/z-state.img") + ); + } +} diff --git a/packages/d2b-vm-harness/src/monitor.rs b/packages/d2b-vm-harness/src/monitor.rs new file mode 100644 index 000000000..4ed2ce506 --- /dev/null +++ b/packages/d2b-vm-harness/src/monitor.rs @@ -0,0 +1,315 @@ +//! The emulator's control monitor. +//! +//! The lane needs three things from the emulator that the console cannot +//! give it: to read the block graph the guest attached, to read the guest's +//! own snapshot list, and to ask the emulator to stop. All three are QMP +//! commands, spoken over the unix socket the launcher creates - the same +//! `qmp-socket` readiness the repository's own service-capability table +//! names. + +use std::{ + io::{BufRead, BufReader, Write}, + os::unix::net::UnixStream, + path::Path, +}; + +use serde_json::{Value, json}; + +use crate::error::{HarnessError, Result}; + +/// A connected QMP session. +pub struct Monitor { + reader: BufReader, + writer: UnixStream, + events: Vec, +} + +/// One writable block device the guest attached, as the emulator reports it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BlockDevice { + /// The device's id on the monitor. + pub id: String, + /// The backing file the emulator resolved. + pub file: String, + /// The image format that file is in. + pub format: String, + /// Whether the emulator can store a snapshot inside that file. + pub snapshottable: bool, +} + +impl Monitor { + /// Connect to a monitor socket, read its greeting, and enter command + /// mode. + #[allow(clippy::disallowed_methods, reason = "synchronous path")] + pub fn connect(socket: &Path) -> Result { + let stream = UnixStream::connect(socket).map_err(|error| { + HarnessError::io(format!("connecting to {}", socket.display()), error) + })?; + Self::from_stream(stream) + } + + #[allow(clippy::disallowed_methods, reason = "synchronous path")] + fn from_stream(stream: UnixStream) -> Result { + let mut monitor = Self { + reader: BufReader::new( + stream + .try_clone() + .map_err(|error| HarnessError::io("cloning the monitor socket", error))?, + ), + writer: stream, + events: Vec::new(), + }; + monitor.read_message()?; + monitor.execute("qmp_capabilities")?; + Ok(monitor) + } + + /// Run one QMP command and return its `return` value. + pub fn execute(&mut self, command: &str) -> Result { + self.execute_with(command, json!({})) + } + + /// Run one QMP command with arguments and return its `return` value. + #[allow(clippy::disallowed_methods, reason = "synchronous path")] + pub fn execute_with(&mut self, command: &str, arguments: Value) -> Result { + let request = json!({"execute": command, "arguments": arguments}); + let mut line = serde_json::to_string(&request) + .map_err(|error| HarnessError::Monitor { + command: command.to_owned(), + detail: error.to_string(), + })?; + line.push('\n'); + self.writer + .write_all(line.as_bytes()) + .map_err(|error| HarnessError::io(format!("writing {command} to the monitor"), error))?; + loop { + let message = self.read_message()?; + if let Some(error) = message.get("error") { + return Err(HarnessError::Monitor { + command: command.to_owned(), + detail: error.to_string(), + }); + } + // The monitor interleaves asynchronous events with command + // replies; they carry neither `return` nor `error`, so they are + // held for the caller and the reply is what ends the loop. + if message.get("return").is_some() { + return Ok(message["return"].clone()); + } + self.events.push(message); + } + } + + /// Ask the emulator to exit. The emulator stops the guest and closes the + /// process; the caller waits for it. + pub fn quit(&mut self) -> Result<()> { + self.execute("quit").map(|_| ()) + } + + /// The guest's block graph, one entry per attached device. + pub fn block_devices(&mut self) -> Result> { + let report = self.execute("query-block")?; + Ok(parse_block_devices(&report)) + } + + #[allow(clippy::disallowed_methods, reason = "synchronous path")] + fn read_message(&mut self) -> Result { + let mut line = String::new(); + let read = self + .reader + .read_line(&mut line) + .map_err(|error| HarnessError::io("reading the emulator monitor", error))?; + if read == 0 { + return Err(HarnessError::Monitor { + command: "read".to_owned(), + detail: "the emulator closed the monitor connection".to_owned(), + }); + } + serde_json::from_str(line.trim()).map_err(|error| HarnessError::Monitor { + command: "read".to_owned(), + detail: format!("{error}: {}", line.trim()), + }) + } + + /// Take the events the monitor delivered while a command was in flight. + /// The lane takes them so a guest that stopped itself is noticed rather + /// than waited on. + pub fn take_events(&mut self) -> Vec { + std::mem::take(&mut self.events) + } +} + +/// Turn a `query-block` report into the devices the lane has to judge. +/// +/// A backend with no image behind it - an empty drive, or a device whose +/// medium is not plugged - is skipped: there is nothing to snapshot and +/// nothing to refuse. A read-only node is snapshottable by construction: +/// the emulator excludes it from a snapshot's device set. Every other node +/// has to be qcow2, because in the current emulator qcow2 is the only format +/// that implements the snapshot vtable, and a node that does not is one a +/// later `snapshot-save` refuses - refusing the whole save, not just that +/// node. +fn parse_block_devices(report: &Value) -> Vec { + report + .as_array() + .into_iter() + .flatten() + .filter_map(|entry| { + let inserted = entry.get("inserted")?; + let read_only = inserted + .get("ro") + .and_then(Value::as_bool) + .unwrap_or(false); + let format = inserted + .get("drv") + .and_then(Value::as_str) + .unwrap_or("unknown") + .to_owned(); + Some(BlockDevice { + // An entry the emulator names by qdev path rather than by + // drive id - a device attached after the guest started, for + // one - carries `"device": ""` rather than omitting the key, + // so an empty id has to read as no id, or the refusal names + // nothing. + id: entry + .get("device") + .and_then(Value::as_str) + .filter(|id| !id.is_empty()) + .or_else(|| entry.get("qdev").and_then(Value::as_str)) + .unwrap_or("unnamed") + .to_owned(), + file: inserted + .get("file") + .and_then(Value::as_str) + .unwrap_or("") + .to_owned(), + snapshottable: read_only || format == "qcow2", + format, + }) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + const GREETING: &str = r#"{"QMP": {"version": {"qemu": {"major": 10, "minor": 2, "micro": 2}}, "capabilities": []}}"#; + + /// A monitor wired to a canned emulator, so the protocol handling the + /// lane relies on is exercised through the same code path a live monitor + /// drives. The canned side writes the greeting and every reply up front + /// and then drains, which is what lets a test deliver an unsolicited + /// event - the case a request/response fake cannot express. + fn monitor_answering(replies: &'static [&'static str]) -> Monitor { + let (lane_end, emulator_end) = UnixStream::pair().expect("a socket pair"); + std::thread::spawn(move || { + let mut reader = BufReader::new(emulator_end.try_clone().expect("clone")); + let mut writer = emulator_end; + for reply in std::iter::once(GREETING).chain(replies.iter().copied()) { + writeln!(writer, "{reply}").expect("write a reply"); + } + // Keep reading so the lane never writes into a closed socket. + let mut line = String::new(); + while reader.read_line(&mut line).expect("read a request") > 0 { + line.clear(); + } + }); + Monitor::from_stream(lane_end).expect("the greeting and handshake succeed") + } + + const BLOCK_REPORT: &str = concat!( + r#"{"return": ["#, + r#"{"device": "file-nix-store", "inserted": {"drv": "raw", "file": "/nix/store/s.img", "ro": true}},"#, + r#"{"device": "virtio0", "inserted": {"drv": "raw", "file": "/run/lane/disk.qcow2", "ro": false}},"#, + r#"{"device": "virtio1", "inserted": {"drv": "qcow2", "file": "/run/lane/state.qcow2", "ro": false}},"#, + r#"{"qdev": "ide0-cd0"}"#, + r#"]}"# + ); + + #[test] + fn a_writable_raw_device_is_reported_as_unsnapshottable() { + let mut monitor = monitor_answering(&[r#"{"return": {}}"#, BLOCK_REPORT]); + let devices = monitor.block_devices().expect("the report parses"); + assert_eq!( + devices + .iter() + .filter(|device| !device.snapshottable) + .map(|device| device.id.as_str()) + .collect::>(), + vec!["virtio0"], + "a writable non-qcow2 node is the one a snapshot-save would refuse" + ); + assert_eq!(devices.len(), 3, "a backend with no image is skipped"); + assert_eq!(devices[0].format, "raw", "a read-only backing is judged raw"); + assert!( + devices[0].snapshottable, + "a read-only backing is excluded from a snapshot, so it never blocks one" + ); + assert!(devices[2].snapshottable, "the qcow2 overlay carries the snapshot"); + } + + /// The shape the real emulator reports for a device attached after the + /// guest started: the entry has a `device` key, but it is empty, and the + /// name lives in `qdev`. Read literally the id is the empty string, and + /// the lane's refusal names a device with no name. + const HOTPLUGGED_BLOCK_REPORT: &str = concat!( + r#"{"return": ["#, + r#"{"device": "lane_drive_0", "inserted": {"drv": "qcow2", "file": "/run/lane/disk.qcow2", "ro": false}},"#, + r#"{"device": "", "qdev": "/machine/peripheral/lane-refusal/virtio-backend", "inserted": {"drv": "raw", "file": "/run/lane/refusal.img", "ro": false}}"#, + r#"]}"# + ); + + #[test] + fn a_device_named_only_by_its_qdev_path_is_named_in_the_refusal() { + let mut monitor = monitor_answering(&[r#"{"return": {}}"#, HOTPLUGGED_BLOCK_REPORT]); + let devices = monitor.block_devices().expect("the report parses"); + let refused = devices + .iter() + .find(|device| !device.snapshottable) + .expect("a writable raw node is refused"); + assert_eq!( + refused.id, "/machine/peripheral/lane-refusal/virtio-backend", + "the refusal names the device by its qdev path, not by an empty id" + ); + assert_eq!(refused.file, "/run/lane/refusal.img"); + assert_eq!(refused.format, "raw"); + assert!( + devices[0].snapshottable, + "the qcow2 root drive is not what failed the lane" + ); + } + + #[test] + fn a_monitor_error_becomes_a_named_failure() { + let mut monitor = monitor_answering(&[ + r#"{"return": {}}"#, + r#"{"error": {"class": "GenericError", "desc": "no such command"}}"#, + ]); + let error = monitor + .execute("snapshot-save") + .expect_err("a monitor error is a failure"); + let rendered = error.to_string(); + assert!(rendered.contains("snapshot-save"), "{rendered}"); + assert!(rendered.contains("no such command"), "{rendered}"); + } + + #[test] + fn an_event_between_a_command_and_its_reply_does_not_end_the_command() { + // The emulator stops the guest's CPUs around a snapshot, and the + // resulting STOP/RESUME arrive as events. Reading one as the reply + // would leave the lane waiting on a command that already answered. + let mut monitor = monitor_answering(&[ + r#"{"return": {}}"#, + r#"{"event": "STOP", "data": {}}"#, + r#"{"return": {"status": "running"}}"#, + ]); + let status = monitor + .execute("query-status") + .expect("the reply after an event is the reply"); + assert_eq!(status["status"], "running"); + let events = monitor.take_events(); + assert_eq!(events.len(), 1, "the event is held for the caller"); + assert_eq!(events[0]["event"], "STOP"); + } +} diff --git a/packages/xtask/src/provider_crate_policy.rs b/packages/xtask/src/provider_crate_policy.rs index db22adbbf..2aae8b4dc 100644 --- a/packages/xtask/src/provider_crate_policy.rs +++ b/packages/xtask/src/provider_crate_policy.rs @@ -8984,6 +8984,8 @@ const COMMITTED_SCOPE: &[CommittedScopeEntry] = &[ reason: "the daemon composition root (and its runtime)" }, CommittedScopeEntry { crate_name: "xtask", class: CommittedScopeClass::Tooling, reason: "the check's own home; every U-unit touches the tooling" }, + CommittedScopeEntry { crate_name: "d2b-vm-harness", class: CommittedScopeClass::Tooling, + reason: "the host-integration lane's own harness; the check's tooling, booted against a lane-owned guest" }, CommittedScopeEntry { crate_name: "d2b-broker", class: CommittedScopeClass::Broker, reason: "the broker binary and its composition/fixture support crates" }, CommittedScopeEntry { crate_name: "d2b-broker-composition", class: CommittedScopeClass::Broker, From 7aaff4d0fc096e83f556231106807b86aee4efc1 Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sat, 26 Sep 2026 13:40:40 -0700 Subject: [PATCH 05/51] feat(vm): give the guest a command channel and the legacy control surface The guest image imported nixpkgs' qemu-vm.nix alone, so it had no way to be commanded: no backdoor service, no console, no channel of any kind. It now imports the same test instrumentation the nix lane's guest ran, leaving testing.backdoor at its default. The launcher declares the virtio-serial console on the command line and binds the host socket before spawn, because backdoor.service requires dev-hvc0.device and a unit whose device is absent at boot is not reliably restarted once it appears. That channel lets the harness re-provide the legacy driver's guest control surface, so every check that has not been ported keeps executing its own assertions and keeps gating the lane. The surface is the driver's, down to its refusal strings and its log lines, and the diagnostics prelude is one file both surfaces read, so a ported check and an unported one report identically. A console rather than a forwarded ssh port because no fixture uses ssh, a forwarded port does not survive snapshot restore, and the console needs no sshd, host keys or key auth in the guest. Corrects a doc comment on reserve_loopback_port that claimed the lane forwards the guest's ssh port. It forwards no port and never did. pkgs is bound in guest-image.nix's let rather than read inside laneGuestModule: the module system's own pkgs is resolved by asking the configuration being built, so a module reaching for it during that fixpoint is infinite recursion. --- .../bazel-owned-legacy-check-surface.md | 9 + nix/test-support/guest-image.nix | 27 + packages/d2b-vm-harness/BUILD.bazel | 12 +- packages/d2b-vm-harness/src/diagnostics.py | 147 ++ packages/d2b-vm-harness/src/guest.rs | 127 +- packages/d2b-vm-harness/src/legacy.rs | 1580 +++++++++++++++++ packages/d2b-vm-harness/src/legacy_bridge.py | 209 +++ packages/d2b-vm-harness/src/lib.rs | 2 + tests/host-integration/lib.nix | 138 +- 9 files changed, 2117 insertions(+), 134 deletions(-) create mode 100644 changelog.d/bazel-owned-legacy-check-surface.md create mode 100644 packages/d2b-vm-harness/src/diagnostics.py create mode 100644 packages/d2b-vm-harness/src/legacy.rs create mode 100644 packages/d2b-vm-harness/src/legacy_bridge.py diff --git a/changelog.d/bazel-owned-legacy-check-surface.md b/changelog.d/bazel-owned-legacy-check-surface.md new file mode 100644 index 000000000..f7e61e56c --- /dev/null +++ b/changelog.d/bazel-owned-legacy-check-surface.md @@ -0,0 +1,9 @@ +### Added + +- Added the host-integration lane's legacy guest-control surface, so a check that has not been ported yet keeps gating the lane with the assertions it already has. An unported check is its fixture's evaluated `testScript` - Python, unchanged - and the surface re-provides what the nix test driver gave it: `machine.execute` with a bounded command timeout, `succeed` and `fail`, `wait_until_succeeds`, `wait_for_unit`, `wait_for_file` and `sleep`, plus the `start_all` every fixture opens with. Each operation carries the driver's own semantics and the driver's own wording - a refused command reads ``command `…` failed (exit code N)``, a timed-out wait reads `action timed out after X seconds (timeout=N)`, a unit that reached `failed` or that is inactive with nothing pending ends the wait at once - because the lane's diagnostics prelude prints that message, and a differently worded refusal is a differently reported failure for the same failure. A wait that runs out also reports the output its last attempt saw, in the driver's own `output:` idiom, which is the observation the prelude exists to recover. +- The guest is reached over the console the legacy driver itself used: a virtio serial console carrying a root shell, spoken to with the driver's own `set -euo pipefail`, base64-framed output and `PIPESTATUS` status. The console is attached through the emulator's monitor, and the guest-side half is any root shell that announces itself with the line the driver waited for - `Spawning backdoor root shell...` - and reads commands from that console, which is what the nix test framework's `backdoor.service` is. Nothing else is required of the guest: no ssh client sits between a check's assertion and the command it ran, and a console is not a forwarded port that a snapshot and a restore would move. A guest that runs no such shell is reported by name, with what its console should have said, instead of leaving the lane waiting on a connection that cannot arrive. +- The operations live in the lane's harness rather than in the Python that calls them. The check's script is executed by an interpreter, but every operation it calls is a request to the harness, which carries it out against the guest, so the shell a command runs under, the retry bounds and the wording of a refusal are implemented once - and a check that ports to Rust reports through the same surface the unported Python ones report through. + +### Changed + +- The fixture diagnostics prelude (issue #513) moved out of `tests/host-integration/lib.nix` into `packages/d2b-vm-harness/src/diagnostics.py`, which `lib.nix` now reads and the lane's assertion surface carries. It is the one copy both lanes interpolate: the nix lane reaches it through the fixture's evaluated `testScript`, and the Bazel lane runs that same script, so a failing unported check reports its stage, its row dumps, its unit journals and its zone debug dump through the same text under either lane. A second copy would be a second dialect of the same diagnostics, and the two would drift the first time one of them gained a helper the other did not. diff --git a/nix/test-support/guest-image.nix b/nix/test-support/guest-image.nix index 3dcee49d5..78552b904 100644 --- a/nix/test-support/guest-image.nix +++ b/nix/test-support/guest-image.nix @@ -68,6 +68,15 @@ let inherit (pkgs) lib; }; + # The nixpkgs module that carries the lane's command channel, named here + # rather than inside the module that imports it. The module system's own + # `pkgs` argument is resolved by asking the configuration it is building, + # so a module whose `imports` reaches for that argument is a module read + # while the fixpoint that reads it is still being computed. This binding + # sees the package set this file was called with instead, which is the + # same pinned set the guest closure is realized from. + testInstrumentation = pkgs.path + "/nixos/modules/testing/test-instrumentation.nix"; + # `d2bDaemonNode` declares `virtualisation.*`, so the guest is evaluated # with the same QEMU VM module the runNixOSTest nodes carry. Evaluating # the node module directly, rather than through the test driver, is what @@ -249,6 +258,24 @@ let laneGuestModule = { config, lib, pkgs, ... }: { + # The lane's command channel is nixpkgs' own test instrumentation, the + # module the nix lane's guests got when the test driver booted them. + # It is imported here rather than reimplemented because the unit it + # declares is the whole channel: `backdoor.service` is a root shell on + # `/dev/hvc0` - a virtio serial console - and it announces itself with + # the greeting the lane's guest-control surface waits for before it + # sends anything. Writing that unit here would be a second greeting and + # a second shell for the same channel. + # + # `testing.backdoor` is left at its default, which is + # `!config.boot.isContainer` and therefore true for a guest that was + # booted, so the unit is declared rather than switched on. What the + # module also brings - a root password for an interactive login, no + # default gateway, the journal forwarded to the serial console - is + # what these guests have always had, and the checks were written + # against that guest rather than against a network-reachable one. + imports = [ testInstrumentation ]; + # The direct-boot shape gets its serial console from the `-append` the # VM module builds. The bootloader shape reads its command line off # the disk instead, so the same console list is declared as kernel diff --git a/packages/d2b-vm-harness/BUILD.bazel b/packages/d2b-vm-harness/BUILD.bazel index 226ccc55a..e00f5b574 100644 --- a/packages/d2b-vm-harness/BUILD.bazel +++ b/packages/d2b-vm-harness/BUILD.bazel @@ -25,7 +25,17 @@ d2b_rust_library( exclude = ["src/bin/**/*.rs"], allow_empty = True, ), - compile_data = ["Cargo.toml"], + # The two Python files the legacy guest-control surface carries, declared + # as compile-time data because that is what embeds them: the diagnostics + # prelude is the one copy the nix lane's fixtures interpolate as well, and + # the bridge is the `machine` object an unported check's assertions call. + # A check cannot be handed a lane whose diagnostics text differs from the + # one its fixture was evaluated with. + compile_data = [ + "Cargo.toml", + "src/diagnostics.py", + "src/legacy_bridge.py", + ], deps = all_crate_deps(normal = True, cargo_only = True), ) diff --git a/packages/d2b-vm-harness/src/diagnostics.py b/packages/d2b-vm-harness/src/diagnostics.py new file mode 100644 index 000000000..b998d4398 --- /dev/null +++ b/packages/d2b-vm-harness/src/diagnostics.py @@ -0,0 +1,147 @@ +# The d2b host-integration lane's fixture diagnostics prelude. +# +# This is the one copy of the prelude a check's `testScript` opens with, and +# it is shared deliberately rather than re-provided per surface. The nix lane +# keeps running the fixtures it already has, so the prelude reaches those +# checks by being interpolated into their evaluated `testScript` +# (`tests/host-integration/lib.nix` reads this file); the Bazel lane runs the +# very same evaluated script, so its legacy guest-control surface and the +# ported Rust checks that follow it report through this text rather than +# through a second copy of it that would drift from the first. A check whose +# diagnostics can be read the same way before and after its port is a check +# whose port is reviewable. +# +# The helpers are diagnostics only: no assertion and no timeout declared +# here changes any of them. The lane's guest-control surface +# (`packages/d2b-vm-harness/src/legacy.rs`) owns what `machine.*` means, and +# the failure path below is what makes a failed check legible - the stage it +# was in, the rows it was asserting on, the journal lines that explain them, +# and the zone's own account of the row that did not settle. + +# ---- d2b fixture diagnostics (issue #513) -------------------------- +# The test driver discards machine.execute output and does not re-print +# the output a timed-out wait_until_succeeds last saw, so a failed lane +# used to leave only the command text in the log. These helpers push the +# row set and the daemon explanation lines into the driver log (stdout +# and stderr of the test driver, that is the lane log). +# +# Diagnostics only: no assertion and no timeout is changed here. +import time as _diag_time + +_diag_t0 = _diag_time.monotonic() +_diag_stage = "startup" + +def _diag_elapsed(): + return f"{_diag_time.monotonic() - _diag_t0:.1f}s" + +def _diag_print(*lines): + for line in lines: + print(line, flush=True) + +def stage(name): + global _diag_stage + _diag_stage = name + _diag_print(f"[d2b] stage={name} t={_diag_elapsed()}") + +def diag(command, label="diagnostic output"): + try: + status, output = machine.execute(command, timeout=120) + except Exception as error: + _diag_print( + f"[d2b] stage={_diag_stage} t={_diag_elapsed()} {label}: " + f"diagnostic command failed: {error}" + ) + return -1 + _diag_print( + f"[d2b] stage={_diag_stage} t={_diag_elapsed()} {label} " + f"(exit {status}):" + ) + _diag_print(command) + for line in output.rstrip().splitlines(): + _diag_print(" " + line) + return status + +def _diag_journal(unit, token): + scope = f"-u {unit} " if unit else "" + select = f"| grep -F -- {token!r} " if token else "" + return ( + f"journalctl {scope}--no-pager -o cat -b -n 4000 2>/dev/null " + f"{select}| tail -n 60 || true" + ) + +def unit_dumps(unit): + """Row dumps for a systemd unit waiting to become active.""" + return [ + ( + f"{unit} status", + f"systemctl status {unit} --no-pager 2>&1 | tail -n 40 " + "|| true", + ), + ] + +# Every fixture drives one zone as one linux user through the same +# public socket, so the composed explanation is available without each +# stage listing the rows it asserted on: `d2b debug` reads the whole +# zone and prints the ownership tree, the row that is not settled, and +# the structured failure behind it. +_diag_zone = "work" +_diag_user = "alice" + +def diag_debug_zone(label="zone explanation"): + """The composed `d2b debug` report, always diagnostic and never + fatal: a failure that happened before the daemon was reachable must + still print its own stage rather than a diagnostic error. Bounded, + because a failure can happen before there is anything to explain.""" + status = diag( + f"runuser -u {_diag_user} -- env " + f"D2B_PUBLIC_SOCKET=/run/d2b/public.sock " + f"timeout 60 d2b --zone {_diag_zone} debug {_diag_zone} 2>&1 " + f"|| true", + label, + ) + return status + +def diag_step(name, action, rows=(), explain=(), wait=None, debug=True): + stage(name) + try: + return action() + except Exception as error: + labels = ", ".join(label for label, _ in rows) or "none" + failing = f" wait={name}" if wait else "" + _diag_print( + f"[d2b] FAIL stage={name} t={_diag_elapsed()}{failing} " + f"rows=[{labels}]: {error}" + ) + if wait: + _diag_print(f"[d2b] failing wait: {wait}") + for label, command in rows: + diag(command, f"row dump: {label}") + for unit, token in explain: + detail = f"journal {unit or 'all'}" + if token: + detail += f" lines matching {token!r}" + diag(_diag_journal(unit, token), detail) + if debug: + diag_debug_zone() + raise + +def diag_unit(name, unit, timeout, debug=True): + """wait_for_unit with the unit status and journal on timeout.""" + return diag_step( + name, + lambda: machine.wait_for_unit(unit, timeout=timeout), + unit_dumps(unit), + [(unit, None)], + debug=debug, + ) + +def diag_wait(name, command, timeout, rows=(), explain=(), debug=True): + return diag_step( + name, + lambda: machine.wait_until_succeeds(command, timeout=timeout), + rows, + explain, + command, + debug=debug, + ) + diff --git a/packages/d2b-vm-harness/src/guest.rs b/packages/d2b-vm-harness/src/guest.rs index 63883fe14..29a6f5d6f 100644 --- a/packages/d2b-vm-harness/src/guest.rs +++ b/packages/d2b-vm-harness/src/guest.rs @@ -29,6 +29,7 @@ use std::{ io::Write, net::TcpListener, os::unix::fs::PermissionsExt, + os::unix::net::UnixListener, path::{Path, PathBuf}, process::{Child, Command, Stdio}, thread::sleep, @@ -50,6 +51,19 @@ const WORK_DIR_PATH_BUDGET: usize = 88; /// cannot carry a socket. const LANE_TEMP_DIR: &str = "d2b-vm-lane"; +/// The guest's command channel: the chardev the launch declares it under, +/// and the socket file that chardev connects to, in the working directory +/// the guest owns. +/// +/// These are the launcher's, not a caller's: the console is part of the +/// invocation, so the socket path is a function of the guest's own working +/// directory and the chardev id is a name the command line and the guest's +/// surface have to agree on. The guest side of the channel is the nix test +/// framework's `backdoor.service` - a root shell on this console - which +/// the guest image carries. +pub const CONSOLE_ID: &str = "d2b-lane-console"; +pub const CONSOLE_SOCKET: &str = "lane-console.sock"; + /// How often the launcher re-reads the guest's console while waiting for /// activation. const CONSOLE_POLL: Duration = Duration::from_millis(500); @@ -189,8 +203,19 @@ impl GuestSpec { } // The lane's own plumbing: no display, the guest's console captured - // for the activation wait and for diagnostics, and a monitor socket - // the launcher owns. + // for the activation wait and for diagnostics, a monitor socket the + // launcher owns, and the guest's command channel. + // + // The channel is on the command line for the same reason the nix test + // driver put it there. Its guest side is a unit that declares + // `requires = [ "dev-hvc0.device" ]`, and a unit whose device is + // absent when systemd reaches it is not reliably restarted when the + // device turns up later: a console hot-attached through the monitor + // after the boot leaves a guest whose root shell never ran. So the + // device exists before the guest executes its first instruction, and + // the host is already listening before the emulator starts at all - + // a chardev with nobody on the other end of its socket blocks the + // launch rather than reaching the guest. push("-display"); push("none"); push("-serial"); @@ -203,6 +228,16 @@ impl GuestSpec { work_dir.join("qmp.sock").display() )); + push("-chardev"); + push(&format!( + "socket,id={CONSOLE_ID},path={}", + work_dir.join(CONSOLE_SOCKET).display() + )); + push("-device"); + push("virtio-serial"); + push("-device"); + push(&format!("virtconsole,chardev={CONSOLE_ID}")); + // The guest's clock follows the emulator's virtual clock, which // stops while the guest does. That is what keeps a restored guest's // view of elapsed time continuous across a restore. @@ -424,11 +459,30 @@ fn resolve_share(source: &str, work_dir: &Path) -> Result { .into_owned()) } +/// Bind the host end of the guest's command channel. +/// +/// The working directory is created by the launch that precedes this, so a +/// socket file still there is the working directory's own: it is removed +/// before the bind rather than around it, because a bind that fails on a +/// stale file would be a launch that failed for a reason its own diagnostics +/// do not name. +#[allow(clippy::disallowed_methods, reason = "synchronous path")] +fn bind_command_channel(work_dir: &Path) -> Result { + let path = work_dir.join(CONSOLE_SOCKET); + if path.exists() { + fs::remove_file(&path) + .map_err(|error| HarnessError::io(format!("removing {}", path.display()), error))?; + } + UnixListener::bind(&path) + .map_err(|error| HarnessError::io(format!("binding {}", path.display()), error)) +} + /// A booted guest that has reported its activation contract. pub struct ActiveGuest { child: Child, monitor: Option, work_dir: PathBuf, + command_channel: Option, shut_down: bool, } @@ -448,6 +502,25 @@ impl ActiveGuest { &self.work_dir } + /// Hand this guest's command channel to whoever will run a check's + /// assertions against it. + /// + /// The socket was bound and put on the launch's command line before the + /// emulator was started, and the emulator has connected to it since; what + /// is left is accepting the connection and waiting for the guest's shell + /// to announce itself. It is taken rather than borrowed because one guest + /// carries one command channel, and a second reader of it would interleave + /// two commands' output into one block. + #[allow(clippy::disallowed_methods, reason = "synchronous path")] + pub fn take_command_channel(&mut self) -> Result { + self.command_channel.take().ok_or_else(|| { + HarnessError::Configuration( + "the guest's command channel has already been handed to a guest-control surface" + .to_owned(), + ) + }) + } + /// Refuse a guest whose attached writable devices cannot carry an /// internal snapshot. /// @@ -525,6 +598,11 @@ impl Drop for ActiveGuest { #[allow(clippy::disallowed_methods, reason = "synchronous path")] pub fn boot(spec: &GuestSpec) -> Result { let work_dir = spec.prepare_work_dir()?; + // The host end of the guest's command channel, bound and listening + // before the emulator exists: the chardev the command line declares + // connects here, and a socket with nobody listening on it is a launch + // that blocks at machine start rather than a guest that boots. + let command_channel = bind_command_channel(&work_dir)?; let command_line = spec.command_line()?; let program = command_line .first() @@ -555,6 +633,7 @@ pub fn boot(spec: &GuestSpec) -> Result { child, monitor: Some(monitor), work_dir, + command_channel: Some(command_channel), shut_down: false, }), Err(error) => { @@ -802,9 +881,18 @@ fn wait_for_exit(child: &mut Child, bound: Duration) -> Result<()> { } } -/// Reserve a loopback port for a forwarded guest port. The lane forwards the -/// guest's ssh port so a check's assertions have the same reach the nix test -/// driver gave them. +/// Reserve a loopback port and hand back its number, with nothing listening +/// on it once this returns. +/// +/// This is a convenience for a caller that is about to bind that port itself, +/// so it never has to choose one and race another guest for it. The lane's +/// own guest-control channel is not one of those ports: a check's assertions +/// reach the guest over the virtio serial console the launch declared, not +/// over a forwarded TCP port, so the guest's ssh capability - which the +/// guest node declares on its own and the lane does not depend on - is +/// neither reached nor forwarded here. Nothing in the lane calls this today; +/// it is the launcher's small piece of the vocabulary a caller would use to +/// add a host-side port of its own. pub fn reserve_loopback_port() -> Result { let listener = TcpListener::bind("127.0.0.1:0") .map_err(|error| HarnessError::io("reserving a loopback port", error))?; @@ -1144,6 +1232,35 @@ mod tests { ); } + #[test] + fn the_guests_command_channel_is_declared_on_the_command_line() { + // The guest's root shell is a unit that requires `dev-hvc0.device`, + // and a unit whose device is absent when systemd reaches it is not + // reliably restarted when the device appears later. A channel added + // to a running guest therefore leaves a guest with no shell at all, + // so the device and the socket the host listens on both belong to the + // launch rather than to whatever drives the booted guest. + let argv = argv(&spec(manifest(3, 3072, &[]))); + assert_eq!( + value_of(&argv, "-chardev"), + "socket,id=d2b-lane-console,path=/run/lane/work/lane-member-0/lane-console.sock" + ); + let devices: Vec<&str> = argv + .iter() + .zip(argv.iter().skip(1)) + .filter(|(flag, _)| flag.as_str() == "-device") + .map(|(_, value)| value.as_str()) + .collect(); + assert!( + devices.contains(&"virtio-serial"), + "the console's bus is attached: {devices:?}" + ); + assert!( + devices.contains(&"virtconsole,chardev=d2b-lane-console"), + "the console is attached to the channel the launch declared: {devices:?}" + ); + } + #[test] fn a_console_carrying_the_marker_is_the_activation_signal() { let console = "[ 2.113456] systemd: Reached target multi-user\n\ diff --git a/packages/d2b-vm-harness/src/legacy.rs b/packages/d2b-vm-harness/src/legacy.rs new file mode 100644 index 000000000..62cce7b84 --- /dev/null +++ b/packages/d2b-vm-harness/src/legacy.rs @@ -0,0 +1,1580 @@ +//! The legacy driver guest-control surface. +//! +//! A check that has not been ported yet is a Python script and has to keep +//! gating the lane, so the lane owes that script the same guest it had: the +//! `machine` object the nix test driver gave it, and the diagnostics prelude +//! its `testScript` opens with. This module is the first of those and the +//! crate owns the second, in one file ([`DIAGNOSTICS`]) that the nix lane's +//! fixtures interpolate as well, so a check's failure reads the same before +//! and after its port. +//! +//! The shape of the surface is the driver's, deliberately. Every operation a +//! fixture calls is here with the driver's semantics: `execute` runs a +//! command under `set -euo pipefail` with the bound the check declared, +//! `succeed` and `fail` are the two assertions on its exit status, +//! `wait_until_succeeds` retries it on the driver's interval, `wait_for_unit` +//! and `wait_for_file` are the two service and file waits, and each one +//! reports in the driver's own words - a refused assertion reads +//! `command \`…\` failed (exit code N)` and a timed-out wait reads +//! `action timed out after X seconds (timeout=N)`, because the lane's +//! diagnostics prelude prints that message and a differently worded refusal +//! is a differently reported failure for the same failure. +//! +//! Two things are worth naming about how it is built. +//! +//! * The guest is reached over the console the legacy driver itself used: a +//! virtio serial console carrying a root shell, spoken to with the driver's +//! own base64-and-`PIPESTATUS` framing. Nothing else is required of the +//! guest: a guest that runs the nix test framework's `backdoor.service` (or +//! any root shell that announces itself with the line below and reads +//! commands from that console) is a guest this surface can drive. It is not +//! a network port, so it survives a snapshot and a restore, and it does not +//! put an ssh client between a check's assertion and the command it ran. +//! * The operations live here rather than in the Python that calls them. The +//! check's script is executed by a Python interpreter, but every operation +//! it calls is a request to this module, so the semantics of an operation - +//! the shell a command runs under, the retry bounds, the wording of a +//! refusal - are implemented once and are the same whether the caller is an +//! unported Python check or a ported Rust one. + +use std::{ + collections::BTreeMap, + env, + fmt, fs, + io::{self, BufRead, Read, Write}, + os::unix::net::{UnixListener, UnixStream}, + path::{Path, PathBuf}, + process::{Child, Command, Stdio}, + thread, + time::{Duration, Instant}, +}; + +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +use crate::{ + error::{HarnessError, Result}, + guest::{ActiveGuest, CONSOLE_ID, report}, +}; + +/// The lane's fixture diagnostics prelude, in the one place it lives. +/// +/// The nix lane interpolates this same file into every fixture's evaluated +/// `testScript` (`tests/host-integration/lib.nix` reads it), and the lane runs +/// that evaluated script, so the diagnostics an unported check prints are +/// printed from this text rather than from a second copy of it. A check that +/// ports keeps reporting through it. +pub const DIAGNOSTICS: &str = include_str!("diagnostics.py"); + +/// The Python side of the surface: the `machine` object an unported check's +/// assertions call, and nothing else. +const BRIDGE: &str = include_str!("legacy_bridge.py"); + +/// The line the guest's root shell announces itself with, exactly as the nix +/// test framework's backdoor service announces it. The greeting is part of +/// the protocol rather than a courtesy: the driver waits for it before it +/// sends anything, because the console is a shell that may still be sourcing +/// a profile when the connection lands. +const SHELL_GREETING: &str = "Spawning backdoor root shell..."; + +/// How long the guest's shell has to announce itself after the console is +/// attached. The guest has already reported activation before the surface +/// attaches, so this bound is about the console rather than about the boot. +const SHELL_GREETING_TIMEOUT: Duration = Duration::from_secs(300); + +/// The interval between attempts of a retrying operation, the driver's own. +const RETRY_INTERVAL: Duration = Duration::from_secs(1); + +/// The bound `execute` applies when a caller names none - which is every +/// `wait_for_file` attempt, because the driver reached `execute` through the +/// default rather than through a declared one. Restated here because the +/// Python side resolves the same default for the calls it forwards, and two +/// declarations of the driver's default would be two chances to disagree. +const EXECUTE_DEFAULT_TIMEOUT: u64 = 900; + +/// How often a check's connection to the harness is looked for, while +/// refusing to block on a check that has already exited. +const ACCEPT_POLL: Duration = Duration::from_millis(20); + +/// The environment variable that names the interpreter a check's script runs +/// under, for a lane that wants a specific one. The lane's test target sets +/// it to the interpreter declared as a runfile; without it the interpreter is +/// resolved from the runfiles tree, and failing that from `PATH`. +const PYTHON: &str = "D2B_VM_HARNESS_PYTHON"; + +/// The smallest read bound the console is given while it waits for its +/// shell. A bound of zero is not a bound at all on a socket: it means wait +/// forever. +const MINIMUM_READ_BOUND: Duration = Duration::from_millis(1); + +/// One check that has not been ported: its name, and its evaluated +/// `testScript` - the fixture's own assertions with the shared diagnostics +/// prelude already interpolated at the top. +/// +/// The script is whatever the fixture evaluated to, not the fixture file: the +/// lane reads the check out of the same evaluation that produced its guest +/// image, so the assertions a check runs here are the assertions it runs +/// under the driver being retired. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LegacyCheck { + /// The check's name, as it appears in the lane's own results. + pub name: String, + /// The check's script. + pub script: String, +} + +impl LegacyCheck { + /// Take a check as its name and its evaluated script. + pub fn new(name: impl Into, script: impl Into) -> Self { + Self { + name: name.into(), + script: script.into(), + } + } +} + +/// What one unported check produced. +/// +/// The diagnostics themselves are already in the lane's report by the time +/// this is returned - the check's own output and the surface's log lines are +/// streamed as they happen, in the order they happened - and `detail` carries +/// them again for the caller that files them under this check's result. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LegacyOutcome { + /// The check's name. + pub name: String, + /// Whether the check's script finished without a failure. + pub passed: bool, + /// Everything the check printed, and everything the surface reported + /// around it. + pub detail: String, +} + +/// Why a guest-control operation did not produce a result. +/// +/// The two cases are kept apart because they are the check's verdict and the +/// lane's, and an unported check is the only one that can fix the first: a +/// refused assertion is the check's own failure and travels to the check's +/// script, while a guest the surface could not reach is a lane failure that +/// no assertion in the check caused. +#[derive(Debug)] +pub enum LegacyError { + /// The guest could not be reached, or answered something unreadable. + Guest(HarnessError), + /// An assertion the check made did not hold. + Assertion(String), +} + +impl fmt::Display for LegacyError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Guest(error) => write!(formatter, "{error}"), + Self::Assertion(message) => formatter.write_str(message), + } + } +} + +impl std::error::Error for LegacyError {} + +impl From for LegacyError { + fn from(error: HarnessError) -> Self { + Self::Guest(error) + } +} + +/// The guest-control surface's own result type. +pub type LegacyResult = std::result::Result; + +/// What one command did in the guest. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CommandResult { + /// The command's exit status. + pub status: i32, + /// Everything it wrote, on either stream. + pub output: String, +} + +/// Quote a string the way the guest's shell needs it quoted. +/// +/// The rules are the interpreter's own rather than a shell's, because the +/// string is handed to `bash -c` as one argument and a different rule would +/// change what the guest runs for a command containing a quote: an ASCII +/// string of unreserved characters passes through, and anything else is +/// single-quoted with an embedded quote closed, double-quoted, and reopened. +fn shlex_quote(text: &str) -> String { + if text.is_empty() { + return "''".to_owned(); + } + let unreserved = |byte: u8| { + byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'%' | b'+' | b',' | b'-' | b'.' | b'/' | b':' | b'=' | b'@') + }; + if text.is_ascii() && text.bytes().all(unreserved) { + return text.to_owned(); + } + format!("'{}'", text.replace('\'', "'\"'\"'")) +} + +/// Decode one base64 block, the framing the guest's shell answers with. +/// +/// The block is ASCII by construction, so the decode is a table lookup per +/// character and a shift-and-or per four. Whitespace is skipped rather than +/// rejected: the framing wraps at the shell's own line width on some +/// guests, and padding is what ends a well-formed block. +fn base64_decode(block: &str) -> std::result::Result, HarnessError> { + const INVALID: u8 = u8::MAX; + let value = |byte: u8| -> u8 { + match byte { + b'A'..=b'Z' => byte - b'A', + b'a'..=b'z' => byte - b'a' + 26, + b'0'..=b'9' => byte - b'0' + 52, + b'+' => 62, + b'/' => 63, + _ => INVALID, + } + }; + let mut out = Vec::with_capacity(block.len() / 4 * 3); + let mut accumulator: u32 = 0; + let mut bits: u32 = 0; + for byte in block.bytes() { + match byte { + b'=' | b'\n' | b'\r' | b' ' | b'\t' => continue, + byte => { + let digit = value(byte); + if digit == INVALID { + return Err(HarnessError::Configuration( + "the guest answered a command with output that is not base64".to_owned(), + )); + } + accumulator = (accumulator << 6) | u32::from(digit); + bits += 6; + if bits >= 8 { + bits -= 8; + out.push(((accumulator >> bits) & 0xff) as u8); + } + } + } + } + Ok(out) +} + +/// The guest-side shell, spoken to the way the legacy driver spoke to it. +/// +/// One connection, one request at a time, which is the shape the driver had: +/// a check's assertions are sequential, and a shell shared by concurrent +/// readers would interleave two commands' output into one block. +struct Console { + reader: io::BufReader, + writer: UnixStream, + work_dir: PathBuf, +} + +impl Console { + /// Take a booted guest's command channel and wait for its shell. + /// + /// The channel is already declared on the guest's launch and the host end + /// of it is already listening, because the guest's half is a unit that + /// requires `dev-hvc0.device` to exist before the guest boots: a console + /// added to a running guest would leave a guest whose root shell systemd + /// never started. What is left here is accepting the connection the + /// emulator made and waiting for the shell, which has to be running by + /// the time the surface finishes attaching - hence a bounded wait that + /// names what it was waiting for. + fn attach(guest: &mut ActiveGuest) -> Result { + let work_dir = guest.work_dir().to_path_buf(); + let listener = guest.take_command_channel()?; + let stream = accept(&listener, None)?; + let mut console = Self { + reader: io::BufReader::new( + stream + .try_clone() + .map_err(|error| HarnessError::io("cloning the console socket", error))?, + ), + writer: stream, + work_dir, + }; + console.await_shell(SHELL_GREETING_TIMEOUT)?; + report(&format!( + "the lane's guest-control console is attached to the guest's {CONSOLE_ID} chardev" + )); + Ok(console) + } + + /// Take a surface over a console that is already being served. + /// + /// The lane attaches a console to a guest it booted; a test stands one up + /// in place of a guest, over a socket pair, and drives the same + /// operations against it. + #[cfg(test)] + fn serving(stream: UnixStream, work_dir: PathBuf) -> Self { + Self { + reader: io::BufReader::new( + stream + .try_clone() + .expect("a console socket can be cloned for reading"), + ), + writer: stream, + work_dir, + } + } + + /// Wait for the guest's shell to announce itself. + /// + /// The greeting is what tells the surface the shell is reading the + /// console rather than still being set up on it, so a console that never + /// greets is a guest with no shell service rather than a slow one. The + /// bound is therefore applied to the reads as well as to the clock: a + /// console with nothing to say is a read that would not return, and + /// bounding the read is what turns the first into the message below + /// rather than into a wait. + #[allow(clippy::disallowed_methods, reason = "synchronous path")] + fn await_shell(&mut self, bound: Duration) -> Result<()> { + self.writer + .set_read_timeout(Some(bound.max(MINIMUM_READ_BOUND))) + .map_err(|error| HarnessError::io("bounding the console's read", error))?; + let mut seen = String::new(); + let start = Instant::now(); + let greeting = loop { + if seen.contains(SHELL_GREETING) { + break true; + } + if start.elapsed() >= bound { + break false; + } + let mut chunk = [0_u8; 4096]; + match self.reader.read(&mut chunk) { + Ok(0) => break false, + Ok(read) => seen.push_str(&String::from_utf8_lossy(&chunk[..read])), + Err(error) + if matches!( + error.kind(), + io::ErrorKind::Interrupted + | io::ErrorKind::WouldBlock + | io::ErrorKind::TimedOut + ) => {} + Err(error) => { + return Err(HarnessError::io("reading the guest's console", error)); + } + } + }; + // A command's own bound is the guest's, and a read that had a bound + // of its own would cut a command off before the guest's bound did. + self.writer + .set_read_timeout(None) + .map_err(|error| HarnessError::io("unbounding the console's read", error))?; + if greeting { + return Ok(()); + } + Err(HarnessError::Configuration(format!( + "the guest's console never announced its root shell with {SHELL_GREETING:?} within {}s, so the lane has no way to run a check's assertions: the guest must run a root shell on a virtio serial console ({SHELL_GREETING}), and it must be running by the time the surface attaches the console", + bound.as_secs() + ))) + } + + /// Run one command in the guest and read back its status and output. + /// + /// The wire form is the driver's, unchanged: the command is run under + /// `set -euo pipefail` so a check's own shell assumptions - a pipeline + /// that fails, an unset variable - fail the way they failed for it, its + /// output is base64-framed so a block with a newline in it is still one + /// block, and its status is read from the pipeline's own `PIPESTATUS` + /// rather than from the status of the framing around it. + #[allow(clippy::disallowed_methods, reason = "synchronous path")] + fn run(&mut self, command: &str, timeout: Option) -> Result { + let deadline = timeout.map(|seconds| format!("timeout {seconds} ")).unwrap_or_default(); + let inner = format!("set -euo pipefail; {command}"); + self.send(&format!( + "{deadline}bash -c {} | (base64 -w 0; echo)\n", + shlex_quote(&inner) + ))?; + let output = base64_decode(self.read_block()?.trim())?; + self.send("echo ${PIPESTATUS[0]}\n")?; + let status = self.read_block()?; + let status = status.trim().parse::().map_err(|error| { + HarnessError::Configuration(format!("the guest answered {status:?} as a status: {error}")) + })?; + Ok(CommandResult { + status, + output: String::from_utf8_lossy(&output).into_owned(), + }) + } + + #[allow(clippy::disallowed_methods, reason = "synchronous path")] + fn send(&mut self, wire: &str) -> Result<()> { + self.writer + .write_all(wire.as_bytes()) + .map_err(|error| HarnessError::io("writing to the guest's console", error)) + } + + /// Read one newline-terminated block from the console. + /// + /// The block ends at the newline the shell's framing adds, which is the + /// only newline in it: the output itself is base64, so it carries none. + /// A read that returns nothing at all is a guest that stopped answering, + /// and is reported as such rather than as an empty output. + #[allow(clippy::disallowed_methods, reason = "synchronous path")] + fn read_block(&mut self) -> Result { + let mut block = String::new(); + let mut chunk = [0_u8; 4096]; + loop { + let read = self + .reader + .read(&mut chunk) + .map_err(|error| HarnessError::io("reading the guest's console", error))?; + if read == 0 { + return Err(HarnessError::Configuration( + "the guest's console closed while a command was running".to_owned(), + )); + } + let decoded = String::from_utf8_lossy(&chunk[..read]); + block.push_str(&decoded); + if decoded.ends_with('\n') { + return Ok(block); + } + } + } +} + +/// Take a connection the lane is waiting for, without waiting forever for one +/// that is never coming. +/// +/// A check's interpreter can die before it asks for the guest at all - the +/// interpreter is missing, the script does not parse - and a blocking accept +/// would then sit on a wait whose only other outcome is the lane's own +/// timeout. The process the connection is expected from is therefore watched +/// while the wait runs, and a process that has already exited ends it. +#[allow(clippy::disallowed_methods, reason = "synchronous path")] +fn accept(listener: &UnixListener, mut check: Option<&mut Child>) -> Result { + listener + .set_nonblocking(true) + .map_err(|error| HarnessError::io("making a lane socket non-blocking", error))?; + loop { + match listener.accept() { + Ok((stream, _)) => return Ok(stream), + Err(error) if error.kind() == io::ErrorKind::WouldBlock => { + if let Some(check) = check.as_deref_mut() + && let Some(status) = check.try_wait().map_err(|error| { + HarnessError::io("asking whether the check's script is still running", error) + })? + { + return Err(HarnessError::Spawn { + detail: format!( + "the check's interpreter exited with {status} before it asked for the guest" + ), + }); + } + thread::sleep(ACCEPT_POLL); + } + Err(error) => return Err(HarnessError::io("accepting a lane connection", error)), + } + } +} + +/// The guest-control surface an unported check's assertions call. +/// +/// Every operation here is one the fixtures call, with the driver's +/// semantics; the log lines around them are the driver's, so a lane log +/// reads the way the lane log read while the driver was the thing driving +/// the guest. +pub struct GuestControl { + console: Console, + notes: String, +} + +impl GuestControl { + /// Run a command in the guest. + /// + /// `timeout` is the bound the command's own execution gets, in seconds; + /// `None` is the driver's unbounded form. The status and the output are + /// returned as they are, and it is the caller's assertion that judges + /// them: `execute` is the one operation here that refuses nothing. + pub fn execute(&mut self, command: &str, timeout: Option) -> LegacyResult { + Ok(self.console.run(command, timeout)?) + } + + /// Run each command in turn, refusing anything that exits non-zero. + /// + /// The outputs are concatenated, as the driver concatenated them, because + /// a check that passes several commands and reads the result is reading + /// one string. + pub fn succeed(&mut self, commands: &[&str], timeout: Option) -> LegacyResult { + let mut output = String::new(); + for command in commands { + let message = format!("must succeed: {command}"); + self.note(&message); + let started = Instant::now(); + let result = self.execute(command, timeout)?; + if result.status != 0 { + self.note(&format!("output: {}", result.output.trim_end())); + return Err(LegacyError::Assertion(format!( + "command `{command}` failed (exit code {})", + result.status + ))); + } + output.push_str(&result.output); + self.finished(&message, started); + } + Ok(output) + } + + /// Run each command in turn, refusing anything that exits zero. + pub fn fail(&mut self, commands: &[&str], timeout: Option) -> LegacyResult { + let mut output = String::new(); + for command in commands { + let message = format!("must fail: {command}"); + self.note(&message); + let started = Instant::now(); + let result = self.execute(command, timeout)?; + if result.status == 0 { + return Err(LegacyError::Assertion(format!( + "command `{command}` unexpectedly succeeded" + ))); + } + output.push_str(&result.output); + self.finished(&message, started); + } + Ok(output) + } + + /// Retry a command until it succeeds, and return the output of the + /// attempt that succeeded. + /// + /// A wait that runs out reports the bound and the driver's message, and + /// then reports the output the last attempt saw - in the driver's own + /// `output:` idiom, the one it logged beside a refused command. That last + /// observation is the reason the lane's diagnostics prelude exists, and + /// losing it again here would put it back where the driver left it. + pub fn wait_until_succeeds(&mut self, command: &str, bound: Duration) -> LegacyResult { + let message = format!("waiting for success: {command}"); + self.note(&message); + let started = Instant::now(); + let mut last = String::new(); + let attempt_bound = Some(bound.as_secs()); + let outcome = self.retry(bound, |control| { + let result = control.execute(command, attempt_bound)?; + last = result.output; + Ok(result.status == 0) + }); + match outcome { + Ok(()) => { + self.finished(&message, started); + Ok(last) + } + Err(error) => { + self.note(&format!("output: {}", last.trim_end())); + Err(error) + } + } + } + + /// Wait until a path exists in the guest. + /// + /// Each attempt is a `test -e` under the default execution bound rather + /// than the wait's own, which is what the driver did: the wait bounds how + /// long the attempts go on for, and each attempt bounds one command. + pub fn wait_for_file(&mut self, path: &str, bound: Duration) -> LegacyResult<()> { + let message = format!("waiting for file '{path}'"); + self.note(&message); + let started = Instant::now(); + let command = format!("test -e {path}"); + let outcome = self.retry(bound, |control| { + Ok(control.execute(&command, Some(EXECUTE_DEFAULT_TIMEOUT))?.status == 0) + }); + if outcome.is_ok() { + self.finished(&message, started); + } + outcome + } + + /// Wait until a systemd unit is active. + /// + /// Two states end the wait early rather than being waited out, because a + /// unit that failed or that is inactive with nothing left to do is not + /// going to become active and the reader of the lane log needs to know + /// that now. The second of the two is the driver's own: "no jobs" is how + /// the guest says it has nothing left in flight for the unit. + pub fn wait_for_unit( + &mut self, + unit: &str, + user: Option<&str>, + bound: Duration, + ) -> LegacyResult<()> { + let message = match user { + Some(user) => format!("waiting for unit {unit} with user {user}"), + None => format!("waiting for unit {unit}"), + }; + self.note(&message); + let started = Instant::now(); + let outcome = self.retry(bound, |control| control.unit_is_active(unit, user)); + if outcome.is_ok() { + self.finished(&message, started); + } + outcome + } + + /// Sleep in guest time, the way the driver's sleep was guest time: the + /// command runs in the guest, so a guest whose clock runs at a different + /// rate still sleeps for the seconds the check asked for. + pub fn sleep(&mut self, seconds: u64) -> LegacyResult<()> { + self.succeed(&[&format!("sleep {seconds}")], None)?; + Ok(()) + } + + /// Whether a unit is active, and the two states that end a wait early. + fn unit_is_active(&mut self, unit: &str, user: Option<&str>) -> LegacyResult { + let state = self.unit_property(unit, "ActiveState", user)?; + if state == "failed" { + return Err(LegacyError::Assertion(format!( + "unit \"{unit}\" reached state \"{state}\"" + ))); + } + if state == "inactive" { + let jobs = self.systemctl("list-jobs --full 2>&1", user)?; + if jobs.output.contains("No jobs") + && self.unit_info(unit, user)?.get("ActiveState") == Some(&state) + { + return Err(LegacyError::Assertion(format!( + "unit \"{unit}\" is inactive and there are no pending jobs" + ))); + } + } + Ok(state == "active") + } + + /// One systemd property of one unit. + fn unit_property( + &mut self, + unit: &str, + property: &str, + user: Option<&str>, + ) -> LegacyResult { + let under_user = match user { + Some(user) => format!(" under user \"{user}\""), + None => String::new(), + }; + let result = self.systemctl( + &format!("--no-pager show \"{unit}\" --property=\"{property}\""), + user, + )?; + if result.status != 0 { + return Err(LegacyError::Assertion(format!( + "retrieving systemctl property \"{property}\" for unit \"{unit}\"{under_user} failed with exit code {}", + result.status + ))); + } + let invalid = || { + LegacyError::Assertion(format!( + "systemctl show --property \"{property}\" \"{unit}\" produced invalid output: {}", + result.output + )) + }; + let first = result.output.split('\n').next().unwrap_or_default(); + let (key, value) = first.split_once('=').ok_or_else(invalid)?; + if key != property { + return Err(invalid()); + } + Ok(value.to_owned()) + } + + /// Every property the guest reports for one unit. + fn unit_info(&mut self, unit: &str, user: Option<&str>) -> LegacyResult> { + let result = self.systemctl(&format!("--no-pager show \"{unit}\""), user)?; + if result.status != 0 { + let under_user = match user { + Some(user) => format!(" under user \"{user}\""), + None => String::new(), + }; + return Err(LegacyError::Assertion(format!( + "retrieving systemctl info for unit \"{unit}\"{under_user} failed with exit code {}", + result.status + ))); + } + Ok(result + .output + .lines() + .filter_map(|line| line.split_once('=')) + .map(|(key, value)| (key.to_owned(), value.to_owned())) + .collect()) + } + + /// Run a `systemctl` query, in the guest's system manager or in a user's. + fn systemctl(&mut self, query: &str, user: Option<&str>) -> LegacyResult { + let Some(user) = user else { + return self.execute(&format!("systemctl {query}"), Some(EXECUTE_DEFAULT_TIMEOUT)); + }; + // The user's query is built by the guest's own shell: `su -l` gives + // the user a login environment, and the quoting keeps a query + // carrying an apostrophe from ending the `$'…'` string early. + let query = query.replace('\'', "\\'"); + self.execute( + &format!( + "su -l {user} --shell /bin/sh -c $'XDG_RUNTIME_DIR=/run/user/`id -u` systemctl --user {query}'" + ), + Some(EXECUTE_DEFAULT_TIMEOUT), + ) + } + + /// Retry an attempt until it is ready, the bound is spent, or it refuses. + /// + /// The bound is spent by the attempts and the intervals between them, and + /// one last attempt is made after it - the driver's arrangement, and the + /// reason a wait that is about to succeed at its bound succeeds rather + /// than failing on an attempt that was never made. + #[allow(clippy::disallowed_methods, reason = "synchronous path")] + fn retry( + &mut self, + bound: Duration, + mut attempt: impl FnMut(&mut Self) -> LegacyResult, + ) -> LegacyResult<()> { + let start = Instant::now(); + while start.elapsed() < bound { + if attempt(self)? { + return Ok(()); + } + thread::sleep(RETRY_INTERVAL); + } + let elapsed = start.elapsed().as_secs_f64(); + if !attempt(self)? { + return Err(LegacyError::Assertion(format!( + "action timed out after {elapsed:.2} seconds (timeout={})", + bound.as_secs() + ))); + } + Ok(()) + } + + /// Report one line the way the driver reported one, into the lane's + /// report and into this check's own record of what it was doing. + fn note(&mut self, line: &str) { + emit_error(&format!("machine: {line}")); + self.notes.push_str(line); + self.notes.push('\n'); + } + + /// The closing line of a logged operation, timed as the driver timed it + /// and reported only when the operation did not refuse. + fn finished(&mut self, message: &str, started: Instant) { + self.note(&format!( + "(finished: {message}, in {:.2} seconds)", + started.elapsed().as_secs_f64() + )); + } + + /// What the surface reported while a check ran, and then stops recording + /// it: the next check's record is its own. + fn take_notes(&mut self) -> String { + std::mem::take(&mut self.notes) + } + + /// Answer one request from a check's script. + fn dispatch(&mut self, request: &Request) -> Reply { + let answered = self.answer(request); + match answered { + Ok(value) => Reply::succeeded(value), + Err(error) => match error { + LegacyError::Assertion(message) => Reply::refused(&message), + LegacyError::Guest(error) => Reply::unreachable(&error.to_string()), + }, + } + } + + /// Carry out one request, or say why it could not be carried out. + fn answer(&mut self, request: &Request) -> LegacyResult { + let timeout = request.seconds("timeout"); + match request.op.as_str() { + "execute" => { + let command = request.text(0, "command")?; + let result = self.execute(&command, timeout)?; + // The driver returned a sentinel status rather than reading + // the block at all for a caller that asked not to. The block + // is read here anyway: a command whose output is left on the + // console is the next command's first line, and a check that + // passes `check_output=False` would be reading it. + if !request.boolean("check_output", true)? { + return Ok(json!([-2, ""])); + } + if !request.boolean("check_return", true)? { + return Ok(json!([-1, result.output])); + } + Ok(json!([result.status, result.output])) + } + "succeed" => { + let commands = request.commands()?; + let borrowed: Vec<&str> = commands.iter().map(String::as_str).collect(); + Ok(Value::String(self.succeed(&borrowed, timeout)?)) + } + "fail" => { + let commands = request.commands()?; + let borrowed: Vec<&str> = commands.iter().map(String::as_str).collect(); + Ok(Value::String(self.fail(&borrowed, timeout)?)) + } + "wait_until_succeeds" => Ok(Value::String( + self.wait_until_succeeds(&request.text(0, "command")?, request.bound("timeout"))?, + )), + "wait_for_file" => { + self.wait_for_file(&request.text(0, "filename")?, request.bound("timeout"))?; + Ok(Value::Null) + } + "wait_for_unit" => { + self.wait_for_unit( + &request.text(0, "unit")?, + request.optional_text("user").as_deref(), + request.bound("timeout"), + )?; + Ok(Value::Null) + } + "sleep" => { + self.sleep(request.number(0, "secs")?)?; + Ok(Value::Null) + } + other => Err(LegacyError::Guest(HarnessError::Configuration(format!( + "a check asked the lane's guest-control surface for {other:?}, which is not one of the operations it re-provides" + )))), + } + } +} + +/// A guest a check's script can be run against, and the surface that runs it. +pub struct LegacyGuest { + control: GuestControl, + work_dir: PathBuf, +} + +impl LegacyGuest { + /// Attach the guest-control console to a booted guest and wait for its + /// shell. + pub fn attach(guest: &mut ActiveGuest) -> Result { + let console = Console::attach(guest)?; + let work_dir = console.work_dir.clone(); + Ok(Self { + control: GuestControl { + console, + notes: String::new(), + }, + work_dir, + }) + } + + /// Run one unported check's script against this guest. + /// + /// The check's script is executed by an interpreter rather than being + /// interpreted by the lane, because the assertions in it are the + /// assertions the check has always made and rewriting them is the port + /// this whole transition is built to make one check at a time. What the + /// script calls is this module: every operation it performs is a request + /// that arrives here and is carried out against the guest, and the log + /// lines around those operations are written as they happen. + pub fn run(&mut self, check: &LegacyCheck) -> Result { + let script = self.work_dir.join("legacy-check.py"); + let control = self.work_dir.join("legacy-control.sock"); + fs::write(&script, &check.script) + .map_err(|error| HarnessError::io(format!("writing {}", script.display()), error))?; + if control.exists() { + fs::remove_file(&control) + .map_err(|error| HarnessError::io(format!("removing {}", control.display()), error))?; + } + let listener = UnixListener::bind(&control) + .map_err(|error| HarnessError::io(format!("binding {}", control.display()), error))?; + let interpreter = interpreter()?; + let mut child = spawn_check(&interpreter, &control, &script)?; + let (served, out, err) = thread::scope(|scope| { + let stdout = child.stdout.take(); + let stderr = child.stderr.take(); + let out = scope.spawn(move || stdout.map_or_else(String::new, |pipe| pump(pipe, false))); + let err = scope.spawn(move || stderr.map_or_else(String::new, |pipe| pump(pipe, true))); + let served = serve(&listener, &mut self.control, &mut child); + let out = out.join().unwrap_or_default(); + let err = err.join().unwrap_or_default(); + (served, out, err) + }); + let status = child + .wait() + .map_err(|error| HarnessError::io("waiting for the check's script", error))?; + served?; + let mut detail = out; + detail.push_str(&err); + detail.push_str(&self.control.take_notes()); + Ok(LegacyOutcome { + name: check.name.clone(), + passed: status.success(), + detail, + }) + } + + /// The working directory this guest's check scripts are written to. + pub fn work_dir(&self) -> &Path { + &self.work_dir + } +} + +/// One request from a check's script. +#[derive(Debug, Deserialize)] +struct Request { + op: String, + #[serde(default)] + args: Vec, + #[serde(default)] + kwargs: BTreeMap, +} + +impl Request { + /// The commands of a `succeed` or a `fail`, which take any number of + /// them and concatenate the outputs. + fn commands(&self) -> LegacyResult> { + self.args + .iter() + .map(|argument| { + argument.as_str().map(str::to_owned).ok_or_else(|| { + LegacyError::Guest(HarnessError::Configuration(format!( + "`{}` was given an argument that is not a command", + self.op + ))) + }) + }) + .collect() + } + + /// A required string argument. + fn text(&self, index: usize, name: &str) -> LegacyResult { + self.args + .get(index) + .and_then(Value::as_str) + .map(str::to_owned) + .ok_or_else(|| self.malformed(name)) + } + + /// An optional string argument. + fn optional_text(&self, name: &str) -> Option { + self.kwargs + .get(name) + .and_then(Value::as_str) + .map(str::to_owned) + } + + /// A required number argument. + fn number(&self, index: usize, name: &str) -> LegacyResult { + self.args + .get(index) + .and_then(Value::as_u64) + .ok_or_else(|| self.malformed(name)) + } + + /// A required boolean argument, with the driver's own default. + fn boolean(&self, name: &str, default: bool) -> LegacyResult { + match self.kwargs.get(name) { + None => Ok(default), + Some(Value::Bool(value)) => Ok(*value), + Some(_) => Err(self.malformed(name)), + } + } + + /// A bound in seconds, where the absence of one means unbounded. + fn seconds(&self, name: &str) -> Option { + self.kwargs + .get(name) + .and_then(Value::as_u64) + } + + /// A bound as a duration, defaulting the way the driver defaulted it. + fn bound(&self, name: &str) -> Duration { + Duration::from_secs(self.seconds(name).unwrap_or(EXECUTE_DEFAULT_TIMEOUT)) + } + + fn malformed(&self, name: &str) -> LegacyError { + LegacyError::Guest(HarnessError::Configuration(format!( + "a check called `{}` without a usable {name}", + self.op + ))) + } +} + +/// One answer to a check's script. +#[derive(Debug, Serialize)] +struct Reply { + ok: bool, + #[serde(skip_serializing_if = "Option::is_none")] + value: Option, + #[serde(skip_serializing_if = "Option::is_none")] + error: Option, + /// Whether the failure was the check's own verdict rather than the + /// lane's, so the script reports it as the assertion it is. + assertion: bool, +} + +impl Reply { + fn succeeded(value: Value) -> Self { + Self { + ok: true, + value: Some(value), + error: None, + assertion: false, + } + } + + fn refused(message: &str) -> Self { + Self { + ok: false, + value: None, + error: Some(message.to_owned()), + assertion: true, + } + } + + fn unreachable(message: &str) -> Self { + Self { + ok: false, + value: None, + error: Some(message.to_owned()), + assertion: false, + } + } +} + +/// Run the check's interpreter, with the bridge and the script it runs. +#[allow(clippy::disallowed_methods, reason = "synchronous path")] +fn spawn_check(interpreter: &Path, control: &Path, script: &Path) -> Result { + let mut command = Command::new(interpreter); + command + // Unbuffered, so the check's own output reaches the lane's report as + // it is printed rather than when the interpreter exits. + .arg("-u") + .arg("-c") + .arg(BRIDGE) + .arg(control) + .arg(script) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + command.spawn().map_err(|error| HarnessError::Spawn { + detail: format!("{}: {error}", interpreter.display()), + }) +} + +/// Answer a check's requests until its script is finished with them. +fn serve( + listener: &UnixListener, + control: &mut GuestControl, + check: &mut Child, +) -> Result<()> { + let stream = accept(listener, Some(check))?; + let reader = io::BufReader::new( + stream + .try_clone() + .map_err(|error| HarnessError::io("cloning the control socket", error))?, + ); + let mut writer = stream; + for line in reader.lines() { + let line = line + .map_err(|error| HarnessError::io("reading a check's request", error))?; + if line.trim().is_empty() { + continue; + } + let reply = match serde_json::from_str::(&line) { + Ok(request) => control.dispatch(&request), + Err(error) => Reply::unreachable(&format!("a check sent a request the surface cannot read: {error}")), + }; + let mut answer = serde_json::to_string(&reply).map_err(|error| { + HarnessError::Configuration(format!("the surface could not answer a request: {error}")) + })?; + answer.push('\n'); + writer + .write_all(answer.as_bytes()) + .map_err(|error| HarnessError::io("answering a check's request", error))?; + writer + .flush() + .map_err(|error| HarnessError::io("answering a check's request", error))?; + } + Ok(()) +} + +/// Read one of the check's streams to its end, reporting it as it arrives. +#[allow(clippy::disallowed_methods, reason = "synchronous path")] +fn pump(pipe: impl io::Read, errors: bool) -> String { + let mut captured = String::new(); + for line in io::BufReader::new(pipe).lines().map_while(std::result::Result::ok) { + if errors { + emit_error(&line); + } else { + report(&line); + } + captured.push_str(&line); + captured.push('\n'); + } + captured +} + +/// Write one line to the lane's report. +fn emit_error(line: &str) { + let mut stderr = io::stderr().lock(); + let _ = writeln!(stderr, "{line}"); + let _ = stderr.flush(); +} + +/// The interpreter a check's script runs under. +/// +/// A lane that pins one names it, which is the hermetic answer and the one +/// the lane's test target uses: the interpreter is a declared runfile rather +/// than whatever `python3` a developer's shell happens to resolve. Without +/// that, the runfiles tree is searched for the declared interpreter, and a +/// contributor running the harness outside Bazel falls back to `PATH`. +fn interpreter() -> Result { + if let Some(named) = env::var_os(PYTHON) { + return Ok(PathBuf::from(named)); + } + for root in [env::var_os("RUNFILES_DIR"), env::var_os("TEST_SRCDIR")] + .into_iter() + .flatten() + { + for candidate in ["python3/bin/python3", "python3+/bin/python3"] { + let path = PathBuf::from(&root).join(candidate); + if path.is_file() { + return Ok(path); + } + } + } + Ok(PathBuf::from("python3")) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A guest that answers the console from a fixed script, and records + /// what it was asked. + /// + /// It speaks the same wire form the guest's shell does, so a test + /// exercises the real protocol rather than a stand-in for it: the + /// command the surface sent, a base64 block, the status request, the + /// status. + fn guest(stream: UnixStream, answers: Vec<(i32, String)>) -> (Vec, Vec) { + let mut reader = io::BufReader::new( + stream + .try_clone() + .expect("a console socket can be cloned for reading"), + ); + let mut writer = stream; + let mut asked = Vec::new(); + let mut statuses = Vec::new(); + for (status, output) in answers { + let mut line = String::new(); + reader.read_line(&mut line).expect("the command line"); + let _ = writer.write_all(format!("{output}\n").as_bytes()); + let mut request = String::new(); + reader.read_line(&mut request).expect("the status request"); + let _ = writer.write_all(format!("{status}\n").as_bytes()); + asked.push(line); + statuses.push(request); + } + (asked, statuses) + } + + /// One answer's worth of base64, the way the guest's shell frames it. + fn block(output: &str) -> String { + const ALPHABET: &[u8; 64] = + b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + let mut encoded = String::new(); + for chunk in output.as_bytes().chunks(3) { + let group = [ + chunk[0], + chunk.get(1).copied().unwrap_or(0), + chunk.get(2).copied().unwrap_or(0), + ]; + let packed = + (u32::from(group[0]) << 16) | (u32::from(group[1]) << 8) | u32::from(group[2]); + // Three bytes are four characters, two are three, and one is two; + // whatever is left of the group is padding. + let characters = chunk.len() * 8 / 6 + 1; + for (index, shift) in [18, 12, 6, 0].into_iter().enumerate() { + let digit = ((packed >> shift) & 0x3f) as usize; + encoded.push(if index < characters { + ALPHABET[digit] as char + } else { + '=' + }); + } + } + encoded + } + + /// Run one operation against a scripted guest, and hand back what the + /// surface reported and what the guest was asked. + #[allow(clippy::disallowed_methods, reason = "synchronous path")] + fn against( + answers: Vec<(i32, String)>, + body: impl FnOnce(&mut GuestControl) -> LegacyResult<()>, + ) -> (LegacyResult<()>, String, Vec, Vec) { + let (surface, console) = UnixStream::pair().expect("a console socket pair"); + thread::scope(|scope| { + let asked = scope.spawn(move || guest(console, answers)); + let mut control = GuestControl { + console: Console::serving(surface, PathBuf::from("/dev/null")), + notes: String::new(), + }; + let outcome = body(&mut control); + let notes = std::mem::take(&mut control.notes); + let (asked, statuses) = asked.join().expect("the scripted guest"); + (outcome, notes, asked, statuses) + }) + } + + /// The refusal a check's script would see, if the surface refused. + fn refused(outcome: &LegacyResult<()>) -> String { + match outcome { + Err(LegacyError::Assertion(message)) => message.clone(), + other => panic!("expected a refused assertion, got {other:?}"), + } + } + + #[test] + fn a_command_reaches_the_guest_in_the_drivers_wire_form() { + let (outcome, notes, asked, statuses) = against(vec![(0, block("hi\n"))], |control| { + let result = control.execute("echo hi", Some(5))?; + assert_eq!(result.status, 0); + assert_eq!(result.output, "hi\n"); + Ok(()) + }); + assert!(outcome.is_ok(), "{outcome:?}"); + assert_eq!(asked.len(), 1); + assert_eq!( + asked[0], + "timeout 5 bash -c 'set -euo pipefail; echo hi' | (base64 -w 0; echo)\n" + ); + assert_eq!(statuses, vec!["echo ${PIPESTATUS[0]}\n".to_owned()]); + assert!(notes.is_empty(), "{notes}"); + } + + #[test] + fn an_unbounded_command_carries_no_bound() { + let (_, _, asked, _) = against(vec![(0, block(""))], |control| { + control.execute("true", None)?; + Ok(()) + }); + assert_eq!( + asked[0], + "bash -c 'set -euo pipefail; true' | (base64 -w 0; echo)\n" + ); + } + + #[test] + fn a_command_carrying_a_quote_still_reaches_the_guest_intact() { + let (_, _, asked, _) = against(vec![(0, block(""))], |control| { + control.execute("sh -c 'echo it'\"'\"'s'", None)?; + Ok(()) + }); + assert_eq!( + asked[0], + "bash -c 'set -euo pipefail; sh -c '\"'\"'echo it'\"'\"'\"'\"'\"'\"'\"'\"'s'\"'\"'' | (base64 -w 0; echo)\n" + ); + } + + #[test] + fn a_refused_command_reports_the_drivers_message_and_its_output() { + let (outcome, notes, _, _) = against(vec![(3, block("no such thing\n"))], |control| { + control.succeed(&["test -e /run/d2b/public.sock"], None)?; + Ok(()) + }); + assert_eq!( + refused(&outcome), + "command `test -e /run/d2b/public.sock` failed (exit code 3)" + ); + assert!(notes.contains("must succeed: test -e /run/d2b/public.sock\n"), "{notes}"); + assert!(notes.contains("output: no such thing\n"), "{notes}"); + } + + #[test] + fn a_command_that_succeeds_returns_its_output_and_logs_its_finish() { + let (outcome, notes, _, _) = against(vec![(0, block("d2bd.service\n"))], |control| { + let output = control.succeed(&["systemctl is-active d2bd.service"], None)?; + assert_eq!(output, "d2bd.service\n"); + Ok(()) + }); + assert!(outcome.is_ok(), "{outcome:?}"); + assert!(notes.contains("(finished: must succeed: systemctl is-active d2bd.service, in "), "{notes}"); + } + + #[test] + fn several_commands_concatenate_their_outputs() { + let (outcome, _, _, _) = against( + vec![(0, block("one\n")), (0, block("two\n"))], + |control| { + let output = control.succeed(&["echo one", "echo two"], None)?; + assert_eq!(output, "one\ntwo\n"); + Ok(()) + }, + ); + assert!(outcome.is_ok(), "{outcome:?}"); + } + + #[test] + fn a_command_that_was_meant_to_fail_is_refused_when_it_succeeds() { + let (outcome, notes, _, _) = against(vec![(0, block(""))], |control| { + control.fail(&["test -e /nope"], None)?; + Ok(()) + }); + assert_eq!( + refused(&outcome), + "command `test -e /nope` unexpectedly succeeded" + ); + assert!(notes.contains("must fail: test -e /nope\n"), "{notes}"); + } + + #[test] + fn a_command_meant_to_fail_returns_its_output_when_it_does() { + let (outcome, _, _, _) = against(vec![(1, block("refused\n"))], |control| { + let output = control.fail(&["d2b list Zone"], None)?; + assert_eq!(output, "refused\n"); + Ok(()) + }); + assert!(outcome.is_ok(), "{outcome:?}"); + } + + #[test] + fn a_wait_that_never_succeeds_reports_its_bound_and_the_last_output() { + let (outcome, notes, asked, _) = + against(vec![(1, block("still starting\n")), (1, block("still starting\n"))], |control| { + control.wait_until_succeeds("systemctl is-active d2bd.service", Duration::from_secs(1))?; + Ok(()) + }); + let message = refused(&outcome); + assert!(message.starts_with("action timed out after "), "{message}"); + assert!(message.ends_with(" seconds (timeout=1)"), "{message}"); + assert_eq!(asked.len(), 2, "one attempt in the loop, one after the bound"); + assert!(notes.contains("output: still starting"), "{notes}"); + assert!(notes.contains("waiting for success: systemctl is-active d2bd.service\n"), "{notes}"); + } + + #[test] + fn a_wait_that_succeeds_returns_the_output_of_the_attempt_that_did() { + let (outcome, _, _, _) = against( + vec![(1, block("not yet\n")), (0, block("active\n"))], + |control| { + let output = control.wait_until_succeeds("systemctl is-active d2bd.service", Duration::from_secs(2)); + assert_eq!(output.unwrap(), "active\n"); + Ok(()) + }, + ); + assert!(outcome.is_ok(), "{outcome:?}"); + } + + #[test] + fn a_file_that_never_appears_reports_after_its_bound() { + let (outcome, notes, asked, _) = against(vec![(1, block("")), (1, block(""))], |control| { + control.wait_for_file("/run/d2b/public.sock", Duration::from_secs(1))?; + Ok(()) + }); + let message = refused(&outcome); + assert!(message.ends_with(" seconds (timeout=1)"), "{message}"); + assert_eq!(asked[0], "timeout 900 bash -c 'set -euo pipefail; test -e /run/d2b/public.sock' | (base64 -w 0; echo)\n"); + assert!(notes.contains("waiting for file '/run/d2b/public.sock'\n"), "{notes}"); + } + + #[test] + fn a_unit_that_failed_ends_the_wait_at_once() { + let (outcome, _, asked, _) = against(vec![(0, block("ActiveState=failed\n"))], |control| { + control.wait_for_unit("d2bd.service", None, Duration::from_secs(30))?; + Ok(()) + }); + assert_eq!(refused(&outcome), "unit \"d2bd.service\" reached state \"failed\""); + assert_eq!(asked.len(), 1, "the state is read once, and refused on it"); + assert_eq!( + asked[0], + "timeout 900 bash -c 'set -euo pipefail; systemctl --no-pager show \"d2bd.service\" --property=\"ActiveState\"' | (base64 -w 0; echo)\n" + ); + } + + #[test] + fn a_unit_that_is_inactive_with_nothing_pending_ends_the_wait() { + let (outcome, _, asked, _) = against( + vec![ + (0, block("ActiveState=inactive\n")), + (0, block("No jobs to be processed.\n")), + (0, block("ActiveState=inactive\nSubState=dead\n")), + ], + |control| { + control.wait_for_unit("d2b-broker.socket", None, Duration::from_secs(30))?; + Ok(()) + }, + ); + assert_eq!( + refused(&outcome), + "unit \"d2b-broker.socket\" is inactive and there are no pending jobs" + ); + assert_eq!(asked.len(), 3, "the state, the job list, and the unit's own state"); + } + + #[test] + fn a_unit_that_is_active_satisfies_the_wait() { + let (outcome, notes, _, _) = against(vec![(0, block("ActiveState=active\n"))], |control| { + control.wait_for_unit("multi-user.target", None, Duration::from_secs(30))?; + Ok(()) + }); + assert!(outcome.is_ok(), "{outcome:?}"); + assert!(notes.contains("waiting for unit multi-user.target\n"), "{notes}"); + } + + #[test] + fn a_units_state_is_read_in_the_users_own_manager() { + let (_, _, asked, _) = against(vec![(0, block("ActiveState=active\n"))], |control| { + control.wait_for_unit("graphical-session.target", Some("alice"), Duration::from_secs(30))?; + Ok(()) + }); + assert!(asked[0].contains("su -l alice --shell /bin/sh -c"), "{asked:?}"); + assert!( + asked[0].contains("systemctl --user --no-pager show"), + "{asked:?}" + ); + } + + #[test] + fn a_guest_that_never_announces_its_shell_is_reported_by_what_it_should_have_said() { + let (surface, console) = UnixStream::pair().expect("a console socket pair"); + let mut writer = console; + let _ = writer.write_all(b"connecting to host...\n"); + let mut control = GuestControl { + console: Console::serving(surface, PathBuf::from("/dev/null")), + notes: String::new(), + }; + let error = control + .console + .await_shell(Duration::from_millis(50)) + .expect_err("a console that never greets is not a usable one"); + let message = error.to_string(); + assert!(message.contains(SHELL_GREETING), "{message}"); + assert!(message.contains("virtio serial console"), "{message}"); + } + + #[test] + fn a_guest_console_that_closes_mid_command_is_reported() { + let (surface, console) = UnixStream::pair().expect("a console socket pair"); + let mut control = GuestControl { + console: Console::serving(surface, PathBuf::from("/dev/null")), + notes: String::new(), + }; + drop(console); + let error = control + .console + .run("true", None) + .expect_err("a console that closed cannot answer"); + assert!(error.to_string().contains("console"), "{error}"); + } + + #[test] + fn a_refused_assertion_travels_to_the_check_as_an_assertion() { + let (surface, console) = UnixStream::pair().expect("a console socket pair"); + thread::scope(|scope| { + let _asked = scope.spawn(move || guest(console, vec![(1, block(""))])); + let mut control = GuestControl { + console: Console::serving(surface, PathBuf::from("/dev/null")), + notes: String::new(), + }; + let request = request("succeed", json!(["test -e /nope"])); + let reply = control.dispatch(&request); + assert!(!reply.ok); + assert!(reply.assertion, "a refused command is the check's own verdict"); + assert_eq!(reply.error.as_deref(), Some("command `test -e /nope` failed (exit code 1)")); + }); + } + + #[test] + fn an_operation_the_surface_does_not_carry_is_named_rather_than_ignored() { + let (surface, _console) = UnixStream::pair().expect("a console socket pair"); + let mut control = GuestControl { + console: Console::serving(surface, PathBuf::from("/dev/null")), + notes: String::new(), + }; + let reply = control.dispatch(&request("wait_for_open_port", json!(["22"]))); + assert!(!reply.ok); + assert!(!reply.assertion, "a missing operation is the lane's, not the check's"); + assert!( + reply + .error + .as_deref() + .unwrap_or_default() + .contains("wait_for_open_port"), + "{reply:?}" + ); + } + + #[test] + fn a_command_whose_output_was_not_wanted_does_not_desync_the_console() { + let (surface, console) = UnixStream::pair().expect("a console socket pair"); + thread::scope(|scope| { + let _asked = scope.spawn(move || { + guest( + console, + vec![(0, block("first\n")), (0, block("second\n"))], + ) + }); + let mut control = GuestControl { + console: Console::serving(surface, PathBuf::from("/dev/null")), + notes: String::new(), + }; + let mut kwargs = BTreeMap::new(); + kwargs.insert("check_output".to_owned(), Value::Bool(false)); + let mut quiet = request("execute", json!(["systemctl restart d2bd.service"])); + quiet.kwargs = kwargs; + let quiet = control.dispatch(&quiet); + assert_eq!(quiet.value, Some(json!([-2, ""]))); + let loud = control.dispatch(&request("execute", json!(["echo second"]))); + assert_eq!( + loud.value, + Some(json!([0, "second\n".to_owned()])), + "the second command reads its own output, not the first one's" + ); + }); + } + + #[test] + fn a_sleep_is_guest_time() { + let (_, _, asked, _) = against(vec![(0, block(""))], |control| { + control.sleep(5)?; + Ok(()) + }); + assert_eq!( + asked[0], + "bash -c 'set -euo pipefail; sleep 5' | (base64 -w 0; echo)\n" + ); + } + + /// One request, as a check's script sends it. + fn request(op: &str, args: Value) -> Request { + serde_json::from_str(&json!({ "op": op, "args": args }).to_string()).expect("a request") + } + + #[test] + fn a_string_is_quoted_the_way_the_guests_shell_needs_it_quoted() { + // The cases are the interpreter's own: an unreserved ASCII string + // passes through, an empty one is two quotes, and anything else is + // single-quoted with an embedded quote closed, double-quoted, and + // reopened. A different rule would change what the guest runs for a + // command carrying a quote. + assert_eq!(shlex_quote(""), "''"); + assert_eq!(shlex_quote("abc"), "abc"); + assert_eq!( + shlex_quote("test -e /run/d2b/public.sock"), + "'test -e /run/d2b/public.sock'" + ); + assert_eq!(shlex_quote("set -euo pipefail; echo hi"), "'set -euo pipefail; echo hi'"); + assert_eq!(shlex_quote("a'b"), "'a'\"'\"'b'"); + assert_eq!(shlex_quote("a\"b"), "'a\"b'"); + assert_eq!(shlex_quote("x$y"), "'x$y'"); + assert_eq!(shlex_quote("a b\nc"), "'a b\nc'"); + assert_eq!(shlex_quote("a+b,c-d.e/f:g=h@i%j_k"), "a+b,c-d.e/f:g=h@i%j_k"); + } + + #[test] + fn a_guests_output_block_decodes_to_what_the_command_wrote() { + assert_eq!(base64_decode("").expect("an empty block"), Vec::::new()); + assert_eq!(base64_decode("aGk=").expect("a block"), b"hi".to_vec()); + assert_eq!(base64_decode("aGVsbG8=").expect("a block"), b"hello".to_vec()); + assert_eq!( + base64_decode("YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXo=").expect("a block"), + b"abcdefghijklmnopqrstuvwxyz".to_vec() + ); + // A block long enough to overflow the accumulator, and one that + // arrives wrapped the way a guest's shell wraps it. + let long = "eHh4".repeat(250); + assert_eq!(base64_decode(&long).expect("a long block").len(), 750); + assert_eq!(base64_decode("aGk=\n").expect("a wrapped block"), b"hi".to_vec()); + assert!(base64_decode("not base64!").is_err()); + } +} diff --git a/packages/d2b-vm-harness/src/legacy_bridge.py b/packages/d2b-vm-harness/src/legacy_bridge.py new file mode 100644 index 000000000..7bb357c1e --- /dev/null +++ b/packages/d2b-vm-harness/src/legacy_bridge.py @@ -0,0 +1,209 @@ +"""The Python side of the lane's legacy guest-control surface. + +A check that has not been ported yet is a Python script: the `testScript` of +the fixture it used to be, with the shared diagnostics prelude already +interpolated at its top. This module is everything around that script - the +`machine` object its assertions call and the `start_all` they open with - and +deliberately nothing else. Every operation is a request to the lane's +harness, which owns the guest and speaks the legacy driver's own protocol to +it, so the semantics of an operation live in exactly one place: the guest a +command runs under, the retry bounds, the wording of a refused assertion, and +the log lines a reader of the lane sees around them. + +The reason it is worth being that thin is the transition itself. Every +unported check has to keep gating the lane with the assertions it already +has, so this surface has to behave like the one it replaces rather than like a +new one: the same operations, the same defaults, the same messages, and the +same exceptions, so a check that fails here fails with what it would have +failed with under the driver that is being retired. When a check ports, its +assertions move into Rust against the same guest-control primitives, and the +diagnostics text it reports through does not move at all. +""" + +import json +import os +import socket +import sys +import traceback + + +class MachineError(Exception): + """The guest could not be reached, or answered something unreadable. + + Named for the driver's own exception of that name, so a traceback from an + unported check reads the way it read before the port. + """ + + +class RequestedAssertionFailed(Exception): + """An assertion the check made did not hold. + + Named for the driver's own exception of that name. The message is the + driver's message, word for word, because the lane's diagnostics prelude + prints it: a check's failure line is that text, and a different wording + would be a different failure report for the same failure. + """ + + +class _Machine: + """The `machine` object an unported check's assertions call. + + Every method is a request: the harness performs the operation against the + guest, including every retry inside it, and answers with the result or + with the failure. A check therefore blocks on one guest operation exactly + as it blocked on one under the driver, and the bounds it declares are the + bounds the harness waits by. + """ + + # The driver's own defaults, restated rather than defaulted in the + # harness: a check that omits an argument has to get the bound the + # driver gave it, and the harness is told which bound was asked for + # rather than choosing one of its own. + EXECUTE_TIMEOUT = 900 + WAIT_TIMEOUT = 900 + + def __init__(self, control): + self._control = control + + def _call(self, op, *args, **kwargs): + return self._control.request(op, args, kwargs) + + def execute(self, command, check_return=True, check_output=True, timeout=EXECUTE_TIMEOUT): + """Run a shell command, returning `(status, output)`.""" + status, output = self._call( + "execute", + command, + check_return=check_return, + check_output=check_output, + timeout=timeout, + ) + return status, output + + def succeed(self, *commands, timeout=None): + """Run each command in turn, refusing anything that exits non-zero.""" + return self._call("succeed", *commands, timeout=timeout) + + def fail(self, *commands, timeout=None): + """Run each command in turn, refusing anything that exits zero.""" + return self._call("fail", *commands, timeout=timeout) + + def wait_until_succeeds(self, command, timeout=WAIT_TIMEOUT): + """Retry a command with one-second intervals until it succeeds.""" + return self._call("wait_until_succeeds", command, timeout=timeout) + + def wait_for_file(self, filename, timeout=WAIT_TIMEOUT): + """Wait until a path exists in the guest.""" + return self._call("wait_for_file", filename, timeout=timeout) + + def wait_for_unit(self, unit, user=None, timeout=WAIT_TIMEOUT): + """Wait until a systemd unit is active, refusing a failed one at once.""" + return self._call("wait_for_unit", unit, user=user, timeout=timeout) + + def sleep(self, secs): + """Sleep in guest time, the way the driver slept in guest time.""" + return self._call("sleep", secs) + + def __getattr__(self, name): + # An operation the lane's surface does not carry is named here rather + # than raising an attribute error about a private attribute: a check + # that reaches for it is a check whose port has a gap, and the gap + # should read as the operation that is missing. A dunder is left to + # raise the plain error, so the interpreter's own lookups are not + # answered with a message about the lane. + if name.startswith("__") and name.endswith("__"): + raise AttributeError(name) + raise AttributeError( + "the lane's legacy guest-control surface has no operation " + f"{name!r}; the legacy driver's surface is the set this lane " + "re-provides, and an assertion calling anything else needs a port" + ) + + +class _Control: + """The request channel to the lane's harness.""" + + def __init__(self, path): + self._socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self._socket.connect(path) + self._reader = self._socket.makefile("rwb") + + def request(self, op, args, kwargs): + self._reader.write( + json.dumps({"op": op, "args": args, "kwargs": kwargs}).encode() + b"\n" + ) + self._reader.flush() + line = self._reader.readline() + if not line: + raise MachineError( + "the lane's harness closed the guest-control channel while " + f"the check was waiting for `{op}` to finish" + ) + answer = json.loads(line) + if not answer.get("ok"): + # The harness distinguishes the two failures the driver + # distinguished: a refused assertion is the check's own verdict + # and is reported as one, and a guest the surface could not reach + # is a lane failure the check cannot be blamed for. + failure = ( + RequestedAssertionFailed + if answer.get("assertion") + else MachineError + ) + raise failure(answer.get("error") or "the operation failed with no message") + value = answer.get("value") + # JSON has no tuple, and `execute` returns one: a check that unpacks + # or indexes the result is reading the driver's `(status, output)`. + return tuple(value) if isinstance(value, list) else value + + +def _report_failure(): + """Print a failure the way the driver printed it. + + Every line carries the prefix the driver gave its test errors, and the + traceback carries the refusal as its last line, so a reader of a lane log + gets the assertion that failed and what it failed on without the two + being separated by anything. + """ + for line in traceback.format_exc().splitlines(): + print("!!! " + line, file=sys.stderr) + + +def start_all(): + """The fixtures' first call, and a no-op on this surface. + + The lane owns the guest: it booted the guest, waited for the guest's own + activation contract, and holds the guest for the whole run. There is + nothing left for a check to start, and the call stays because the + assertion bodies around it are unchanged until each check ports. + """ + + +def main(): + if len(sys.argv) != 3: + print( + "usage: legacy_bridge.py ", + file=sys.stderr, + ) + return 2 + control_path, script_path = sys.argv[1], sys.argv[2] + machine = _Machine(_Control(control_path)) + with open(script_path) as script: + source = script.read() + symbols = { + "__name__": "__main__", + "machine": machine, + "start_all": start_all, + } + try: + exec(compile(source, "", "exec"), symbols) + except Exception: # noqa: BLE001 - a check's own failure, whatever kind + # Every failure a check can raise is reported the same way, because a + # reader of the lane log should not have to know which exception class + # the assertion behind a red check happened to use. + _report_failure() + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/packages/d2b-vm-harness/src/lib.rs b/packages/d2b-vm-harness/src/lib.rs index b92e7093b..a7f9df0cf 100644 --- a/packages/d2b-vm-harness/src/lib.rs +++ b/packages/d2b-vm-harness/src/lib.rs @@ -20,9 +20,11 @@ pub mod guest; pub mod host; pub mod manifest; pub mod monitor; +pub mod legacy; pub use error::{HarnessError, Result, UnsnapshottableDevice}; pub use guest::{ActiveGuest, GuestSpec, boot, report, reserve_loopback_port}; +pub use legacy::{LegacyCheck, LegacyError, LegacyGuest, LegacyOutcome}; pub use host::{Capability, HostFacts, require_this_host}; pub use manifest::GuestManifest; pub use monitor::{BlockDevice, Monitor}; diff --git a/tests/host-integration/lib.nix b/tests/host-integration/lib.nix index ae9f426d8..c02787719 100644 --- a/tests/host-integration/lib.nix +++ b/tests/host-integration/lib.nix @@ -507,134 +507,16 @@ rec { # was asserting on; `explain` entries are (journal unit or null, token) whose # last daemon lines explain those rows. Diagnostics only: every assertion and # timeout is passed through unchanged. - fixtureDiagnostics = '' - # ---- d2b fixture diagnostics (issue #513) -------------------------- - # The test driver discards machine.execute output and does not re-print - # the output a timed-out wait_until_succeeds last saw, so a failed lane - # used to leave only the command text in the log. These helpers push the - # row set and the daemon explanation lines into the driver log (stdout - # and stderr of the test driver, that is the lane log). - # - # Diagnostics only: no assertion and no timeout is changed here. - import time as _diag_time - - _diag_t0 = _diag_time.monotonic() - _diag_stage = "startup" - - def _diag_elapsed(): - return f"{_diag_time.monotonic() - _diag_t0:.1f}s" - - def _diag_print(*lines): - for line in lines: - print(line, flush=True) - - def stage(name): - global _diag_stage - _diag_stage = name - _diag_print(f"[d2b] stage={name} t={_diag_elapsed()}") - - def diag(command, label="diagnostic output"): - try: - status, output = machine.execute(command, timeout=120) - except Exception as error: - _diag_print( - f"[d2b] stage={_diag_stage} t={_diag_elapsed()} {label}: " - f"diagnostic command failed: {error}" - ) - return -1 - _diag_print( - f"[d2b] stage={_diag_stage} t={_diag_elapsed()} {label} " - f"(exit {status}):" - ) - _diag_print(command) - for line in output.rstrip().splitlines(): - _diag_print(" " + line) - return status - - def _diag_journal(unit, token): - scope = f"-u {unit} " if unit else "" - select = f"| grep -F -- {token!r} " if token else "" - return ( - f"journalctl {scope}--no-pager -o cat -b -n 4000 2>/dev/null " - f"{select}| tail -n 60 || true" - ) - - def unit_dumps(unit): - """Row dumps for a systemd unit waiting to become active.""" - return [ - ( - f"{unit} status", - f"systemctl status {unit} --no-pager 2>&1 | tail -n 40 " - "|| true", - ), - ] - - # Every fixture drives one zone as one linux user through the same - # public socket, so the composed explanation is available without each - # stage listing the rows it asserted on: `d2b debug` reads the whole - # zone and prints the ownership tree, the row that is not settled, and - # the structured failure behind it. - _diag_zone = "work" - _diag_user = "alice" - - def diag_debug_zone(label="zone explanation"): - """The composed `d2b debug` report, always diagnostic and never - fatal: a failure that happened before the daemon was reachable must - still print its own stage rather than a diagnostic error. Bounded, - because a failure can happen before there is anything to explain.""" - status = diag( - f"runuser -u {_diag_user} -- env " - f"D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - f"timeout 60 d2b --zone {_diag_zone} debug {_diag_zone} 2>&1 " - f"|| true", - label, - ) - return status - - def diag_step(name, action, rows=(), explain=(), wait=None, debug=True): - stage(name) - try: - return action() - except Exception as error: - labels = ", ".join(label for label, _ in rows) or "none" - failing = f" wait={name}" if wait else "" - _diag_print( - f"[d2b] FAIL stage={name} t={_diag_elapsed()}{failing} " - f"rows=[{labels}]: {error}" - ) - if wait: - _diag_print(f"[d2b] failing wait: {wait}") - for label, command in rows: - diag(command, f"row dump: {label}") - for unit, token in explain: - detail = f"journal {unit or 'all'}" - if token: - detail += f" lines matching {token!r}" - diag(_diag_journal(unit, token), detail) - if debug: - diag_debug_zone() - raise - - def diag_unit(name, unit, timeout, debug=True): - """wait_for_unit with the unit status and journal on timeout.""" - return diag_step( - name, - lambda: machine.wait_for_unit(unit, timeout=timeout), - unit_dumps(unit), - [(unit, None)], - debug=debug, - ) - - def diag_wait(name, command, timeout, rows=(), explain=(), debug=True): - return diag_step( - name, - lambda: machine.wait_until_succeeds(command, timeout=timeout), - rows, - explain, - command, - debug=debug, - ) - ''; + # + # The text itself lives with the lane's own assertion surface, in + # `packages/d2b-vm-harness/src/diagnostics.py`, and is read from there + # rather than kept here. The Bazel lane runs these very same evaluated + # scripts, so a check that has not been ported yet reports its failure + # through this text under either lane; a second copy of it would be a + # second dialect of the same diagnostics, and the two would drift the first + # time one of them gained a helper the other did not. + fixtureDiagnostics = + builtins.readFile ../../packages/d2b-vm-harness/src/diagnostics.py; # Re-exported so tests can assert against the shared declaration. inherit mkGuestSystem mkRuntimeCloudHypervisorArtifact From e94f7a46bcc9eb4932ad3eb8ec6c63ac937ed25b Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sat, 26 Sep 2026 16:34:38 -0700 Subject: [PATCH 06/51] feat(vm): build one guest image per check and add the lane pool The lane had two guest images and eleven checks, and nine of those checks declare their own nodes.machine. Each image is now built from its own fixture, so the node, the machine size, the drive layout and the evaluated assertions all come out of one file - there is nowhere left for a second copy of a check's node to drift. The pool groups images by the emulator invocation they declare, is sized from the distinct-invocation count against a budget the guest configurations declare, and refuses an unsnapshottable drive before the pool is built rather than running a check without restore. This lands with the lane running and failing at one known blocker, and the blocker is recorded rather than papered over. Every lane guest mounts the host Nix store over 9p, and QEMU refuses an internal snapshot while a VirtFS export is mounted: the emulator monitor refused savevm 'lane-base': Error: Migration is disabled when VirtFS export path '.../xchg' is mounted in the guest The export cannot be dropped. Setting the manifest's shared directories to [] and booting the built image directly reproduces a kernel panic at activation - mount: /sysroot/nix/.ro-store: special device nix-store does not exist - because the guest is booted from a host store that is not there. Settled: the lane takes external qcow2 overlays per pool member instead of HMP savevm, which QEMU permits with virtiofs mounted, leaves every guest byte-identical to the one the fixtures were written against, and keeps the rule that a check never runs without restore. R6's "internal snapshots" wording no longer describes the mechanism and is corrected in the plan's decision record. Four defects in this wiring are fixed here, none of them visible without running the build. The test rule's src named the genrule rather than its output file, which failed every target at analysis. The genrule's argument parsing had lost its substituters positional. The declared source set was missing three trees the guest evaluation reaches by relative path - the manpages and shell completions the d2b module tree installs, and the packaging tree the observability modules read - and the two shape images had masked all three by never importing those modules. And the fixture capture imported a { pkgs, self }: function without calling it, so every check looked like one that declared no assertions. --- BUILD.bazel | 10 + bazel/checks/vm/BUILD.bazel | 99 ++- bazel/checks/vm/defs.bzl | 221 ++++-- changelog.d/bazel-owned-guest-pool.md | 48 ++ nix/test-support/guest-image.nix | 276 ++++++- packages/d2b-vm-harness/BUILD.bazel | 9 +- .../d2b-vm-harness/src/bin/d2b-vm-harness.rs | 695 +++++++++++++++++- packages/d2b-vm-harness/src/guest.rs | 57 +- packages/d2b-vm-harness/src/host.rs | 62 ++ packages/d2b-vm-harness/src/lib.rs | 2 +- packages/d2b-vm-harness/src/manifest.rs | 166 +++++ packages/d2b-vm-harness/src/monitor.rs | 78 ++ 12 files changed, 1640 insertions(+), 83 deletions(-) create mode 100644 changelog.d/bazel-owned-guest-pool.md diff --git a/BUILD.bazel b/BUILD.bazel index 1171e2051..ea15be787 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -172,6 +172,16 @@ filegroup( visibility = ["//visibility:public"], ) +# keep +filegroup( + name = "completions_workspace_sources", + srcs = glob( + ["completions/**/*"], + allow_empty = True, + ), + visibility = ["//visibility:public"], +) + # keep filegroup( name = "examples_workspace_sources", diff --git a/bazel/checks/vm/BUILD.bazel b/bazel/checks/vm/BUILD.bazel index ee3d02a05..c3aa045ff 100644 --- a/bazel/checks/vm/BUILD.bazel +++ b/bazel/checks/vm/BUILD.bazel @@ -1,4 +1,4 @@ -load(":defs.bzl", "guest_boot_test", "guest_image") +load(":defs.bzl", "guest_boot_test", "guest_image", "lane_test") package(default_visibility = ["//visibility:public"]) @@ -8,17 +8,44 @@ package(default_visibility = ["//visibility:public"]) # configuration - rather than the whole workspace. _GUEST_SOURCES = [ "//:Cargo.lock", - "//:Cargo.toml", - "//:d2b_resource_schemas_v3", - "//:generated_cli_shell_artifacts", "//:flake.lock", "//:flake.nix", "//:nix_workspace_sources", "//:nixos_modules_workspace_sources", "//:packages_workspace_sources", + # The manpages, read out of the staged tree by the d2b module tree: + # `nixos-modules/host-daemon.nix` installs `../docs/manpages/d2b.1`, + # so a per-check guest resolves a path out of the docs tree that the + # two shape images never needed. An input reached by a relative path + # out of another input is an input, and the action copies declared + # sources and nothing else. + "//:docs_workspace_sources", + # The shell completions, for the same reason as the manpages above: the + # d2b module tree installs `../completions/d2b.{bash,fish,zsh}`. + "//:completions_workspace_sources", + # The packaging tree, reached as `../pkgs/signoz*` by the observability + # modules the per-check guests import. + "//:pkgs_workspace_sources", + # The host-integration fixtures themselves. A per-check guest is built + # from its own fixture, so the fixtures are inputs to every image: a + # change to a check's node or to its assertions rebuilds that check's + # guest, which is the only way the guest and the check it runs can be + # said to come from one declaration. + "//:tests_root_sources", + # The provider acceptance manifests the check fixtures sign their + # artifacts against. `//:tests_root_sources` globs `tests/**/*` but does + # not cross a package boundary, and `tests/fixtures` is a Bazel package, + # so this tree is a second declared input rather than part of that one. + "//tests/fixtures:fixture_sources", + # The diagnostics prelude the fixtures interpolate and the lane's own + # guest-control surface embeds, read out of the tree by both. + "//packages/d2b-vm-harness:src/diagnostics.py", + # The provider manifests the acceptance fixtures sign their artifacts + # against, read by path out of the tree by the fixtures' own helpers. + "//packages/d2b-provider-guest-cloud-hypervisor:provider-manifest.json", + "//packages/d2b-provider-guest-cloud-hypervisor:root-config.schema.json", "//packages/d2b-broker:src/ops/state-posture-contract.json", "//:rust-toolchain.toml", - "//tests/fixtures:fixture_sources", ] # The d2b host binaries every guest image is built against. @@ -80,6 +107,54 @@ guest_image( tags = _IMAGE_TAGS, ) +# One guest per check, built from that check's own fixture. +# +# The list is the lane's check inventory and nothing else: each entry names +# the fixture the guest is read out of, and the guest's memory, vCPU count, +# disk, drives, device options and assertions all come out of that file +# during the evaluation. Three of these checks boot a plain NixOS node with +# no d2b daemon host at all, two turn nftables on, two install acceptance +# artifacts, and two boot a nested guest on the writable-store shape; that +# is exactly why the lane cannot serve them from one image, and why this list +# has no column for any of it. +# +# The target name is the check's own name - the `vmChecks` attribute name, +# which is what `D2B_VM_CHECK` and `D2B_HOST_VM_CHECK` carry - so a +# contributor can name the target they want without a second naming scheme. +_CHECKS = [ + "bridge-isolation", + "daemon-smoke", + "device-worker-launch", + "guest-agent-cap-confinement", + "guest-shell-service", + "privilege-oracle", + "resource-operator-activation", + "runtime-cloud-hypervisor-guest-preflight", + "state-posture-contract", + "virtiofsd-volume-runtime", + "wayland-proxy", +] + +[ + guest_image( + name = "guest_image_" + check, + check = "tests/host-integration/%s.nix" % check, + srcs = _GUEST_SOURCES, + cloud_hypervisor_controller = _CONTROLLER, + flake = "//:flake.nix", + flake_lock = "//:flake.lock", + host_tools = _HOST_TOOLS, + nix = "@nix//:bin/nix", + node_shape = check, + substituters = "https://cache.nixos.org/", + tags = _IMAGE_TAGS, + ) + for check in _CHECKS +] + +_CHECK_IMAGES = [":guest_image_" + check for check in _CHECKS] + + # The emulator every lane guest boots, taken from the same pinned nix # package set the guest closure is realized from, so the emulator and the # guest are at one nixpkgs revision and a snapshot taken by one version is @@ -103,6 +178,19 @@ guest_boot_test( image = ":guest_image_writable_store", ) +# The lane. It owns the whole run: it reads every check's own guest image, +# groups them by the emulator invocation they declare, sizes a pool from the +# distinct invocations against the host budget the guest configurations +# declared, and runs each selected check against a snapshot-restored copy of +# the guest that check was built for. +lane_test( + name = "host_integration_lane_run", + images = _CHECK_IMAGES, + emulator = _EMULATOR, + harness = _HARNESS, + python = "@python3//:bin/python3", +) + # The lane's own suite. The harness crate deliberately carries no test # aggregate - the repository's test census would force-register such a crate # into the main package suite - so the targets that lint and exercise it are @@ -116,5 +204,6 @@ test_suite( "//packages/d2b-vm-harness:d2b_vm_harness_test", ":guest_boot_daemon", ":guest_boot_writable_store", + ":host_integration_lane_run", ], ) diff --git a/bazel/checks/vm/defs.bzl b/bazel/checks/vm/defs.bzl index da4de5652..6fda3e9e8 100644 --- a/bazel/checks/vm/defs.bzl +++ b/bazel/checks/vm/defs.bzl @@ -45,7 +45,8 @@ out="$4" source_manifest="$5" substituters="$6" controller="$7" -shift 7 +check="$8" +shift 8 # Bazel hands over execroot-relative paths. Nix resolves its configuration # against the working directory, so the output tree is anchored here. @@ -92,6 +93,15 @@ done < "$source_manifest" root="$(CDPATH= cd -- "$source" && pwd -P)" [ -f "$source/$flake" ] || fail "the declared flake ($flake) is not among the copied sources" [ -f "$source/$lock" ] || fail "the declared flake lock ($lock) is not among the copied sources" +# The check's fixture is staged with everything else and reaches the +# evaluation as a path relative to this tree, which is how the guest finds +# its own `./lib.nix` and its node module again: a change to the fixture is a +# change to the image, not a number the image already recorded. +check_argument="[ ]" +if [ -n "$check" ]; then + [ -f "$source/$check" ] || fail "the declared check fixture ($check) is not among the copied sources" + check_argument="[ \\"$check\\" ]" +fi # The d2b host binaries, staged by output name into the bundle the guest # closure reads. The guest-side package refuses any other inventory, so a @@ -126,7 +136,7 @@ done [ "$reachable" -eq 1 ] || fail "none of the declared substituters ($substituters) is reachable" system="$("$nix_bin" eval --raw --impure --expr builtins.currentSystem)" || fail "could not read the nix system" -expr="(builtins.getFlake \\"path:$root\\").guestImage.\\"$system\\" { rawBundle = \\"$bundle\\"; rawCloudHypervisorController = $controller_argument; nodeShape = \\"$node_shape\\"; }" +expr="(builtins.getFlake \\"path:$root\\").guestImage.\\"$system\\" { rawBundle = \\"$bundle\\"; rawCloudHypervisorController = $controller_argument; extraModules = $check_argument; nodeShape = \\"$node_shape\\"; }" echo "guest-image $label: realizing the guest from declared inputs" >&2 image="$(nix_run "$nix_bin" build \\ --option store "local?store=$store_dir" \\ @@ -150,6 +160,20 @@ cp -a "$image/." "$out/" def _guest_image_impl(ctx): output = ctx.actions.declare_directory(ctx.label.name) source_manifest = ctx.actions.declare_file(ctx.label.name + ".sources") + check = ctx.attr.check + + # The check's fixture is a declared source like every other input, and it + # reaches the evaluation as a path relative to the tree the action copies + # rather than as a path into the execroot: an absolute path would make + # `import` copy that one file into the store under a flat name, and its + # own `./lib.nix` would then resolve outside any tree at all. Naming it + # as a path rather than as a label is also what lets the fixtures stay + # outside a Bazel package - a package boundary there would make the root + # package's own reference to one of them an invalid label. + if check: + staged = [source.short_path for source in ctx.files.srcs] + if check not in staged: + fail("guest_image %s: the check fixture %s is not among srcs" % (ctx.label, check)) ctx.actions.write( output = source_manifest, content = "\n".join([source.path for source in ctx.files.srcs]) + "\n", @@ -170,6 +194,7 @@ def _guest_image_impl(ctx): source_manifest.path, ctx.attr.substituters, controller.path if controller else "", + check, ] + [tool.path for tool in ctx.files.host_tools], command = _GUEST_IMAGE_COMMAND % ( str(ctx.label), @@ -195,15 +220,23 @@ guest_image = rule( allow_single_file = True, mandatory = True, ), - # Which of the re-homed node's two guest shapes this image evaluates: - # `daemon` for the daemon/broker host checks, `writable-store` for - # the checks that boot a nested guest, which replaces the root drive - # and boots through a bootloader. It is a declared attribute rather - # than a lane constant, so the shape a guest is built from is part of - # the action's key rather than an ambient fact. + # The check this guest belongs to, as a declared label on that check's + # own fixture file. The guest's node, its machine size, its drive + # layout and its assertions are all read out of that file during the + # evaluation, so the lane carries no list of which check wants which + # guest: the only place a check's guest is declared is the check. + "check": attr.string(), + # The name this guest reports on its console. With a `check` it is + # the check's own name, which is what makes a launcher that booted + # the wrong image say so in one line. Without one it names the shape + # of the re-homed node the image evaluates: `daemon` for the + # daemon/broker host shape, `writable-store` for the shape that + # replaces the root drive and boots through a bootloader. It is a + # declared attribute rather than a lane constant, so the guest a + # target builds is part of the action's key rather than an ambient + # fact. "node_shape": attr.string( default = "daemon", - values = ["daemon", "writable-store"], ), # The guest's own binaries, taken in the target configuration: the # guest runs the binaries this build produces, not a second copy @@ -240,17 +273,16 @@ set -eu # assumed - a wrong guess here is a guest that never boots, with a path error # instead of a boot error. runfiles="$(CDPATH= cd -- "$(dirname -- "$0")/../../../.." && pwd -P)" - export D2B_VM_HARNESS_CYCLES="{cycles}" -export D2B_VM_HARNESS_EMULATOR="$runfiles/{emulator}" -export D2B_VM_HARNESS_IMAGE="$runfiles/{image}" +export D2B_VM_HARNESS_EMULATOR="$runfiles/__EMULATOR__" +export D2B_VM_HARNESS_IMAGE="$runfiles/__IMAGE__" # A lane-scoped working directory that outlives each individual guest, and is # this test's own rather than the sandboxed temporary directory the current # Bazel release does not expose to a sandboxed action. The harness resolves a # relative one against its own working directory. export D2B_VM_HARNESS_WORK_ROOT="{work_root}" -exec "$runfiles/{harness}" "$@" +exec "$runfiles/__HARNESS__" "$@" """ _LANE_TAGS = [ @@ -281,54 +313,145 @@ def guest_boot_test(name, image, emulator, harness, timeout = "eternal"): remote execution and out of any aggregate that would replay a guest's verdict. """ - runner = name + "_runner.sh" - script = _LANE_RUNNER_SCRIPT.format( - cycles = "2", - emulator = "$(rlocationpath %s)" % emulator, - harness = "$(rlocationpath %s)" % harness, - image = "$(rlocationpath %s)" % image, - work_root = "d2b-vm-lane-work/%s" % name, + _write_runner( + name = name, + script = _LANE_RUNNER_SCRIPT.format( + cycles = "2", + work_root = "d2b-vm-lane-work/%s" % name, + ), + runfiles = { + "__HARNESS__": harness, + "__EMULATOR__": emulator, + "__IMAGE__": image, + }, + srcs = [emulator, harness, image], + tags = _LANE_TAGS, + timeout = timeout, ) - # The runner is written through a genrule rather than handed to the test - # rule's `env`, following `nix_native_test`: a runfile location is - # expanded only where a rule expands it, and a binary cannot resolve its - # own runfiles. Each location becomes a placeholder first so the `$` - # escaping genrule's own expansion needs does not touch the shell script - # around it. - make_vars = { - "$(rlocationpath %s)" % harness: "__HARNESS__", - "$(rlocationpath %s)" % emulator: "__EMULATOR__", - "$(rlocationpath %s)" % image: "__IMAGE__", - } - for make_var, placeholder in make_vars.items(): - script = script.replace(make_var, placeholder) - script = script.replace("$", "$$") - for make_var, placeholder in make_vars.items(): - script = script.replace(placeholder, make_var) +def _write_runner(name, script, runfiles, srcs, tags, timeout): + """Write a generated shell runner and register the test that runs it. + The runner is written through a genrule rather than handed to the test + rule's `env`, following `nix_native_test`: a runfile location is expanded + only where a rule expands it, and the harness is a binary, not a shell + script that could resolve its own runfiles. Each location becomes a + placeholder first so the `$` escaping genrule's own expansion needs does + not touch the shell script around it. + """ + locations = list(runfiles.items()) + for index, (placeholder, _) in enumerate(locations): + script = script.replace(placeholder, "@LANE_RUNFILE_%d@" % index) + script = script.replace("$", "$$") + for index, (_, label) in enumerate(locations): + script = script.replace("@LANE_RUNFILE_%d@" % index, "$(rlocationpath %s)" % label) native.genrule( - name = runner, - srcs = [ - emulator, - harness, - image, - ], + name = name + "_runner.sh", + srcs = srcs, outs = [name + "_runner"], cmd = "\"$(execpath @python3//:bin/python3)\" -c 'import pathlib,sys; p=pathlib.Path(sys.argv[1]); p.write_text(sys.stdin.read()); p.chmod(0o755)' \"$(OUTS)\" <<'EOF'\n%s\nEOF" % script, - tags = _LANE_TAGS, + tags = tags, tools = ["@python3//:bin/python3"], ) native_test( name = name, + # The genrule's output file, not the genrule target: `src` names a + # file the test rule reads, and a label that resolves to a rule + # rather than to an output leaves it with no file to run. src = ":" + name + "_runner", - data = [ - ":" + name + "_runner", - harness, - image, - emulator, - ], + data = srcs, size = "large", + tags = tags, + timeout = timeout, + ) + +_LANE_SUITE_RUNNER_SCRIPT = """\ +#!/bin/sh +set -eu + +# A test runs with its working directory inside the runfiles tree, not at its +# root, so the root is derived from this script's own location rather than +# assumed - a wrong guess here is a guest that never boots, with a path error +# instead of a boot error. +runfiles="$(CDPATH= cd -- "$(dirname -- "$0")/../../../.." && pwd -P)" + +export D2B_VM_HARNESS_EMULATOR="$runfiles/__EMULATOR__" +export D2B_VM_HARNESS_IMAGES="$runfiles/__IMAGES__" +# A check that has not been ported yet is a Python script, and the +# interpreter it runs under is part of what the lane is. Left to itself the +# surface resolves whatever `python3` a developer's shell happens to find, so +# the interpreter is a declared runfile from the same pinned nix package set +# as the guest it drives. +export D2B_VM_HARNESS_PYTHON="$runfiles/__PYTHON__" +# A lane-scoped working directory that outlives each individual guest, and is +# this test's own rather than the sandboxed temporary directory the current +# Bazel release does not expose to a sandboxed action. The harness resolves a +# relative one against its own working directory. +export D2B_VM_HARNESS_WORK_ROOT="d2b-vm-lane-work/{name}" + +# The harness is asked for the `lane` subcommand rather than being left to its +# own default. With no argument it runs the single-guest self-check, which is +# the other target's job and which asks for `D2B_VM_HARNESS_IMAGE` - a variable +# the lane has no reason to set, because the lane reads the whole image list +# instead. Anything a contributor passes on the command line reaches the +# lane's own selection, which reads `--check`. +exec "$runfiles/__HARNESS__" lane "$@" +""" + +def lane_test(name, images, emulator, harness, python, timeout = "eternal"): + """The lane: one guest per check's own configuration, pooled and run. + + The images are the graph outputs of one `guest_image` per check, so each + check's guest is built from that check's own fixture. The pool the + harness builds from them is sized from the distinct emulator invocations + among them, not from a count chosen here: adding a check to the lane adds + a check here, and the pool follows. + + The target's result is never cacheable. A guest's verdict depends on + what the host did while it ran, so a second invocation re-runs every + selected check rather than replaying what the first one concluded, and + `no-cache` is what says that to the Bazel graph. + + The pool runs concurrently inside this one action, so the lane does not + depend on Bazel scheduling several targets to overlap them, and live test + output cannot serialize it either way. Each check reports under its own + name in the JUnit document the action writes, which is what makes one + failed check identifiable from the lane's own output. + """ + listing = name + "_images.txt" + native.genrule( + name = name + "_images", + srcs = images, + outs = [listing], + # A single `%s`, not `%%s`: this fragment is a plain string literal + # that nothing `%`-formats, so the escape would reach the shell + # doubled and `printf` would write a literal `%s` as the only line of + # the listing. The `$(rlocationpath ...)` parts are expanded by Bazel + # before the shell sees them, which is why the paths arrive intact. + cmd = "printf '%s\\n' " + " ".join(["$(rlocationpath %s)" % image for image in images]) + " > $@", tags = _LANE_TAGS, + ) + _write_runner( + name = name, + script = _LANE_SUITE_RUNNER_SCRIPT.format(name = name), + runfiles = { + "__HARNESS__": harness, + "__EMULATOR__": emulator, + "__IMAGES__": ":" + listing, + "__PYTHON__": python, + }, + # The images are the test's own data, not only the listing genrule's + # inputs. A `$(rlocationpath)` line names a runfile of the *action* + # that produced it, so without the images here the test's runfiles + # tree carries the listing and nothing it points at, and the lane + # fails on the first manifest it tries to read. + srcs = [ + ":" + listing, + ] + images + [ + emulator, + harness, + python, + ], + tags = _LANE_TAGS + ["no-cache"], timeout = timeout, ) diff --git a/changelog.d/bazel-owned-guest-pool.md b/changelog.d/bazel-owned-guest-pool.md new file mode 100644 index 000000000..7cff660ec --- /dev/null +++ b/changelog.d/bazel-owned-guest-pool.md @@ -0,0 +1,48 @@ +--- +type: feat +--- + +Build one guest per host-integration check, pool them, and run the lane as a single Bazel test + +The type-10 lane had two guest images and eleven checks, and nine of the +checks declared a `nodes.machine` of their own - three a plain NixOS node +with no d2b daemon host at all, two turning nftables on, two installing +acceptance artifacts, and two booting a nested guest on the writable-store +shape. One image could not be all of those, so the checks that needed a guest +of their own were running against a guest that lacked what they needed. + +Each check now gets a `guest_image` of its own, and the guest is built from +that check's own fixture rather than from a second copy of its node: the +guest-image evaluation reads the node, the machine size, the drive layout and +the evaluated assertions straight out of the fixture file, so there is no +inventory anywhere in the tree that can disagree with a check about which +guest it boots. + +The lane test target owns the run. It groups the checks by the emulator +invocation they declare - not by closure, which is what makes two of them the +same shape - sizes a pool from the number of distinct invocations against the +budget the guest configurations declared and the memory, vCPUs and working +directory this host actually has, refuses before the pool is built if any +attached writable device cannot carry an internal snapshot, proves on a member +of every distinct invocation that a restored guest is the fresh boot its check +was written against, runs each check against a snapshot-restored copy of its +own guest, and retires rather than restores a member that has run a nested +guest. Each check reports under its own name in the lane's JUnit document, and +the lane's result is never cacheable. + +The interpreter an unported check's script runs under is a declared runfile +from the pinned nix package set rather than whatever `python3` a developer's +shell happens to resolve. + +The pool does not yet reach a check. A guest built from any of these +configurations mounts the host Nix store over 9p - the QEMU VM module's own +`mountHostNixStore`, which its direct-boot shape is built around - and QEMU +refuses an internal snapshot while a VirtFS export is mounted: +`Migration is disabled when VirtFS export path '...' is mounted in the guest +using mount_tag 'nix-store'`. The guest cannot boot without that export: +without it the activation's `/sysroot/nix/.ro-store` mount fails and the +system panics. So the guests are snapshottable and not snapshot-capable at +the same time, and every check fails at the snapshot rather than at an +assertion. Restoring from an external qcow2 overlay rather than an internal +`savevm` is the change that reconciles this with the restore-only rule; it is +not made here. diff --git a/nix/test-support/guest-image.nix b/nix/test-support/guest-image.nix index 78552b904..1ff369376 100644 --- a/nix/test-support/guest-image.nix +++ b/nix/test-support/guest-image.nix @@ -8,12 +8,20 @@ # the same contract the legacy `D2B_HOST_TOOL_BUNDLE` handoff passes, and # the caller content-addresses them. # -# Two guest shapes are declared here, and the shape is a declared input -# rather than a second copy of the numbers: `daemon` is the node the current -# `vmChecks` fixtures boot for the daemon/broker host checks, and -# `writable-store` is the node the two nested-guest checks boot, which -# replaces the root drive and boots through a bootloader. Per-check module -# contributions arrive through `extraModules`. +# `nodeShape` names the guest this image is, and it is a declared input +# rather than a second copy of the numbers behind it. With no `extraModules` +# the guest is one of the re-homed node's two shapes, named for what it is: +# `daemon` is the node the daemon/broker host checks boot, and +# `writable-store` is the node the nested-guest checks boot, which replaces +# the root drive and boots through a bootloader. +# +# With `extraModules` the guest is a check's own. The entry is that check's +# fixture file, and the node, the boot shape, the machine size, and the +# assertions all come out of that one file - so "every check runs against a +# guest built from its own node configuration" is a property of where the +# numbers are read from rather than a discipline the lane has to maintain. +# There is no list here of which check wants which guest, because there is +# nowhere for such a list to disagree with a fixture. # # The output is one store path holding the guest's system closure, the root # disk in the shape's own format, and a manifest of what a launcher needs to @@ -21,12 +29,27 @@ # kernel command line resolved, the per-check device options, and the # activation contract the launcher waits for on the guest's serial console. # The manifest is the only record of the invocation shape: the launcher -# renders it and never restates a number the re-homed node declared. +# renders it and never restates a number the node declared. It carries the +# two things the lane's pool needs that a boot cannot tell it - what one +# member costs the host, and the bound the pool is sized against - and, for +# a check's own guest, the check's evaluated assertions. { pkgs, self, bazelHostTools, rawBundle, extraModules ? [ ], nodeShape ? "daemon" }: let inherit (pkgs) lib; + # A workspace-relative path, as the Bazel action hands it over, resolved + # back inside this tree. + # + # The resolution is the whole trick. The action stages the declared sources + # into a directory and evaluates the flake from there, so `../..` is that + # staged tree and a fixture reached through it keeps resolving its own + # `./lib.nix` and `../../nix/test-support/...` inside it. An absolute path + # would not: `import` would copy the single fixture file into the store + # under a flat name, and its next relative import would land outside any + # tree at all. + fixturePath = declared: builtins.toPath "${toString ../..}/${declared}"; + # The lane's activation contract, in the repository's own field names: # `nixos-modules/lib.nix` describes every service capability with a # readiness signal and a contract that signal buys. The lane's guest @@ -51,6 +74,11 @@ let activationStallSeconds = 90; activationStallRepeatSeconds = 600; + # Where a node declares the units its own activation is complete when. The + # re-homed daemon node writes it; a node that never wanted the daemon host + # does not, and that guest's activation falls back to `multi-user.target`. + acceptanceUnitsFile = "/etc/d2b/daemon-acceptance-units"; + # Refuse an incomplete handoff before a guest closure is evaluated. The # host-tool package repeats this check when it is built, but that is the # wrong place to learn a binary is missing: by then the guest closure has @@ -77,27 +105,165 @@ let # same pinned set the guest closure is realized from. testInstrumentation = pkgs.path + "/nixos/modules/testing/test-instrumentation.nix"; + # One guest per check, built from that check's own node declaration. + # + # `extraModules` carries a check's fixture file - a path under this tree, + # written the way Bazel writes a workspace-relative path - rather than a + # module. A fixture is `pkgs.testers.runNixOSTest { nodes.machine = ...; + # testScript = ...; }`, and the lane needs both halves of that pair: the + # node is the guest, and the evaluated script is the check the lane runs + # against it. Reading them out of the fixture's own file is the only way + # "every check runs against a guest built from its own node configuration" + # can be true rather than approximately true, because there is then nowhere + # for a second copy of a check's node to drift. + # + # The read builds no driver. For this evaluation only, `runNixOSTest` is + # replaced by a function that hands back the test module it was given, so a + # check costs one guest evaluation rather than one test derivation and the + # Python driver under it. + # + # That substitution is load-bearing, not a convenience. The real + # `runNixOSTest` returns the *evaluated* test: its `nodes` are + # `config.nodesCompat`, and the nix test framework builds that by merging + # each node's evaluated `eval-config` result with `config = `. So `checkFixture.nodes.machine` is an attrset carrying `config` + # next to `appstream`, `boot`, `systemd`, `users` and the rest of the + # configuration namespace - an evaluated configuration, not a module. Lifting + # it into the guest system as a module is what the module system rejects + # with "Module `:anon-N:anon-M' has an unsupported attribute `appstream'", + # and dropping the offending keys would instead import a frozen + # configuration that never saw the guest's own QEMU VM module. The identity + # hands back the fixture's *declaration* - `{ name, nodes.machine, + # testScript; }` - which is the thing that was written down and the thing the + # guest has to be built from. + fixturePkgs = + pkgs // { + testers = pkgs.testers // { + runNixOSTest = testModule: testModule; + }; + }; + checkFixture = + if extraModules == [ ] then + null + else + let + loaded = import (fixturePath (builtins.head extraModules)); + in + # A fixture is a module *function* of `{ pkgs, self }`, not a module. + # Reading it without calling it yields a function, and a function has + # neither `.nodes` nor `.testScript` - so an uncalled fixture looks + # exactly like one that declared no assertions. + if builtins.isFunction loaded then + loaded { + pkgs = fixturePkgs; + inherit self; + } + else + loaded; + # The fixture's own name for the check, which is the name the lane reports + # it under and the name a contributor filters it by: the `vmChecks` + # attribute name is the fixture's file stem, and that is what the make + # target's selection variables carry. + checkName = + if checkFixture == null then + null + else + lib.removeSuffix ".nix" (builtins.baseNameOf (builtins.head extraModules)); + checkNodes = if checkFixture == null then [ ] else lib.attrValues (checkFixture.nodes or { }); + checkScript = + if checkFixture == null then + null + else + let + declared = checkFixture.testScript or null; + in + if declared == null then + throw '' + d2b guest image: the fixture for check '${checkName}' declares no testScript, + so there is nothing for the lane to run against the guest it declares. + '' + else if builtins.isFunction declared then + # The driver calls a function-shaped script with the nodes it + # evaluated. This one is handed the fixture's own node declarations, + # which is what such a script interpolates. + declared { nodes = checkFixture.nodes; containers = { }; } + else + declared; + + # The host budget the lane's pool is sized against, declared here rather + # than in the launcher: it is a property of the guests this file + # configures, so a check added tomorrow cannot silently change what the + # pool is allowed to hold. The launcher reads it out of the manifest and + # combines it with what the host actually has, which is the only half it + # could not know at build time. + poolBudget = { + # A contributor's machine is running a browser, an editor, and the rest + # of their day alongside the lane, so the lane takes a share of what is + # free rather than of what is installed. + memoryShareNumerator = 2; + memoryShareDenominator = 3; + # vCPU count is bounded the same way, and a pool member that cannot get + # its declared vCPUs runs a guest that starves the very handshakes the + # re-homed node raised its core count for. + coreShareNumerator = 3; + coreShareDenominator = 4; + # The working directory a member needs is the root disk its own node + # declared: the launcher copies that disk into the directory it owns. + workingDirectoryFollowsDisk = true; + }; + # `d2bDaemonNode` declares `virtualisation.*`, so the guest is evaluated # with the same QEMU VM module the runNixOSTest nodes carry. Evaluating # the node module directly, rather than through the test driver, is what # makes the result a bootable system closure the lane's own launcher can # use. + # + # With a check fixture in hand the node is the fixture's own: a plain node + # for a check that never wanted the d2b daemon host, the re-homed daemon + # node plus its per-check contributions for a check that did, and the + # writable-store node for a check that boots a nested guest. Reading it + # rather than rebuilding it here is what lets one lane carry checks that + # declared three different shapes of guest. + guestNode = + if checkNodes == [ ] then + # `d2bCloudHypervisorNode` is `d2bDaemonNode` with the writable store + # opted into, so the two shapes are one declaration read two ways and + # cannot drift apart. + d2bNode.d2bDaemonNode { writableStore = nodeShape == "writable-store"; } + else + builtins.head checkNodes; + + # The node's own name inside the fixture - `nodes.machine`, in every + # fixture in the tree today - which the guest has to answer for. + # + # The nix test framework binds each node as a submodule of the `nodes` + # option, and the module system hands a submodule the `name` argument + # itself: "the sole exception to this is the argument `name` which is + # provided by parent modules to a submodule and contains the attribute + # name the submodule is bound to" (`lib/modules.nix`). The lane evaluates + # the node as a *top-level* module, so nothing provides `name`, and a node + # that imports a module asking for it by name - `nixos-modules/guest-broker.nix` + # does, to build its `--authority-id guest-` - fails the evaluation + # with "attribute 'name' missing". The value supplied here is the one the + # framework would have supplied, so the guest's authority id is the one the + # fixture's own guest had rather than a lane-invented substitute. The two + # shape-only images have no fixture node and get no argument, which is why + # they are left exactly as they were. + checkNodeName = + if lib.length checkNodes == 1 then + builtins.head (lib.attrNames (checkFixture.nodes or { })) + else + null; evaluated = import (pkgs.path + "/nixos/lib/eval-config.nix") { system = pkgs.stdenv.hostPlatform.system; modules = [ (pkgs.path + "/nixos/modules/virtualisation/qemu-vm.nix") - # `d2bCloudHypervisorNode` is `d2bDaemonNode` with the writable store - # opted into, so the two shapes are one declaration read two ways and - # cannot drift apart. - (d2bNode.d2bDaemonNode { - extra = { imports = extraModules; }; - writableStore = nodeShape == "writable-store"; - }) + guestNode { virtualisation.host.pkgs = pkgs; } laneGuestModule - ]; + ] ++ lib.optional (checkNodeName != null) { _module.args.name = checkNodeName; }; }; guest = evaluated.config; toplevel = guest.system.build.toplevel; @@ -225,6 +391,40 @@ let cores = guest.virtualisation.cores; memorySizeMib = guest.virtualisation.memorySize; }; + # The check this guest was built for, and the one thing about a guest + # that is not part of its invocation: which check it exists for, and + # whether that check boots a nested guest inside it. `null` on the two + # shape-only images, which carry no check. + check = + if checkName == null then + null + else + { + name = checkName; + # The fixture's own name for the check, kept because it is what + # appears in a `vmChecks` derivation and in a driver log line, and + # a reader comparing the two should not have to know they differ. + testName = checkFixture.name or checkName; + # `useBootLoader` is the writable-store shape: the root drive is a + # writable overlay on an installed system image, which is the + # shape the Cloud Hypervisor checks boot their nested guest on. + # Restoring a member that has run a nested guest is not something + # the lane will attempt, so the flag that keeps it from attempting + # one is read off the node rather than off a list of check names + # the lane maintains. + nestedGuest = useBootLoader; + }; + # What one pool member costs the host, in the three currencies R8 names. + # Read off the node's own declared fields so the pool's bound cannot + # drift from the guests it is bounding: the memory and the vCPU count are + # the machine the launcher passes the emulator, and the working directory + # is the root disk the launcher copies into the directory it owns. + footprint = { + memorySizeMib = guest.virtualisation.memorySize; + cores = guest.virtualisation.cores; + workingDirectoryMib = diskSizeMib; + }; + pool = poolBudget; boot = { method = if useBootLoader then "bootloader" else "direct"; kernel = if useBootLoader then null else "kernel"; @@ -241,7 +441,7 @@ let contract = "d2b-daemon-acceptance"; marker = activationMarker; inherit serialDevice; - acceptanceUnitsFile = "/etc/d2b/daemon-acceptance-units"; + inherit acceptanceUnitsFile; shape = nodeShape; }; init = "${toplevel}/init"; @@ -393,11 +593,26 @@ let } >/dev/${serialDevice} 2>&1 || true say "end ordering state" } + # The units the contract waits on. A node that declares its own + # - the re-homed daemon node lists the three units whose activation + # the d2b checks were written against - is waited on for those. A + # node that declares none is a check that boots a plain NixOS guest + # and never wanted the daemon host at all, and waiting on an empty + # list would report the guest ready before a single one of its own + # units had started. `multi-user.target` is the unit those checks' + # own assertions already wait for, so the guest's readiness means + # the same thing to the launcher as it does to the check. # Plain seconds. The systemd time span above carries an `s` because # that is a duration; shell arithmetic does not, and `1800s` is not # a number - it is a base the shell cannot read - so a deadline # written that way aborts this script on its first statement under # `set -e`, and the guest reports nothing at all. + units_file=/run/d2b-lane-acceptance-units + if [ -r ${acceptanceUnitsFile} ]; then + cat ${acceptanceUnitsFile} >"$units_file" + else + printf 'multi-user.target\n' >"$units_file" + fi deadline=$(( $(date +%s) + ${toString activationTimeoutSeconds} )) units="" while read -r unit; do @@ -430,7 +645,7 @@ let sleep 1 done say "$unit is active" - done "$out/manifest.sorted" mv "$out/manifest.sorted" "$out/manifest.json" + + # The check's own assertions, evaluated out of its fixture and carried + # beside the guest they run against. The lane runs this text through the + # guest-control surface it re-provides for unported checks, so a check + # that has not been ported yet executes the assertions it always did + # rather than a lane-authored paraphrase of them. + ${lib.optionalString (checkScript != null) '' + cat >"$out/check.py" <<'PY' + ${checkScript} + PY + ''} '' diff --git a/packages/d2b-vm-harness/BUILD.bazel b/packages/d2b-vm-harness/BUILD.bazel index e00f5b574..53f0c01a0 100644 --- a/packages/d2b-vm-harness/BUILD.bazel +++ b/packages/d2b-vm-harness/BUILD.bazel @@ -14,7 +14,14 @@ load( package(default_visibility = ["//bazel/checks:__pkg__", "//bazel/checks/vm:__pkg__"]) exports_files( - ["BUILD.bazel", "Cargo.toml"], + [ + "BUILD.bazel", + "Cargo.toml", + # The nix lane's fixtures interpolate this same text into their + # evaluated assertions, and they read it out of the tree by path, so + # the guest-image action has to be able to stage it. + "src/diagnostics.py", + ], visibility = ["//visibility:public"], ) diff --git a/packages/d2b-vm-harness/src/bin/d2b-vm-harness.rs b/packages/d2b-vm-harness/src/bin/d2b-vm-harness.rs index 095775774..aad8683c4 100644 --- a/packages/d2b-vm-harness/src/bin/d2b-vm-harness.rs +++ b/packages/d2b-vm-harness/src/bin/d2b-vm-harness.rs @@ -16,10 +16,17 @@ //! that is how the Bazel test target hands it the emulator from the pinned //! nix package set and the guest image from the guest-image action. -use std::{env, fs, path::{Path, PathBuf}, process::ExitCode, time::Duration}; +use std::{ + env, fs, + path::{Path, PathBuf}, + process::ExitCode, + thread, + time::{Duration, Instant}, +}; use d2b_vm_harness::{ - GuestSpec, HarnessError, boot, host, manifest::GuestManifest, report, + Footprint, GuestSpec, HarnessError, HostFacts, LegacyCheck, LegacyGuest, boot, host, + manifest::GuestManifest, report, }; use serde_json::json; @@ -51,13 +58,20 @@ const REFUSAL_DEVICE: &str = "lane-refusal"; const REFUSAL_DEVICE_BYTES: u64 = 8 * 1024 * 1024; fn main() -> ExitCode { - match run() { - Ok(report) => { + let mut arguments = env::args().skip(1); + let outcome = match arguments.next().as_deref() { + Some("lane") => run_lane(arguments.collect()), + Some("self-check") | None => run().map(|report| { for line in report { report_line(&line); } - ExitCode::SUCCESS - } + }), + Some(other) => Err(HarnessError::Configuration(format!( + "unknown subcommand {other:?}; the harness runs the lane or a guest self-check" + ))), + }; + match outcome { + Ok(()) => ExitCode::SUCCESS, Err(error) => { report_line(&format!("FAIL {error}")); ExitCode::FAILURE @@ -65,6 +79,675 @@ fn main() -> ExitCode { } } +// --------------------------------------------------------------------------- +// The lane. +// +// One image per check, one pool member per distinct emulator invocation, one +// result per check. The shape of the run, and why each part is here: +// +// * the images are read off the graph, each one built from its own check's +// fixture, and each one's manifest says what that check's guest is; +// * the checks are grouped by the invocation they declare, because that - +// not the closure - is what decides whether two guests can be booted +// side by side on one host and run one after another on one member; +// * the pool is sized from the distinct invocations against the budget the +// guest configurations declared and what this host has free, so the +// bound is a number the run measured rather than a constant somebody +// picked; +// * every member is proven snapshottable from its declaration before the +// first one boots, and again from the running guest's own block graph +// before its snapshot is taken; +// * a member that has run a nested guest is retired rather than restored; +// * every check runs against a snapshot-restored copy of its own guest, +// after a marker gate has compared that guest fresh against that guest +// restored. + +/// The image list the lane's target generated, one guest image per line. +const IMAGES: &str = "D2B_VM_HARNESS_IMAGES"; +/// The checks a contributor selected, as the existing selection variables +/// carry them: a whitespace- or comma-separated list of check names. +const CHECKS: &str = "D2B_VM_CHECK"; + +/// The snapshot tag one member is restored from. The emulator's tags live in +/// the guest's own directory, so a member only ever has to name its own. +const SNAPSHOT_TAG: &str = "lane-base"; + +/// One check, its own guest, and what that guest costs. +#[derive(Clone)] +struct LaneGuest { + image_dir: PathBuf, + manifest: GuestManifest, + name: String, + nested_guest: bool, + footprint: Footprint, +} + +/// A set of checks whose guests are booted the same way, and which therefore +/// run one at a time rather than all at once. +#[derive(Clone)] +struct InvocationGroup { + key: String, + guests: Vec, +} + +impl InvocationGroup { + /// What one member of this group costs: the most expensive guest the + /// group holds, because the group runs its members one after another and + /// the expensive one is the one that has to fit. + fn footprint(&self) -> Footprint { + self.guests + .iter() + .map(|guest| guest.footprint) + .max_by_key(|footprint| (footprint.memory_size_mib, footprint.cores)) + .unwrap_or(Footprint { + memory_size_mib: 0, + cores: 0, + working_directory_mib: 0, + }) + } + + /// The checks in this group, by name. + fn names(&self) -> String { + self.guests + .iter() + .map(|guest| guest.name.as_str()) + .collect::>() + .join(", ") + } +} + +/// What one check produced, and what a reader of the lane's output needs to +/// act on it. +struct CheckResult { + name: String, + group: String, + passed: bool, + seconds: f64, + detail: String, +} + +/// The equivalence gate's marker: the guest-side facts a restored guest has +/// to reproduce for the restore to be indistinguishable from the fresh boot +/// the checks were written against. +/// +/// It is a marker rather than a checksum on purpose. A checksum of the guest +/// would differ between the two by everything the boot legitimately changed +/// between the two moments - the uptime, the boot id, whatever a periodic +/// job wrote - and a gate that reports that difference has told the reader +/// nothing about whether the restore worked. What it compares instead is the +/// set of conditions each check's own assertions were written against: the +/// guest's declared activation units are active, the d2b host-tool set the +/// closure was built with is installed, the state disk the node mounted is +/// mounted where it mounted it, and the guest's random pool is out of its +/// initialising state. A restore that drops any of those is a restore that +/// would break a check, and that is the failure worth catching. +const EQUIVALENCE_MARKER: &str = r#" +import subprocess +import sys + +start_all() + +def report(label, command): + status, output = machine.execute(command, timeout=60) + print("d2b-lane-marker {}={} {}".format(label, status, output.strip())) + +units_file = "/etc/d2b/daemon-acceptance-units" +try: + with open(units_file) as handle: + units = [line.strip() for line in handle if line.strip()] +except OSError: + units = ["multi-user.target"] + +for unit in units: + status, output = machine.execute( + "systemctl show {} --property=ActiveState --property=SubState --property=Result" + " --no-pager".format(unit), + timeout=60, + ) + print("d2b-lane-marker unit {}= {} {}".format(unit, status, output.strip().replace("\n", " "))) + +report("acceptanceUnitsFile", "cat {}".format(units_file)) +report("stateDisk", "findmnt -n -o TARGET,SOURCE,FSTYPE /var/lib/d2b || true") +report("hostTools", "ls /run/d2b-host-tools 2>/dev/null || ls /nix/var/nix/profiles/default/bin 2>/dev/null | head -n 0; echo inventory") +report("activation", "systemctl show d2b-lane-activation --property=ActiveState --property=Result --no-pager") +report("crng", "journalctl -b --no-pager -o cat | grep -c 'crng init done' || true") +report("backdoor", "systemctl is-active backdoor.service") +print("d2b-lane-marker complete") +"#; + +/// Run the lane: read the images, size the pool, run the checks, report. +#[allow(clippy::disallowed_methods, reason = "synchronous path")] +fn run_lane(arguments: Vec) -> Result<(), HarnessError> { + let emulator = required_path(EMULATOR)?; + let work_root = work_root(env::current_dir().map_err(|error| { + HarnessError::io("reading the lane's working directory", error) + })?)?; + fs::create_dir_all(&work_root) + .map_err(|error| HarnessError::io(format!("creating {}", work_root.display()), error))?; + + // The host preconditions gate everything after them, and they are asked + // before a single guest image is read: a host that cannot run the lane + // should learn so in a second, not after eleven closures. + host::require_this_host()?; + + let selected = selection(&arguments); + let guests = read_guests(&selected)?; + if guests.is_empty() { + return Err(HarnessError::Configuration( + "no guest image carried a check the lane could select".to_owned(), + )); + } + + // A device that cannot carry an internal snapshot fails the lane here, + // before the pool is built and before any guest boots. The running + // guest's own block graph is asked again once it is up, but a pool that + // boots five members and then refuses the sixth has spent the run. + for guest in &guests { + let refused = guest.manifest.unsnapshottable_drives(); + if !refused.is_empty() { + return Err(HarnessError::Configuration(format!( + "the guest image for check '{}' attaches a writable device that cannot carry an \ + internal snapshot, and the lane will not run without restore: {}", + guest.name, + refused.join("; ") + ))); + } + } + + let groups = group_by_invocation(guests); + let admitted = admit(&groups, &work_root)?; + for group in &admitted { + report_line(&format!( + "pool: {} ({} vCPU, {} MiB, {} MiB of working directory) <- {}", + group.names(), + group.footprint().cores, + group.footprint().memory_size_mib, + group.footprint().working_directory_mib, + group.key, + )); + } + + let outcomes: Vec> = + thread::scope(|scope| { + let emulator = emulator.as_path(); + let work_root = work_root.as_path(); + let handles: Vec<_> = admitted + .iter() + .map(|group| scope.spawn(move || run_group(group, emulator, work_root))) + .collect(); + handles + .into_iter() + .flat_map(|handle| { + handle + .join() + .unwrap_or_else(|_| panic!("a pool group thread panicked")) + }) + .collect() + }); + +/// One invocation group's checks, in the order they were selected. +/// +/// Sequential on purpose: a group is a set of checks whose guests are booted +/// the same way, and the pool restores a member between them rather than +/// booting a second member of the same shape beside it. The concurrency is +/// across groups, which is where the memory and the vCPUs differ. +#[allow(clippy::disallowed_methods, reason = "synchronous path")] +fn run_group( + group: &InvocationGroup, + emulator: &Path, + work_root: &Path, +) -> Vec> { + group + .guests + .iter() + .map(|guest| run_check(guest, emulator, work_root)) + .collect() +} + + let mut results: Vec = Vec::new(); + let mut failures: Vec = Vec::new(); + for outcome in outcomes { + match outcome { + Ok(result) => { + report_line(&format!( + "{}: {} in {:.1}s", + if result.passed { "PASS" } else { "FAIL" }, + result.name, + result.seconds, + )); + if !result.passed { + failures.push(result.name.clone()); + } + results.push(result); + } + Err((name, error)) => { + report_line(&format!("FAIL {name}: {error}")); + failures.push(name.clone()); + results.push(CheckResult { + name, + group: String::new(), + passed: false, + seconds: 0.0, + detail: error.to_string(), + }); + } + } + } + + write_junit(&results)?; + if failures.is_empty() { + report_line(&format!("lane: {} checks, all green", results.len())); + Ok(()) + } else { + Err(HarnessError::Configuration(format!( + "{} of {} checks failed: {}", + failures.len(), + results.len(), + failures.join(", ") + ))) + } +} + +/// The checks this run was asked for, from the target's own argument or from +/// the environment variable contributors already use. +#[allow(clippy::disallowed_methods, reason = "synchronous path")] +fn selection(arguments: &[String]) -> Vec { + let from_arguments: Vec = arguments + .iter() + .skip_while(|argument| *argument != "--check") + .skip(1) + .flat_map(|argument| argument.split([',', ' '])) + .map(str::trim) + .filter(|name| !name.is_empty()) + .map(str::to_owned) + .collect(); + if !from_arguments.is_empty() { + return from_arguments; + } + env::var(CHECKS) + .unwrap_or_default() + .split([',', ' ', '\n', '\t']) + .map(str::trim) + .filter(|name| !name.is_empty()) + .map(str::to_owned) + .collect() +} + +/// Every check the lane can run, read off the images the target declared. +/// +/// A name is matched against both the check's own name and the name its +/// fixture gave it, because a contributor filtering the lane has whichever +/// of the two they have read. +#[allow(clippy::disallowed_methods, reason = "synchronous path")] +fn read_guests(selected: &[String]) -> Result, HarnessError> { + let listing = fs::read_to_string(required_path(IMAGES)?).map_err(|error| { + HarnessError::io("reading the lane's image list", error) + })?; + let mut guests = Vec::new(); + for line in listing.lines().map(str::trim).filter(|line| !line.is_empty()) { + let image_dir = resolve_runfile(PathBuf::from(line)); + let manifest = GuestManifest::load(&image_dir)?; + let Some(check) = &manifest.check else { + continue; + }; + if !selected.is_empty() + && !selected.iter().any(|name| name == &check.name || name == &check.test_name) + { + continue; + } + guests.push(LaneGuest { + image_dir, + footprint: manifest.footprint, + name: check.name.clone(), + nested_guest: check.nested_guest, + manifest, + }); + } + guests.sort_by(|left, right| left.name.cmp(&right.name)); + Ok(guests) +} + +/// A path out of the lane's image listing, resolved the way the rest of the +/// harness resolves a runfile. +/// +/// The listing is generated by the test target from `$(rlocationpath)`, so +/// every line is relative to the *runfiles root* - `_main/...` - while a +/// test's own working directory is the `_main` directory inside that tree. +/// Taking a line as written would look for `_main/_main/...`. An absolute path +/// is left alone, and with no runfiles tree declared the path is the caller's +/// own, which is what a contributor running the harness by hand means. +fn resolve_runfile(path: PathBuf) -> PathBuf { + if path.is_absolute() { + return path; + } + let root = [env::var_os("RUNFILES_DIR"), env::var_os("TEST_SRCDIR")] + .into_iter() + .flatten() + .next(); + match root { + Some(root) => PathBuf::from(root).join(&path), + None => path, + } +} + +/// Group the checks by the invocation their guests declare. +/// +/// The order is the group's cheapest member first and the group name after +/// it, so the pool admits the smallest groups first and the admission is +/// reproducible rather than dependent on which label the graph happened to +/// list. +fn group_by_invocation(guests: Vec) -> Vec { + let mut groups: Vec = Vec::new(); + for guest in guests { + let key = guest.manifest.invocation_key(); + match groups.iter_mut().find(|group| group.key == key) { + Some(group) => group.guests.push(guest), + None => groups.push(InvocationGroup { + key, + guests: vec![guest], + }), + } + } + groups.sort_by(|left, right| { + let left_cost = left.footprint(); + let right_cost = right.footprint(); + (left_cost.memory_size_mib, left_cost.cores, left.key.clone()) + .cmp(&(right_cost.memory_size_mib, right_cost.cores, right.key.clone())) + }); + groups +} + +/// How many of the groups this host can hold at once, and which. +/// +/// The pool's size is the number of distinct invocations, bounded by what the +/// host has: the budget the guest configurations declared, applied to the +/// memory and the vCPUs this host has free, and the working directory the +/// admitted members will copy their root disks into. A group that does not +/// fit is not an error - it waits for a member to finish - and a host that +/// cannot hold even the cheapest group is a host that cannot run the lane at +/// all, which is worth saying plainly rather than booting a guest that +/// cannot get the memory its node declared. +fn admit( + groups: &[InvocationGroup], + work_root: &Path, +) -> Result, HarnessError> { + let first = groups + .first() + .ok_or_else(|| HarnessError::Configuration("the lane has no checks to run".to_owned()))?; + let budget = first.guests[0].manifest.pool; + let available_memory = HostFacts::available_memory_mib().ok_or_else(|| { + HarnessError::Configuration( + "this host does not report how much memory it has free, so the pool cannot be sized \ + against a budget rather than a guess" + .to_owned(), + ) + })?; + let available_cores = HostFacts::available_cores(); + // Multiply first, then divide, the way the core budget below does. + // Written as `available * numerator.checked_div(denominator)` the + // method call binds to the numerator alone, so the share is taken of `2` + // rather than of what this host has, and a two-in-three budget is a + // truncation to zero - a pool that admits nothing, reported as a + // configuration the guest never got to say anything about. + let memory_budget = available_memory + .checked_mul(budget.memory_share_numerator) + .map(|scaled| scaled / budget.memory_share_denominator.max(1)) + .filter(|share| *share > 0) + .ok_or_else(|| { + HarnessError::Configuration( + "the declared memory budget is not a share of anything".to_owned(), + ) + })?; + let core_budget = (available_cores * budget.core_share_numerator + / budget.core_share_denominator.max(1)) + .max(1); + report_line(&format!( + "budget: {} MiB of the {} MiB this host has free, and {} of its {} vCPUs", + memory_budget, + available_memory, + core_budget, + available_cores, + )); + + let free_directory = HostFacts::free_working_directory_mib(work_root); + let mut memory_used: u64 = 0; + let mut cores_used: u64 = 0; + let mut directory_used: u64 = 0; + let mut admitted = Vec::new(); + for (index, group) in groups.iter().enumerate() { + let cost = group.footprint(); + let fits_memory = memory_used + cost.memory_size_mib <= memory_budget; + let fits_cores = cores_used + u64::from(cost.cores) <= core_budget; + let fits_directory = free_directory.is_none_or(|free| { + directory_used + cost.working_directory_mib <= free + }); + if index == 0 && !(fits_memory && fits_cores) { + return Err(HarnessError::Configuration(format!( + "this host cannot hold even the cheapest pool member: the smallest invocation \ + needs {} MiB and {} vCPUs, and the budget is {} MiB and {} vCPUs out of {} MiB \ + and {} vCPUs free", + cost.memory_size_mib, + cost.cores, + memory_budget, + core_budget, + available_memory, + available_cores + ))); + } + if !(fits_memory && fits_cores && fits_directory) { + report_line(&format!( + "pool: {} does not fit the budget and will run after a member finishes", + group.names() + )); + continue; + } + memory_used += cost.memory_size_mib; + cores_used += u64::from(cost.cores); + directory_used += cost.working_directory_mib; + admitted.push(group.clone()); + } + if let Some(free) = free_directory { + report_line(&format!( + "working directory: {} MiB needed for the admitted pool, {} MiB free at {}", + directory_used, + free, + work_root.display() + )); + } else { + report_line(&format!( + "working directory: {} MiB needed for the admitted pool; free space at {} could not \ + be measured, so the declared bound is enforced by each member's own copy", + directory_used, + work_root.display() + )); + } + Ok(admitted) +} +/// Boot one check's own guest, prove the restore against it, and run the +/// check on a restored copy of it. +#[allow(clippy::disallowed_methods, reason = "synchronous path")] +fn run_check( + guest: &LaneGuest, + emulator: &Path, + work_root: &Path, +) -> std::result::Result { + let name = guest.name.clone(); + let started = Instant::now(); + let outcome = run_check_inner(guest, emulator, work_root, started); + match outcome { + Ok(result) => Ok(result), + Err(error) => Err((name, error)), + } +} + +#[allow(clippy::disallowed_methods, reason = "synchronous path")] +fn run_check_inner( + guest: &LaneGuest, + emulator: &Path, + work_root: &Path, + started: Instant, +) -> Result { + let mut spec = GuestSpec::new( + guest.manifest.clone(), + &guest.image_dir, + emulator, + work_root, + &guest.name, + ); + spec.activation_timeout = Duration::from_secs( + optional_u64(ACTIVATION_TIMEOUT, 1800)?, + ); + let mut active = boot(&spec)?; + // The declared pass said this guest can be snapshotted; the running + // guest's own block graph is the authority, and it is asked before the + // snapshot rather than at the first restore. + active.require_snapshottable()?; + + let mut surface = LegacyGuest::attach(&mut active)?; + let marker = surface + .run(&LegacyCheck::new(format!("{}-marker", guest.name), EQUIVALENCE_MARKER)) + .map_err(|error| { + HarnessError::Configuration(format!( + "the equivalence marker could not be read from the fresh guest: {error}" + )) + })?; + if !marker.passed { + return Err(HarnessError::Configuration(format!( + "the equivalence marker failed against a freshly booted guest, so it cannot say \ + anything about a restored one:\n{}", + marker.detail + ))); + } + active.save_snapshot(SNAPSHOT_TAG)?; + active.restore(SNAPSHOT_TAG)?; + let restored = surface + .run(&LegacyCheck::new(format!("{}-marker", guest.name), EQUIVALENCE_MARKER)) + .map_err(|error| { + HarnessError::Configuration(format!( + "the equivalence marker could not be read from the restored guest: {error}" + )) + })?; + if !restored.passed { + return Err(HarnessError::Configuration(format!( + "the equivalence marker failed against a restored guest:\n{}", + restored.detail + ))); + } + let fresh_text = marker_text(&marker.detail); + let restored_text = marker_text(&restored.detail); + if fresh_text != restored_text { + return Err(HarnessError::Configuration(format!( + "a restored guest is not the fresh boot its check was written against\n--- fresh boot\n{}\n--- restored\n{}", + fresh_text, restored_text + ))); + } + report_line(&format!( + "{}: the restored guest matches the fresh boot on every marker ({} bytes)", + guest.name, + fresh_text.len() + )); + + // The check runs on a restored guest: the marker gate above left the + // member where its snapshot was taken, and the gate is not part of what + // the check is being handed. + active.restore(SNAPSHOT_TAG)?; + let script = fs::read_to_string(guest.image_dir.join("check.py")).map_err(|error| { + HarnessError::io( + format!("reading the assertions of check '{}'", guest.name), + error, + ) + })?; + let outcome = surface.run(&LegacyCheck::new(&guest.name, script))?; + let seconds = started.elapsed().as_secs_f64(); + report_line(&format!( + "{}: {} after the restore ({seconds:.1}s total on this member)", + guest.name, + if guest.nested_guest { + "ran a nested guest; this member is retired rather than restored" + } else { + "restored from the pool's snapshot" + }, + )); + if guest.nested_guest { + // Retired, not restored. A guest with a live guest inside it has no + // defined restored state, and a member that has run a nested guest + // is torn down here rather than handed to the next check. + let _ = active.discard_snapshot(SNAPSHOT_TAG); + } + active.shutdown()?; + Ok(CheckResult { + name: guest.name.clone(), + group: guest.manifest.invocation_key(), + passed: outcome.passed, + seconds, + detail: outcome.detail, + }) +} + +/// The marker lines out of a check's own output, with the lane's own log +/// lines around them dropped: the surface's log describes what it did, and +/// two runs of the same gate legitimately log different durations. +fn marker_text(detail: &str) -> String { + detail + .lines() + .filter(|line| line.trim_start().starts_with("d2b-lane-marker")) + .collect::>() + .join("\n") +} + +/// Write the lane's JUnit document: one testcase per check, carrying that +/// check's own diagnostics. +/// +/// Bazel names the file through `XML_OUTPUT_FILE` when it runs a test under +/// its XML wrapper, which is the default; the fallback keeps the document +/// findable when the harness is run by hand. +#[allow(clippy::disallowed_methods, reason = "synchronous path")] +fn write_junit(results: &[CheckResult]) -> Result<(), HarnessError> { + let path = env::var_os("XML_OUTPUT_FILE").map(PathBuf::from).unwrap_or_else(|| { + let directory = env::var_os("TEST_SRCDIR") + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from(".")); + let target = env::var("TEST_TARGET").unwrap_or_else(|_| "host_integration_lane".to_owned()); + directory.join(format!("{}.test.xml", target.replace(['/', ':'], "_"))) + }); + let mut document = String::from("\n\n"); + for result in results { + document.push_str(&format!( + " \n \n", + escape(&result.group), + u8::from(!result.passed), + escape(&result.group), + escape(&result.name), + seconds = result.seconds, + )); + if result.passed { + document.push_str(" \n"); + } else { + document.push_str(&format!( + " {}\n \n", + escape(&result.name), + escape(&result.detail), + )); + } + document.push_str(" \n"); + } + document.push_str("\n"); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent) + .map_err(|error| HarnessError::io(format!("creating {}", parent.display()), error))?; + } + fs::write(&path, document) + .map_err(|error| HarnessError::io(format!("writing {}", path.display()), error)) +} + +fn escape(text: &str) -> String { + text.replace('&', "&") + .replace('<', "<") + .replace('>', ">") + .replace('"', """) + .replace('\'', "'") +} + /// The lane's working directory, resolved against the caller's own directory /// when the caller named a relative one. The lane's directory has to outlive /// each individual guest, so it cannot be a per-action temporary directory diff --git a/packages/d2b-vm-harness/src/guest.rs b/packages/d2b-vm-harness/src/guest.rs index 29a6f5d6f..c0157e15b 100644 --- a/packages/d2b-vm-harness/src/guest.rs +++ b/packages/d2b-vm-harness/src/guest.rs @@ -551,6 +551,48 @@ impl ActiveGuest { } } + /// Take the pool's snapshot of this guest, under a tag of the lane's + /// choosing. + /// + /// This is the point the pool snapshots at: after activation has + /// completed, after every attached writable device has been proven to + /// carry a snapshot, and before any check has touched the guest. A + /// snapshot taken later would be a snapshot of whatever the last check + /// left behind. + pub fn save_snapshot(&mut self, tag: &str) -> Result<()> { + self.monitor_mut()?.save_snapshot(tag) + } + + /// Restore this guest from a snapshot it took itself. + /// + /// The guest's command channel survives the restore. An internal + /// snapshot captures the guest's memory and its devices, not the host + /// socket at the far end of the guest's console, and the emulator is the + /// same process across the restore - so the connection the launcher + /// accepted before the boot is still the connection the guest's root + /// shell is reading from afterwards. That is also why the snapshot is + /// taken with the channel idle: bytes already in the console are not + /// part of what a restore rolls back. + pub fn restore(&mut self, tag: &str) -> Result<()> { + self.monitor_mut()?.load_snapshot(tag) + } + + /// Whether this guest currently holds a snapshot under a tag. + pub fn holds_snapshot(&mut self, tag: &str) -> Result { + Ok(self.monitor_mut()?.snapshot_tags()?.iter().any(|held| held == tag)) + } + + /// Drop a snapshot, which is what retiring a member frees. + pub fn discard_snapshot(&mut self, tag: &str) -> Result<()> { + self.monitor_mut()?.delete_snapshot(tag) + } + + fn monitor_mut(&mut self) -> Result<&mut Monitor> { + self.monitor.as_mut().ok_or_else(|| { + HarnessError::Configuration("the guest has no monitor".to_owned()) + }) + } + /// Ask the emulator to stop, wait for the process to go, and remove the /// working directory. pub fn shutdown(mut self) -> Result<()> { @@ -916,7 +958,7 @@ pub fn report(line: &str) { mod tests { use super::*; use crate::manifest::{ - Activation, Boot, Drive, ImageFiles, Machine, SharedDirectory, + Activation, Boot, Drive, Footprint, ImageFiles, Machine, PoolBudget, SharedDirectory, }; fn manifest(cores: u32, memory: u32, extra: &[&str]) -> GuestManifest { @@ -934,6 +976,19 @@ mod tests { cores, memory_size_mib: memory, }, + check: None, + footprint: Footprint { + memory_size_mib: u64::from(memory), + cores, + working_directory_mib: 8192, + }, + pool: PoolBudget { + memory_share_numerator: 2, + memory_share_denominator: 3, + core_share_numerator: 3, + core_share_denominator: 4, + working_directory_follows_disk: true, + }, boot: Boot { method: "direct".to_owned(), kernel: Some("kernel".to_owned()), diff --git a/packages/d2b-vm-harness/src/host.rs b/packages/d2b-vm-harness/src/host.rs index 5d431f641..ddda02109 100644 --- a/packages/d2b-vm-harness/src/host.rs +++ b/packages/d2b-vm-harness/src/host.rs @@ -151,6 +151,68 @@ impl HostFacts { .and_then(|(_, value)| value.as_deref()) .map(is_enabled) } + + /// The memory this host has free, in MiB, or `None` when it will not + /// say. + /// + /// `MemAvailable` and not `MemFree`: the lane runs on a contributor's own + /// machine, where the memory a guest can have is the memory the kernel + /// says is reclaimable, not the memory nothing happens to be using. + #[allow(clippy::disallowed_methods, reason = "synchronous path")] + pub fn available_memory_mib() -> Option { + meminfo_field("/proc/meminfo", "MemAvailable") + } + + /// The vCPUs this host will schedule, which is the ceiling the pool's + /// core share is a fraction of. + pub fn available_cores() -> u64 { + std::thread::available_parallelism() + .map(|cores| cores.get() as u64) + .unwrap_or(1) + } + + /// The free space on the filesystem the lane's working directory is on, + /// in MiB, or `None` when it cannot be read. + /// + /// Read with `df` rather than from a syscall the crate has no binding + /// for. A lane that cannot measure the space its own members copy their + /// root disks into would find out from `ENOSPC` halfway through a run, + /// which is the failure this exists to turn into a decision made before + /// the first guest boots. + #[allow(clippy::disallowed_methods, reason = "synchronous path")] + pub fn free_working_directory_mib(path: &Path) -> Option { + let output = std::process::Command::new("df") + .args(["-P", "--block-size=1", "--output=avail"]) + .arg(path) + .output() + .ok()?; + if !output.status.success() { + return None; + } + let text = String::from_utf8(output.stdout).ok()?; + text.lines() + .nth(1)? + .split_whitespace() + .next()? + .parse::() + .ok() + .map(|bytes| bytes / (1024 * 1024)) + } +} + +/// One `key: value kB` field out of `/proc/meminfo`, in MiB. +#[allow(clippy::disallowed_methods, reason = "synchronous path")] +fn meminfo_field(path: &str, key: &str) -> Option { + let text = fs::read_to_string(path).ok()?; + let line = text + .lines() + .find(|line| line.starts_with(key) && line.as_bytes().get(key.len()) == Some(&b':'))?; + line[key.len() + 1..] + .split_whitespace() + .next()? + .parse::() + .ok() + .map(|kib| kib / 1024) } #[allow(clippy::disallowed_methods, reason = "synchronous path")] diff --git a/packages/d2b-vm-harness/src/lib.rs b/packages/d2b-vm-harness/src/lib.rs index a7f9df0cf..424bbdc95 100644 --- a/packages/d2b-vm-harness/src/lib.rs +++ b/packages/d2b-vm-harness/src/lib.rs @@ -26,5 +26,5 @@ pub use error::{HarnessError, Result, UnsnapshottableDevice}; pub use guest::{ActiveGuest, GuestSpec, boot, report, reserve_loopback_port}; pub use legacy::{LegacyCheck, LegacyError, LegacyGuest, LegacyOutcome}; pub use host::{Capability, HostFacts, require_this_host}; -pub use manifest::GuestManifest; +pub use manifest::{CheckRecord, Footprint, GuestManifest, PoolBudget}; pub use monitor::{BlockDevice, Monitor}; diff --git a/packages/d2b-vm-harness/src/manifest.rs b/packages/d2b-vm-harness/src/manifest.rs index 3d3ff2469..b5f7a7220 100644 --- a/packages/d2b-vm-harness/src/manifest.rs +++ b/packages/d2b-vm-harness/src/manifest.rs @@ -31,6 +31,15 @@ pub struct GuestManifest { pub node_shape: String, /// The files the launcher boots, relative to the image root. pub image: ImageFiles, + /// The check this guest was built for, and whether that check boots a + /// nested guest inside it. `None` on a shape-only image, which carries + /// no check. + pub check: Option, + /// What one pool member costs the host. + pub footprint: Footprint, + /// The bound the lane's pool is sized against, declared next to the + /// guests rather than supplied by a caller. + pub pool: PoolBudget, /// The machine size the node declared. pub machine: Machine, /// How the guest is booted, and with what. @@ -80,6 +89,57 @@ pub struct ImageFiles { pub system_image: Option, } +/// The check a guest image was built for. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct CheckRecord { + /// The check's name, as the lane reports it and as a contributor filters + /// it. + pub name: String, + /// The name the check's own fixture gave it, which is what appears in a + /// driver log line. + pub test_name: String, + /// Whether this guest runs a guest of its own. A member that has is + /// retired rather than restored, because restoring a guest with a live + /// guest inside it is not defined behaviour. + pub nested_guest: bool, +} + +/// What one pool member costs the host, in the three currencies the pool is +/// bounded in. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct Footprint { + /// The memory the member's guest reserves, in MiB. + pub memory_size_mib: u64, + /// The vCPUs the member's guest holds. + pub cores: u32, + /// The lane working directory the member needs for its own root disk, in + /// MiB. The launcher copies that disk into the directory the member + /// owns, so this is the disk the node declared rather than a number the + /// lane chose. + pub working_directory_mib: u64, +} + +/// The share of the host the pool may take, as the guest configurations +/// declared it. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct PoolBudget { + /// The numerator of the memory share. + pub memory_share_numerator: u64, + /// The denominator of the memory share. + pub memory_share_denominator: u64, + /// The numerator of the vCPU share. + pub core_share_numerator: u64, + /// The denominator of the vCPU share. + pub core_share_denominator: u64, + /// Whether a member's working directory follows the disk its node + /// declared. Declared rather than assumed, because a node that grows its + /// root disk grows the lane's working directory with it. + pub working_directory_follows_disk: bool, +} + /// The machine size the node declared. #[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] @@ -206,6 +266,105 @@ impl GuestManifest { pub fn uses_bootloader(&self) -> bool { self.boot.method == "bootloader" } + + /// The identity of the emulator invocation this guest is booted with. + /// + /// Two checks share a pool member when their guests are booted the same + /// way, and that is a statement about the invocation alone: the machine + /// size, the root disk, the boot method, the drives, the per-check device + /// options, and the networking. The closure deliberately does not appear + /// in it. The system closure and the host-tool package are what make one + /// guest a different guest from another, and a group of checks that + /// shared a closure would be a group that shared an image - which is the + /// thing the lane stopped doing, because it is why one guest could not + /// serve a check that wanted nftables, a vsock device, or a nested guest. + pub fn invocation_key(&self) -> String { + let drives: Vec = self + .drives + .iter() + .map(|drive| { + serde_json::json!({ + "interface": drive.interface, + "cache": drive.cache, + "werror": drive.werror, + "bootIndex": drive.boot_index, + "serial": drive.serial, + "format": drive.format, + // An image-relative file is the guest's own root disk and + // is the same file for every guest; a store path is a + // device the node attached itself, and the path is what + // says which one. + "ownDisk": !Path::new(&drive.file).is_absolute(), + }) + }) + .collect(); + let key = serde_json::json!({ + "method": self.boot.method, + "memorySizeMib": self.machine.memory_size_mib, + "cores": self.machine.cores, + "diskSizeMib": self.image.disk_size_mib, + "bootloader": self.image.system_image.is_some(), + "drives": drives, + "extraOptions": self.extra_options, + "networkingOptions": self.networking_options, + "shares": self + .shared_directories + .iter() + .map(|share| (share.mount_tag.clone(), share.security_model.clone())) + .collect::>(), + }); + serde_json::to_string(&key).unwrap_or_default() + } + + /// Refuse a guest whose attached writable devices cannot carry an + /// internal snapshot, before a pool is built around it. + /// + /// The emulator's own verdict is the authority - it is asked again over + /// the monitor once the guest is running - but it can only be asked of a + /// running guest, and a pool that boots five members and then discovers + /// the sixth cannot be snapshotted has already spent the run it was + /// supposed to explain. This is the pass that reads it off the + /// declaration instead: in the current emulator qcow2 is the only + /// format with the snapshot vtable, a single writable node without it + /// fails the save for the whole guest rather than for that node, and a + /// node that asks for an ephemeral overlay is one whose writes live in a + /// qcow2 the emulator creates for it. + pub fn unsnapshottable_drives(&self) -> Vec { + let mut refused: Vec = Vec::new(); + for drive in &self.drives { + let own_disk = !Path::new(&drive.file).is_absolute(); + let image_format = own_disk.then_some(self.image.disk_format.as_str()); + if !matches!(drive.format.as_deref().or(image_format), Some("qcow2")) { + refused.push(format!( + "{} ({})", + drive.file, + drive.format.as_deref().unwrap_or("no format") + )); + } + } + // A drive the node attached through the option list rather than + // through the drive list: the state disk the re-homed node mounts at + // `/var/lib/d2b` is declared exactly this way. + for (index, option) in self.extra_options.iter().enumerate() { + if !option.starts_with("file=") { + continue; + } + let option_is_drive = self + .extra_options + .get(index.wrapping_sub(1)) + .is_some_and(|previous| previous == "-drive"); + if !option_is_drive { + continue; + } + if !option.contains("snapshot=on") + && !option.contains("format=qcow2") + && !option.contains("readonly=on") + { + refused.push(option.clone()); + } + } + refused + } } #[cfg(test)] @@ -217,6 +376,13 @@ mod tests { "system": "x86_64-linux", "nodeShape": "daemon", "image": {"disk": "disk.qcow2", "diskFormat": "qcow2", "diskSizeMib": 8192, "systemImage": null}, + "check": {"name": "daemon-smoke", "testName": "d2b-daemon-smoke", "nestedGuest": false}, + "footprint": {"memorySizeMib": 3072, "cores": 3, "workingDirectoryMib": 8192}, + "pool": { + "memoryShareNumerator": 2, "memoryShareDenominator": 3, + "coreShareNumerator": 3, "coreShareDenominator": 4, + "workingDirectoryFollowsDisk": true + }, "machine": {"cores": 3, "memorySizeMib": 3072}, "boot": {"method": "direct", "kernel": "kernel", "initrd": "initrd", "append": "console=ttyS0"}, "drives": [ diff --git a/packages/d2b-vm-harness/src/monitor.rs b/packages/d2b-vm-harness/src/monitor.rs index 4ed2ce506..bc48b2502 100644 --- a/packages/d2b-vm-harness/src/monitor.rs +++ b/packages/d2b-vm-harness/src/monitor.rs @@ -112,6 +112,84 @@ impl Monitor { Ok(parse_block_devices(&report)) } + /// Save the guest's whole state - every device, plus the machine state - + /// under a tag. + /// + /// `savevm` is the human-monitor spelling of it rather than the QMP + /// `snapshot-save` command, and the difference matters: `snapshot-save` + /// takes the list of devices to capture, and a list that is wrong in + /// either direction produces a snapshot that restores a guest missing + /// state or a guest whose extra state was never captured, both of which + /// look like a healthy restore. `savevm` captures what the guest has, and + /// refuses the whole save if any single device refuses - which is the + /// property the lane depends on, since one raw attached device would + /// otherwise be the one thing a later `loadvm` cannot undo. + pub fn save_snapshot(&mut self, tag: &str) -> Result<()> { + self.human(&format!("savevm {}", Self::quote(tag))) + .map(|_| ()) + } + + /// Restore the guest from a snapshot it saved itself. + pub fn load_snapshot(&mut self, tag: &str) -> Result<()> { + self.human(&format!("loadvm {}", Self::quote(tag))) + .map(|_| ()) + } + + /// Remove a snapshot, which is what retiring a member frees. + pub fn delete_snapshot(&mut self, tag: &str) -> Result<()> { + self.human(&format!("delvm {}", Self::quote(tag))) + .map(|_| ()) + } + + /// The tags this guest currently holds a snapshot under. + pub fn snapshot_tags(&mut self) -> Result> { + let report = self.execute("snapshot-list")?; + Ok(report + .get("snapshots") + .and_then(|snapshots| snapshots.as_array()) + .map(|snapshots| { + snapshots + .iter() + .filter_map(|snapshot| { + snapshot + .get("tag") + .and_then(|tag| tag.as_str()) + .map(str::to_owned) + }) + .collect() + }) + .unwrap_or_default()) + } + + /// Run one human-monitor command, treating anything the monitor printed + /// on stderr as a refusal. + /// + /// The human monitor reports a failure in the text it returns rather than + /// in the QMP envelope, so an empty return would turn a refused + /// `loadvm` into a silent success - and a lane that believed it had + /// restored a guest, onto a guest still in whatever state the last check + /// left it in, is the failure this whole layer exists to prevent. + fn human(&mut self, command: &str) -> Result { + let output = self.execute_with( + "human-monitor-command", + json!({ "command-line": command }), + )?; + let text = output.as_str().unwrap_or_default().trim().to_owned(); + if text.is_empty() { + Ok(text) + } else { + Err(HarnessError::Monitor { + command: command.to_owned(), + detail: text, + }) + } + } + + /// A snapshot tag, quoted for the human monitor's own parser. + fn quote(tag: &str) -> String { + format!("'{}'", tag.replace('\'', "'\\''")) + } + #[allow(clippy::disallowed_methods, reason = "synchronous path")] fn read_message(&mut self) -> Result { let mut line = String::new(); From 5430c46290a6053f7bd33d55f7d3e24a407d640f Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sat, 26 Sep 2026 19:30:11 -0700 Subject: [PATCH 07/51] feat(vm): run the lane on restored guests; nine of eleven checks pass The lane boots a pool of guests, snapshots each member against every writable device, and runs each check against a member restored from that snapshot. Per-check selection works, and the nested-guest members are retired rather than restored. Full lane: 768.2s, all four invocation groups admitted against the declared budget with nothing dropped. Nine of the eleven checks pass. Two fail, and both are assertion timeouts inside the guest rather than restore failures - the restore had already completed and the check ran on the restored guest. device-worker-launch never reaches tpm-worker-ready, with Endpoint/tpm-ctrl-tpm0 reporting "phase":"Failed" while the GPU and volume provider are Ready. virtiofsd-volume-runtime never reaches volume-realized. Both ran in a four-group wave and neither was re-run alone, so contention sensitivity is unresolved. R13's "indistinguishable from the current driver" is not claimed: the diagnostics shape is right, but no side-by-side against the nix driver was produced. Snapshots are external qcow2 overlays, not internal. Internal snapshots are unavailable to a guest with a VirtFS export mounted: HMP savevm and QMP snapshot-save are both refused with "Migration is disabled when VirtFS export path ... is mounted in the guest", and blockdev-snapshot-internal-sync is accepted but writes a disk-only snapshot with VM_SIZE 0 B that the emulator then refuses to load. A disk-only restore leaves the guest's page cache, tmpfs and every service's state behind, so the lane resets the guest onto the restored disk; the reset is the price of a member being the fresh boot its check was written against. Six defects, all found by running the lane, each invisible until the one before it was fixed. Block node names exceeded the emulator's 31-byte limit, and one check's name alone exceeds it. target_for resolved only against the launch node, so a device could not be mapped back to its backing file once a layer sat on top. The dirty-layer node was dropped unconditionally after a detach that had already freed it. Re-attach put a device path in the id slot, which is not an id. The unplug wait matched a stale DEVICE_DELETED left in the monitor buffer, so a second restore proceeded with the device still attached. And a node created by blockdev-snapshot-sync is dropped the moment nothing references it, so the restore attached to a collected, read-only node instead of re-opening the layer. The pool does not buy wall clock. The reported 0.3-0.6s restore is host-side block-graph work plus issuing the reset; the guest's reboot onto the restored disk is awaited outside that timed region and costs about what a fresh boot costs. KTD5's speed rationale is not supported by what was measured here. The image action no longer escalates a user the nix daemon already trusts: a Bazel action has no terminal to answer a prompt on, and the first trust probe discarded nix store ping's output because it goes to stderr. --- bazel/checks/vm/defs.bzl | 39 +- .../bazel-owned-guest-external-snapshot.md | 23 + changelog.d/bazel-owned-guest-restore.md | 31 + .../d2b-vm-harness/src/bin/d2b-vm-harness.rs | 129 ++- packages/d2b-vm-harness/src/guest.rs | 673 ++++++++++++++-- packages/d2b-vm-harness/src/legacy.rs | 21 + packages/d2b-vm-harness/src/lib.rs | 6 +- packages/d2b-vm-harness/src/manifest.rs | 69 ++ packages/d2b-vm-harness/src/monitor.rs | 758 +++++++++++++++--- 9 files changed, 1539 insertions(+), 210 deletions(-) create mode 100644 changelog.d/bazel-owned-guest-external-snapshot.md create mode 100644 changelog.d/bazel-owned-guest-restore.md diff --git a/bazel/checks/vm/defs.bzl b/bazel/checks/vm/defs.bzl index 6fda3e9e8..12405ca27 100644 --- a/bazel/checks/vm/defs.bzl +++ b/bazel/checks/vm/defs.bzl @@ -64,13 +64,42 @@ fail() { echo "guest-image $label: $1" >&2 exit 1 } + +# How this action reaches the store, decided once because it decides both the +# flags and whether realizing the closure escalates at all. +# +# A `local` store writes into the store directory itself, so it is only +# available where that directory is writable. A daemon that reports this user +# as trusted builds and adds the paths itself, which is the multi-user answer +# to the same question - and it is the one that must not be paired with a +# `local` store, because that store bypasses the daemon and would then need a +# write permission the user does not have. Only the third case, a store this +# process cannot write and a daemon that does not trust it, escalates. A +# Bazel action has no terminal, so an escalation that cannot proceed fails +# here - naming what is missing - rather than blocking on a password prompt +# that nothing will answer. +store_options="" +escalate=0 +if [ -w "$store_dir" ]; then + store_options="--option store local?store=$store_dir" +elif "$nix_bin" store ping --json 2>/dev/null | grep -q '"trusted":true'; then + # The daemon is the store: no override, and no escalation. The probe reads + # the machine output, which is the one on stdout: the human rendering of + # `store ping` goes to stderr, so a probe that discards stderr would read + # nothing at all and an escalated action would look untrusted. + : +elif [ -n "$sudo_bin" ]; then + store_options="--option store local?store=$store_dir" + escalate=1 +else + fail "the nix store is not writable and the nix daemon does not trust this user, so the guest closure cannot be realized on this host" +fi + nix_run() { - if [ -w "$store_dir" ]; then - "$@" - elif [ -n "$sudo_bin" ]; then + if [ "$escalate" -eq 1 ]; then "$sudo_bin" -E "$@" else - fail "the nix store is not writable and no sudo is available, so the guest closure cannot be realized on this host" + "$@" fi } @@ -139,7 +168,7 @@ system="$("$nix_bin" eval --raw --impure --expr builtins.currentSystem)" || fail expr="(builtins.getFlake \\"path:$root\\").guestImage.\\"$system\\" { rawBundle = \\"$bundle\\"; rawCloudHypervisorController = $controller_argument; extraModules = $check_argument; nodeShape = \\"$node_shape\\"; }" echo "guest-image $label: realizing the guest from declared inputs" >&2 image="$(nix_run "$nix_bin" build \\ - --option store "local?store=$store_dir" \\ + $store_options \\ --option substituters "$substituters" \\ --option build-users-group "" \\ --option sandbox true \\ diff --git a/changelog.d/bazel-owned-guest-external-snapshot.md b/changelog.d/bazel-owned-guest-external-snapshot.md new file mode 100644 index 000000000..4a63a9af6 --- /dev/null +++ b/changelog.d/bazel-owned-guest-external-snapshot.md @@ -0,0 +1,23 @@ +--- +type: fix +--- + +Restore a pool member from an external qcow2 overlay instead of an internal snapshot + +The lane's equivalence gate refused every check at the same line: the +emulator will not take an internal snapshot while a VirtFS export is mounted +in the guest, and every lane guest mounts one, because it is booted from a +host store path and panics at activation without it. The gate now takes an +*external* snapshot - a qcow2 overlay per writable device, taken with +`blockdev-snapshot-sync` - and a restore drops the layer the check dirtied, +takes a fresh one over the frozen image, and restarts the guest on it. + +The restore is disk-only, and the gate says so rather than claiming a +RAM+disk restore it no longer performs: what it proves is that a guest booted +from the snapshot-restored disk matches the fresh boot the check's assertions +were written against. Every writable device is covered, including the state +disk the node attached through its option list, because `/var/lib/d2b` holds +the daemon's store and a snapshot that left it out would hand the second +check on a member the first check's rows. The guest's own drives are attached +as block nodes rather than as drives, which is what lets a restore take the +device off its dirty layer and put it on a clean one. diff --git a/changelog.d/bazel-owned-guest-restore.md b/changelog.d/bazel-owned-guest-restore.md new file mode 100644 index 000000000..e498e3a1b --- /dev/null +++ b/changelog.d/bazel-owned-guest-restore.md @@ -0,0 +1,31 @@ +### Fixed + +- Refused a block node the host-integration lane could not name. Every block + node the lane hands the emulator is now built inside the emulator's own + 31-byte name limit: a check's name alone can run past it + (`runtime-cloud-hypervisor-guest-preflight` does), and the refusal arrived as + a monitor error at the first snapshot - after the guest had booted - rather + than at the launch. The check's readable name is kept in the device id, in + the layer file, and in the lane's report, where the limit does not reach. +- Put a restored device back on a layer the lane owns. A node the emulator + creates for a snapshot is dropped as soon as nothing references it, and a + restore builds its chain with no device on the block graph, so the device was + being re-attached to a node name that no longer existed. The restore now + re-opens the layer the snapshot wrote and attaches the device to that, which + is also what keeps the restored disk writable where the emulator's own + snapshot node inherited the read-only image beneath it. +- Re-attached a restored device under the id its launch declared. The emulator + reports no device id for a node-backed drive, so a restore that read the id + back out of `query-block` was re-attaching the device under its device + *path*, which the emulator refuses as a device model name; the properties + ride as their own arguments for the same reason, with `bootindex` as the + integer the emulator takes rather than the text of one. +- Waited for a device unplug the emulator actually confirmed. The wait matched + any earlier detach's `DEVICE_DELETED` still sitting in the monitor's event + buffer, so a second restore went ahead with the device still attached and + failed on `Node ... is in use` instead of rolling the member back. +- Reported both halves of a restore's cost. The lane reported its own + block-graph work as the restore, which is a fraction of a second, while the + guest's reboot onto the restored disk - the half that decides whether a pool + member is cheaper to reuse than to boot - was folded into that same figure + and never measured. diff --git a/packages/d2b-vm-harness/src/bin/d2b-vm-harness.rs b/packages/d2b-vm-harness/src/bin/d2b-vm-harness.rs index aad8683c4..e45f57088 100644 --- a/packages/d2b-vm-harness/src/bin/d2b-vm-harness.rs +++ b/packages/d2b-vm-harness/src/bin/d2b-vm-harness.rs @@ -25,8 +25,8 @@ use std::{ }; use d2b_vm_harness::{ - Footprint, GuestSpec, HarnessError, HostFacts, LegacyCheck, LegacyGuest, boot, host, - manifest::GuestManifest, report, + ActiveGuest, Footprint, GuestSpec, HarnessError, HostFacts, LegacyCheck, LegacyGuest, + SnapshotPoint, boot, host, manifest::GuestManifest, report, }; use serde_json::json; @@ -108,10 +108,6 @@ const IMAGES: &str = "D2B_VM_HARNESS_IMAGES"; /// carry them: a whitespace- or comma-separated list of check names. const CHECKS: &str = "D2B_VM_CHECK"; -/// The snapshot tag one member is restored from. The emulator's tags live in -/// the guest's own directory, so a member only ever has to name its own. -const SNAPSHOT_TAG: &str = "lane-base"; - /// One check, its own guest, and what that guest costs. #[derive(Clone)] struct LaneGuest { @@ -598,59 +594,96 @@ fn run_check_inner( optional_u64(ACTIVATION_TIMEOUT, 1800)?, ); let mut active = boot(&spec)?; - // The declared pass said this guest can be snapshotted; the running - // guest's own block graph is the authority, and it is asked before the - // snapshot rather than at the first restore. + // The declared pass said this guest's drives can be snapshotted; the + // running guest's own block graph is the authority, and it is asked + // before the snapshot rather than at the first restore. active.require_snapshottable()?; + let mut point = SnapshotPoint::new(&spec)?; + let marker = guest.manifest.activation.marker.clone(); + let activation_bound = spec.activation_timeout; let mut surface = LegacyGuest::attach(&mut active)?; - let marker = surface + let fresh = surface .run(&LegacyCheck::new(format!("{}-marker", guest.name), EQUIVALENCE_MARKER)) .map_err(|error| { HarnessError::Configuration(format!( "the equivalence marker could not be read from the fresh guest: {error}" )) })?; - if !marker.passed { + if !fresh.passed { return Err(HarnessError::Configuration(format!( "the equivalence marker failed against a freshly booted guest, so it cannot say \ anything about a restored one:\n{}", - marker.detail + fresh.detail ))); } - active.save_snapshot(SNAPSHOT_TAG)?; - active.restore(SNAPSHOT_TAG)?; - let restored = surface + // The member's snapshot, taken here: the guest has activated, every + // writable device has been proven restorable, and no check has run. + active.take_snapshot(&mut point)?; + report_line(&format!( + "{}: snapshot taken against every writable device, the guest's own disks underneath", + guest.name + )); + + // The gate. What it proves is narrower than a RAM+disk restore could have + // proved, and it says so: the restore is disk-only, so what is compared + // is a guest *booted from the snapshot-restored disk* against the fresh + // boot the check's assertions were written against - same units, same + // mounts, same host tools, same random pool, reported by the guest + // itself. + let restored = restore_and_measure( + &mut active, + &mut surface, + &mut point, + &guest.name, + &marker, + activation_bound, + )?; + let restored_marker = surface .run(&LegacyCheck::new(format!("{}-marker", guest.name), EQUIVALENCE_MARKER)) .map_err(|error| { HarnessError::Configuration(format!( "the equivalence marker could not be read from the restored guest: {error}" )) })?; - if !restored.passed { + if !restored_marker.passed { return Err(HarnessError::Configuration(format!( - "the equivalence marker failed against a restored guest:\n{}", - restored.detail + "the equivalence marker failed against a guest booted from the restored disk:\n{}", + restored_marker.detail ))); } - let fresh_text = marker_text(&marker.detail); - let restored_text = marker_text(&restored.detail); + let fresh_text = marker_text(&fresh.detail); + let restored_text = marker_text(&restored_marker.detail); if fresh_text != restored_text { return Err(HarnessError::Configuration(format!( - "a restored guest is not the fresh boot its check was written against\n--- fresh boot\n{}\n--- restored\n{}", + "a guest booted from the snapshot-restored disk is not the fresh boot its check was \ + written against\n--- fresh boot\n{}\n--- booted from the restored disk\n{}", fresh_text, restored_text ))); } report_line(&format!( - "{}: the restored guest matches the fresh boot on every marker ({} bytes)", + "{}: a guest booted from the restored disk matches the fresh boot on every marker \ + ({} bytes); the restore itself took {restored:.1}s", guest.name, fresh_text.len() )); - // The check runs on a restored guest: the marker gate above left the - // member where its snapshot was taken, and the gate is not part of what - // the check is being handed. - active.restore(SNAPSHOT_TAG)?; + // The check runs on a guest that was handed back by a restore, never on + // the one the gate read its markers from: the marker runs are themselves + // writes, and a check that inherited them would be a check handed state + // no fresh member would have had. + let second = restore_and_measure( + &mut active, + &mut surface, + &mut point, + &guest.name, + &marker, + activation_bound, + )?; + report_line(&format!( + "{}: second restore onto the same snapshot took {second:.1}s and wrote a layer of its own", + guest.name + )); let script = fs::read_to_string(guest.image_dir.join("check.py")).map_err(|error| { HarnessError::io( format!("reading the assertions of check '{}'", guest.name), @@ -665,14 +698,18 @@ fn run_check_inner( if guest.nested_guest { "ran a nested guest; this member is retired rather than restored" } else { - "restored from the pool's snapshot" + "ran against a guest booted from the member's snapshot" }, )); if guest.nested_guest { // Retired, not restored. A guest with a live guest inside it has no // defined restored state, and a member that has run a nested guest // is torn down here rather than handed to the next check. - let _ = active.discard_snapshot(SNAPSHOT_TAG); + report_line(&format!( + "{}: retired without another restore; every layer it wrote is removed with its \ + working directory", + guest.name + )); } active.shutdown()?; Ok(CheckResult { @@ -684,6 +721,42 @@ fn run_check_inner( }) } +/// Put the member back on its snapshot, wait for the guest to boot from the +/// restored disk, and hand the command channel to the guest's new shell. +/// +/// The console resynchronisation is the part that is easy to get wrong: the +/// channel is one connection for the life of the emulator process, so the +/// bytes the guest wrote while it was shutting down are still in it when the +/// new guest comes up. Reading a command's output from that position decodes +/// whatever the old guest left behind. +/// +/// Two durations are reported, because they are two different costs and only +/// one of them is the pool's: the restore is the lane's own block-graph work +/// on the host, and the wait is the guest rebooting onto the disk it was +/// handed. A pool whose member is restored between checks pays both, so a +/// measurement that stopped at the first would compare a restore against a +/// fresh boot and leave out the boot. +fn restore_and_measure( + active: &mut ActiveGuest, + surface: &mut LegacyGuest, + point: &mut SnapshotPoint, + name: &str, + marker: &str, + bound: Duration, +) -> Result { + let seen = active.activations(marker); + let seconds = active.restore(point)?.as_secs_f64(); + let waiting = Instant::now(); + active.await_reactivation(seen, bound, marker)?; + let reactivation = waiting.elapsed().as_secs_f64(); + surface.resync()?; + report_line(&format!( + "{name}: restored onto the snapshot in {seconds:.1}s, and the guest activated again \ + {reactivation:.1}s after that" + )); + Ok(seconds) +} + /// The marker lines out of a check's own output, with the lane's own log /// lines around them dropped: the surface's log describes what it did, and /// two runs of the same gate legitimately log different durations. diff --git a/packages/d2b-vm-harness/src/guest.rs b/packages/d2b-vm-harness/src/guest.rs index c0157e15b..631de7c93 100644 --- a/packages/d2b-vm-harness/src/guest.rs +++ b/packages/d2b-vm-harness/src/guest.rs @@ -38,10 +38,17 @@ use std::{ use crate::{ error::{HarnessError, Result, UnsnapshottableDevice}, - manifest::GuestManifest, - monitor::Monitor, + manifest::{Drive, GuestManifest}, + monitor::{BlockDevice, Cache, Monitor}, }; +/// How long the guest has to acknowledge that a device was detached. The +/// guest is running when this happens - a paused guest never reads the +/// machine's hotplug registers, so the unplug would never complete - and an +/// unplug it does not complete in seconds is an unplug it is not going to +/// complete at all. +const DETACH_BOUND: Duration = Duration::from_secs(60); + /// The longest a guest's own directory may be while still carrying a Unix /// socket. The limit is 108 bytes; the headroom covers the socket name the /// emulator appends. @@ -244,24 +251,33 @@ impl GuestSpec { push("-rtc"); push("base=utc,clock=vm"); + // The drives the guest node declared are attached as block *nodes* + // rather than as drives. It is the same device to the guest - the + // same model, the same serial, the same boot order, the same file - + // and it is what a restore needs: a drive holds a reference to + // whichever node is on top of it, and the emulator drops an unused + // drive's whole chain, so a restore could not take the device off a + // dirty layer and put it on a clean one. A node holds nothing but + // the device, so dropping the device drops exactly the layer. for (index, drive) in manifest.drives.iter().enumerate() { let file = self.drive_file(drive.file.as_str(), &work_dir); - let drive_id = format!("lane_drive_{index}"); - let mut drive_options = vec![ - ("index".to_owned(), index.to_string()), - ("id".to_owned(), drive_id.clone()), - ("if".to_owned(), "none".to_owned()), - ("file".to_owned(), file), - ]; - if let Some(format) = &drive.format { - drive_options.push(("format".to_owned(), format.clone())); - } - drive_options.push(("cache".to_owned(), drive.cache.clone())); - drive_options.push(("werror".to_owned(), drive.werror.clone())); - push("-drive"); - push(&render_options(&drive_options)); - - let mut device_options = vec![("drive".to_owned(), drive_id)]; + // The node name and the device id are not the same budget: the + // emulator caps a block node name at `NODE_NAME_LIMIT` and a + // device id at 127 characters, so the node is shortened and the + // id - which is what a reader sees in `query-block` and what + // `device_del` names - keeps the check's own name. + let node = declared_node(&self.name, index); + let device_id = declared_device_id(&self.name, index); + push("-blockdev"); + push(&render_blockdev( + &node, + &drive_format(manifest, drive), + &file, + &drive.cache, + &drive.werror, + )?); + + let mut device_options = vec![("drive".to_owned(), node.clone()), ("id".to_owned(), device_id.clone())]; if let Some(boot_index) = &drive.boot_index { device_options.push(("bootindex".to_owned(), boot_index.clone())); } @@ -421,6 +437,125 @@ fn render_options(options: &[(String, String)]) -> String { .join(",") } +/// The image format one declared drive is opened as. +/// +/// A drive the guest node attached to a store path declares its own format; +/// the guest's own root disk is the image the guest-image action produced, +/// and the format is the image's. +fn drive_format(manifest: &GuestManifest, drive: &Drive) -> String { + match &drive.format { + Some(format) => format.clone(), + None if Path::new(&drive.file).is_absolute() => "raw".to_owned(), + None => manifest.image.disk_format.clone(), + } +} + +/// The longest block node name the emulator accepts, in bytes. +/// +/// The pinned emulator refuses a longer one at the point the node is created, +/// with `Node name too long`, which for a member the lane has already booted +/// is the snapshot it was about to take. A check's own name can exceed this by +/// itself - `runtime-cloud-hypervisor-guest-preflight` is longer than the +/// whole budget - so every name the lane gives the emulator is built to fit. +const NODE_NAME_LIMIT: usize = 31; + +/// The prefix a member's block node names carry. +/// +/// The emulator's budget does not hold a check's name plus anything else, so +/// what the emulator sees is a prefix of it. The check's whole name is still +/// what the lane reports, what the member's working directory is called, and +/// what the member's device ids carry - the names a reader looks at. +fn member_token(member: &str) -> String { + member + .chars() + .map(|character| { + if character.is_ascii_alphanumeric() || character == '-' { + character + } else { + '-' + } + }) + .take(12) + .collect() +} + +/// A block node name, held to the emulator's own limit. +/// +/// Every name the lane hands the emulator goes through here rather than +/// through a comment: a name that outgrows the budget is a member that boots +/// and then fails at its first snapshot, which is the failure this is cheap +/// to prevent and expensive to read. +fn bounded_node_name(name: String) -> String { + debug_assert!( + name.len() <= NODE_NAME_LIMIT, + "the block node name {name:?} is {} bytes, past the emulator's {NODE_NAME_LIMIT}", + name.len() + ); + name +} + +/// The block node a member's own declared drive is attached as. +/// +/// The launch declares this node and the snapshot finds it again by this +/// name, so the two share one spelling rather than each building it. +fn declared_node(member: &str, index: usize) -> String { + bounded_node_name(format!("{}-d{index}", member_token(member))) +} + +/// The device id a member's own declared drive is attached with. +/// +/// Shared for the same reason as the node name, and for one more: a device +/// the launch attached with a node-backed drive is reported by `query-block` +/// with no id at all - the emulator fills that field for a `-drive` and not +/// for a `-blockdev` - so a restore that asked the report what to call the +/// device again would re-attach it under a device *path*, which is not a +/// name the emulator accepts. The id is the launch's own, and it is what the +/// restore attaches the device back with. +fn declared_device_id(member: &str, index: usize) -> String { + format!("{member}-device-{index}") +} + +/// One block node, as the launch declares it. +/// +/// The spelling is the emulator's own `-blockdev` JSON, because the restore +/// has to name the same options when it re-opens the frozen image: a node +/// re-opened with different caching is a different device to the guest than +/// the one it stands in for. +/// +/// The caching is spelled as the cache object's own keys rather than as the +/// `-drive` mode name, because the blockdev cache object has no `writeback` +/// key and the pinned emulator refuses the whole command line over one - +/// `Parameter 'cache.writeback' is unexpected`. The four `-drive` modes map +/// onto the two keys it does accept: `no-flush` for the two that ignore +/// flushes, `direct` for the one that bypasses the host's write cache. +/// +/// `werror` is not spelled at all, because `-blockdev` has no such option: +/// it belongs to the legacy `-drive` form, and the pinned emulator refuses a +/// node carrying one at the top level, on the file child, and in the +/// option-string form alike (`Parameter 'werror' is unexpected`). A block +/// node's I/O error action is the `-drive` default, `report`, so a node that +/// declared `report` - which is every node in the lane - is already the +/// device it asked for and needs nothing on the command line. A node that +/// declared anything else cannot have it, and running it as `report` anyway +/// would hand the guest a different device than its configuration +/// described, so it is refused here rather than dropped. +fn render_blockdev(node: &str, format: &str, file: &str, cache: &str, werror: &str) -> Result { + if werror != "report" { + return Err(HarnessError::Configuration(format!( + "the guest declares the drive error action werror={werror:?}, which a block node \ + cannot carry: -blockdev has no werror, and a block node's I/O error action is always \ + the report default" + ))); + } + Ok(serde_json::json!({ + "node-name": node, + "driver": format, + "file": { "driver": "file", "filename": file }, + "cache": Cache::parse(cache)?.as_options(), + }) + .to_string()) +} + /// A host directory the guest shares over 9p, with the VM module's /// `TMPDIR`-relative sources resolved against the working directory the lane /// owns. @@ -486,6 +621,235 @@ pub struct ActiveGuest { shut_down: bool, } +/// What a restore puts back: the frozen image, and the device that has to be +/// attached to a fresh layer over it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RestoreTarget { + /// The image the member's snapshot froze. + pub file: PathBuf, + /// The format that image is in. + pub format: String, + /// The caching mode it was written with. + pub cache: Cache, + /// The model the guest's configuration attached it as. + pub model: String, + /// The id the device is attached with, which a restore attaches it back + /// under. + /// + /// The emulator does not report one for a node-backed drive, so it cannot + /// be read back out of `query-block`: it is the launch's own id, carried + /// here, and a device the node attached through its own option list - + /// which the launch gave no id at all - is given one of the member's. + pub device_id: String, + /// The properties that device was declared with. + pub properties: Vec<(String, String)>, + /// A short, stable label for this device within its member. + /// + /// The node names a restore builds are named after this rather than after + /// the device's file, because a device's file name does not fit the + /// emulator's block-node limit: the state disk is a store path whose + /// basename alone can run past it. Two devices of one member differ in + /// their label, and a device keeps its label for the member's whole life. + pub label: String, +} + +/// A member's snapshot, and the state a restore of it is assembled from. +/// +/// One member has one snapshot for its whole run, and every device it +/// attached is in it. The devices come from two places and are matched +/// differently, because that is how the guest's own configuration declares +/// them: a drive the node declared is attached as a named block node, and a +/// drive it attached through its option list rides an overlay the emulator +/// created over a store path, which the lane finds by the store image +/// underneath it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SnapshotPoint { + /// Where this member's overlays live. + work_dir: PathBuf, + /// The name overlays and their frozen images are named under, which has + /// to be unique across the members a run holds at once. + member: String, + /// Declared drives, by the node the launch attached them as. + declared: Vec<(String, RestoreTarget)>, + /// Ephemeral drives, by the store image the emulator put an overlay + /// over. + ephemeral: Vec<(String, RestoreTarget)>, + /// How many layers this member has taken. Every restore's layer is a + /// file of its own, so a restore cannot inherit the writes of the + /// restore before it. + generation: u64, +} + +impl SnapshotPoint { + /// Read a member's snapshot out of the guest configuration it will boot. + pub fn new(spec: &GuestSpec) -> Result { + let work_dir = spec.work_dir(); + let declared = spec + .manifest + .drives + .iter() + .enumerate() + .map(|(index, drive)| -> Result<(String, RestoreTarget)> { + let mut properties = Vec::new(); + if let Some(boot_index) = &drive.boot_index { + properties.push(("bootindex".to_owned(), boot_index.clone())); + } + if let Some(serial) = &drive.serial { + properties.push(("serial".to_owned(), serial.clone())); + } + Ok(( + declared_node(&spec.name, index), + RestoreTarget { + file: PathBuf::from(spec.drive_file(&drive.file, &work_dir)), + format: drive_format(&spec.manifest, drive), + cache: Cache::parse(&drive.cache)?, + model: device_model(&drive.interface).to_owned(), + device_id: declared_device_id(&spec.name, index), + properties, + label: format!("d{index}"), + }, + )) + }) + .collect::>>()?; + let ephemeral = spec + .manifest + .ephemeral_drives() + .into_iter() + .enumerate() + .map(|(index, drive)| -> Result<(String, RestoreTarget)> { + Ok(( + drive.file.clone(), + RestoreTarget { + file: PathBuf::from(&drive.file), + format: drive.format, + cache: Cache::parse(&drive.cache)?, + // The VM module attaches an ephemeral drive with + // `if=virtio` and nothing else, which is a virtio-blk + // device with no serial and no boot order. + model: device_model(&drive.interface).to_owned(), + device_id: format!("{}-e{index}-device", spec.name), + properties: Vec::new(), + label: format!("e{index}"), + }, + )) + }) + .collect::>>()?; + Ok(Self { + work_dir, + member: spec.name.clone(), + declared, + ephemeral, + generation: 0, + }) + } + + /// The image one device's snapshot is frozen in. + /// + /// A device the launch attached as a named node is that node's. A device + /// riding an overlay the emulator created is the store image underneath + /// it, read off the block graph rather than guessed at: the emulator + /// names the overlay itself, and that name changes every time the + /// emulator starts. + fn target_for(&self, monitor: &mut Monitor, device: &BlockDevice) -> Result<&RestoreTarget> { + // The node the launch declared, for a device no snapshot has touched + // yet. + if let Some((_, target)) = self.declared.iter().find(|(node, _)| *node == device.node) { + return Ok(target); + } + // A layer this member has taken since: a snapshot puts the lane's own + // overlay on top of the node the launch declared, and every restore + // puts a fresh one there, so after the first snapshot the device is + // named by a layer the lane built rather than by the launch's name. + // The layer is named off the target's label, so it is read back the + // same way. + let token = member_token(&self.member); + let layered = self.declared.iter().find(|(_, target)| { + (0..=self.generation).any(|generation| { + format!("{token}.{}.l{generation}", target.label) == device.node + }) + }); + if let Some((_, target)) = layered { + return Ok(target); + } + // A device riding an overlay the emulator created - the state disk a + // node attaches through its own option list - is found by the image + // underneath instead, which is the file the guest's configuration + // declared: the emulator names that overlay itself, and the name + // changes every time the emulator starts. The base file is read the + // same way for a device the launch declared, because it is what both + // kinds of device have in common whatever is on top of them. + let backing = monitor.backing_files(&device.node)?; + let matched = self + .declared + .iter() + .chain(self.ephemeral.iter()) + .find(|(_, target)| { + let file = target.file.to_string_lossy(); + backing.iter().any(|backing| backing == file.as_ref()) + }); + if let Some((_, target)) = matched { + return Ok(target); + } + Err(HarnessError::Configuration(format!( + "the guest attached the writable device {} on {}, which the member's snapshot does \ + not account for, so a restore could not put it back", + device.describe(), + device.file + ))) + } + + /// The layer one device's next writes go into. + pub fn overlay_for(&self, target: &RestoreTarget) -> PathBuf { + self.work_dir.join(format!( + "{}.layer-{}.qcow2", + Self::stem(&target.file), + self.generation + )) + } + + /// The node name one device's next layer is opened as. + pub fn overlay_node(&self, target: &RestoreTarget) -> String { + bounded_node_name(format!( + "{}.{}.l{}", + member_token(&self.member), + target.label, + self.generation + )) + } + + /// The node name one device's frozen image is re-opened as. + pub fn frozen_node(&self, target: &RestoreTarget) -> String { + bounded_node_name(format!( + "{}.{}.f{}", + member_token(&self.member), + target.label, + self.generation + )) + } + + /// Start the next layer. + pub fn next_layer(&mut self) { + self.generation += 1; + } + + /// A short, unique-enough name for one image's layers. + fn stem(file: &Path) -> String { + let name = file + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_default(); + name.chars() + .map(|character| { + if character.is_ascii_alphanumeric() || character == '-' { + character + } else { + '-' + } + }) + .collect() + } +} + impl ActiveGuest { /// The guest's monitor, for the pool's snapshot and restore work. pub fn monitor(&mut self) -> Option<&mut Monitor> { @@ -522,7 +886,7 @@ impl ActiveGuest { } /// Refuse a guest whose attached writable devices cannot carry an - /// internal snapshot. + /// external snapshot. /// /// This runs at boot, before any check does, because the alternative is /// discovering it at the first restore - after a suite has already run @@ -533,58 +897,173 @@ impl ActiveGuest { .as_mut() .ok_or_else(|| HarnessError::Configuration("the guest has no monitor".to_owned()))? .block_devices()?; - let unsnapshottable: Vec = devices + let refused: Vec = devices .iter() - .filter(|device| !device.snapshottable) + .filter(|device| !device.rotatable) .map(|device| UnsnapshottableDevice { - device: device.id.clone(), + device: device.describe(), file: device.file.clone(), format: device.format.clone(), }) .collect(); - if unsnapshottable.is_empty() { + if refused.is_empty() { Ok(()) } else { - Err(HarnessError::NotSnapshottable { - devices: unsnapshottable, - }) + Err(HarnessError::NotSnapshottable { devices: refused }) + } + } + + /// Take the member's snapshot: an external overlay per writable device, + /// with the state the guest has reached frozen underneath it. + /// + /// This is the point the member snapshots at: after activation has + /// completed, after every attached writable device has been proven + /// restorable, and before any check has touched the guest. A snapshot + /// taken later would be a snapshot of whatever the last check left + /// behind. + /// + /// Every writable device the guest attached is covered, not only the + /// ones the guest node declared. The state disk the node attached + /// through its option list rides an overlay the emulator put over a raw + /// store path, and `/var/lib/d2b` lives on it: a snapshot that left it + /// out would hand the second check on this member the first check's + /// daemon store, which is exactly the state a member's snapshot exists + /// to prevent. + pub fn take_snapshot(&mut self, point: &mut SnapshotPoint) -> Result<()> { + for device in self.rotatable_devices()? { + let target = point.target_for(self.monitor_mut()?, &device)?.clone(); + self.monitor_mut()?.take_overlay( + &device, + &point.overlay_for(&target), + &point.overlay_node(&target), + )?; } + Ok(()) } - /// Take the pool's snapshot of this guest, under a tag of the lane's - /// choosing. + /// Return every writable device to the member's snapshot and restart the + /// guest on it. + /// + /// The restore is disk-only, and deliberately so. An internal snapshot + /// would roll the guest's memory back with its disk, but the emulator + /// refuses to take one while a VirtFS export is mounted in the guest - + /// and every lane guest mounts one, because it is booted from a host + /// store path and panics at activation without it. So what is restored is + /// the disk, and the guest is *restarted onto it*, which is what makes + /// the restored disk the state the guest is actually running on: a reset + /// discards the page cache, the mounted filesystems and every service's + /// state, all of which belong to the check that ran before. /// - /// This is the point the pool snapshots at: after activation has - /// completed, after every attached writable device has been proven to - /// carry a snapshot, and before any check has touched the guest. A - /// snapshot taken later would be a snapshot of whatever the last check - /// left behind. - pub fn save_snapshot(&mut self, tag: &str) -> Result<()> { - self.monitor_mut()?.save_snapshot(tag) + /// Each device is taken off its dirty layer, the layer is dropped, a + /// fresh one is taken over the frozen image, and the device is put back on + /// it. The sequence is per device and explicit because the emulator offers + /// no single command for it: a block node cannot be re-pointed while a + /// device is attached to it, and a device can only be attached to a node + /// by name - which is why the launch declares the guest's own drives as + /// nodes rather than as drives. + /// + /// The second restore of a member is the one that has to be right: it + /// takes its fresh layer over the same frozen image rather than over + /// whatever the first restore left behind, so a check never sees the check + /// before it. That is what the layer generation is for - every restore + /// writes a layer and a node name of its own, because the emulator refuses + /// to open a node twice or to write over a file that holds an open + /// image. + pub fn restore(&mut self, point: &mut SnapshotPoint) -> Result { + let started = Instant::now(); + let devices = self.rotatable_devices()?; + let mut targets = Vec::with_capacity(devices.len()); + for device in &devices { + let target = point.target_for(self.monitor_mut()?, device)?.clone(); + self.monitor_mut()?.detach_device(device, DETACH_BOUND)?; + targets.push((device.clone(), target)); + } + point.next_layer(); + for (device, _) in &targets { + // The layer the check that ran on this member wrote into, freed + // before a fresh one is taken over the frozen image. It is + // usually gone already: the detach above took the last device off + // it, and the emulator drops an unused node's chain. + self.monitor_mut()?.drop_node_if_present(&device.node)?; + } + for (device, target) in &targets { + self.monitor_mut()?.add_frozen_image( + &point.frozen_node(target), + &target.file, + &target.format, + target.cache, + )?; + let frozen = BlockDevice { + node: point.frozen_node(target), + ..device.clone() + }; + self.monitor_mut()?.take_overlay( + &frozen, + &point.overlay_for(target), + &point.overlay_node(target), + )?; + } + for (_, target) in &targets { + self.monitor_mut()?.add_overlay_node( + &point.overlay_node(target), + &point.overlay_for(target), + target.cache, + )?; + self.monitor_mut()?.attach_device( + &target.model, + &point.overlay_node(target), + &target.device_id, + &target.properties, + )?; + } + self.monitor_mut()?.reset_machine()?; + Ok(started.elapsed()) } - /// Restore this guest from a snapshot it took itself. + /// Wait for the guest to report activation again, after a restore reset + /// it. /// - /// The guest's command channel survives the restore. An internal - /// snapshot captures the guest's memory and its devices, not the host - /// socket at the far end of the guest's console, and the emulator is the - /// same process across the restore - so the connection the launcher - /// accepted before the boot is still the connection the guest's root - /// shell is reading from afterwards. That is also why the snapshot is - /// taken with the channel idle: bytes already in the console are not - /// part of what a restore rolls back. - pub fn restore(&mut self, tag: &str) -> Result<()> { - self.monitor_mut()?.load_snapshot(tag) + /// The wait counts markers rather than looking for one: the console is + /// a file the emulator appends to for the life of the process, so the + /// marker the first boot wrote is still in it, and a wait that looked + /// for a marker would be satisfied by a guest that had not re-activated + /// at all. + pub fn await_reactivation(&mut self, seen: usize, bound: Duration, marker: &str) -> Result<()> { + let console = self.work_dir.join("console.log"); + let deadline = Instant::now() + bound; + loop { + if read_console(&console).matches(marker).count() > seen { + return Ok(()); + } + if Instant::now() >= deadline { + return Err(HarnessError::NotActivated { + bound, + marker: marker.to_owned(), + console_tail: activation_failure_tail(&read_console(&console)), + }); + } + sleep(CONSOLE_POLL); + } } - /// Whether this guest currently holds a snapshot under a tag. - pub fn holds_snapshot(&mut self, tag: &str) -> Result { - Ok(self.monitor_mut()?.snapshot_tags()?.iter().any(|held| held == tag)) + /// How many times the guest has reported activation on its console. + pub fn activations(&self, marker: &str) -> usize { + read_console(&self.work_dir.join("console.log")) + .matches(marker) + .count() } - /// Drop a snapshot, which is what retiring a member frees. - pub fn discard_snapshot(&mut self, tag: &str) -> Result<()> { - self.monitor_mut()?.delete_snapshot(tag) + /// The guest's writable devices, as the emulator's own block graph + /// reports them. + fn rotatable_devices(&mut self) -> Result> { + Ok(self + .monitor + .as_mut() + .ok_or_else(|| HarnessError::Configuration("the guest has no monitor".to_owned()))? + .block_devices()? + .into_iter() + .filter(|device| device.rotatable) + .collect()) } fn monitor_mut(&mut self) -> Result<&mut Monitor> { @@ -1081,12 +1560,29 @@ mod tests { let argv = argv(&bigger); assert_eq!(value_of(&argv, "-m"), "16384"); assert_eq!(value_of(&argv, "-smp"), "8"); - let drive = value_of(&argv, "-drive"); + let node = value_of(&argv, "-blockdev"); assert!( - drive.contains("file=/run/lane/work/lane-member-0/disk.qcow2"), - "the root drive is the guest's own writable copy: {drive}" + node.contains(r#""filename":"/run/lane/work/lane-member-0/disk.qcow2""#), + "the root drive is the guest's own writable copy: {node}" + ); + assert!( + !node.contains("\"format\""), + "the node pinned no format, so the image's own is used: {node}" + ); + // The node is bound to the device by name rather than through a + // drive, which is what lets a restore take the device off its dirty + // layer and put it on a clean one. The name is the one the restore + // finds the node by again, so the two are asserted against each other + // rather than against a spelling either of them could drift from. + let bound = value_of(&argv, "-blockdev"); + let node: serde_json::Value = serde_json::from_str(bound).expect("a block node is JSON"); + let name = node["node-name"].as_str().expect("a block node is named"); + assert!( + argv.iter() + .any(|argument| argument.contains("virtio-blk-pci,") + && argument.contains(&format!("drive={name}"))), + "the device is attached to the node {name:?}: {argv:?}" ); - assert!(!drive.contains("format="), "the node pinned no format: {drive}"); } #[test] @@ -1152,7 +1648,7 @@ mod tests { .unwrap_or_else(|| panic!("{needle} is on the command line: {argv:?}")) }; assert!(position("virtio-rng-pci") < position("-net")); - assert!(position("-net") < position("-drive")); + assert!(position("-net") < position("-blockdev")); } #[test] @@ -1193,6 +1689,60 @@ mod tests { .any(|pair| pair == ["-device", "vhost-vsock-pci,guest-cid=3"])); } + #[test] + fn every_block_node_name_the_emulator_is_handed_fits_its_limit() { + // The longest name the lane runs under and the longest device file it + // attaches: `runtime-cloud-hypervisor-guest-preflight` is longer than + // the emulator's whole budget for a block node by itself, and the + // state disk is a store path whose basename is longer again. Both are + // named on the launch and named again on every restore, and the + // emulator refuses a longer one with `Node name too long` at the + // point the node is created - which is a member that booted and then + // failed at its first snapshot. + let mut declared = manifest(3, 3072, &[]); + declared.extra_options = vec![ + "-drive".to_owned(), + "file=/nix/store/0123456789bcdefghijklmnopqrstuv-d2b-state.img,format=raw,if=virtio,snapshot=on" + .to_owned(), + ]; + let spec = GuestSpec::new( + declared, + "/run/lane/image", + "/nix/store/qemu/bin/qemu-kvm", + "/run/lane/work", + "runtime-cloud-hypervisor-guest-preflight", + ); + + for pair in argv(&spec).windows(2).filter(|pair| pair[0] == "-blockdev") { + let node: serde_json::Value = + serde_json::from_str(&pair[1]).expect("a block node is JSON"); + let name = node["node-name"].as_str().expect("a block node is named"); + assert!(name.len() <= NODE_NAME_LIMIT, "the launch declares {name:?}"); + } + + let point = SnapshotPoint::new(&spec).expect("the member's snapshot reads out of its spec"); + for (node, _) in &point.declared { + assert!(node.len() <= NODE_NAME_LIMIT, "the launch declares {node:?}"); + } + let targets: Vec<&RestoreTarget> = point + .declared + .iter() + .chain(point.ephemeral.iter()) + .map(|(_, target)| target) + .collect(); + assert_eq!(targets.len(), 2, "the root disk and the state disk"); + for target in &targets { + for name in [point.overlay_node(target), point.frozen_node(target)] { + assert!(name.len() <= NODE_NAME_LIMIT, "a restore declares {name:?}"); + } + } + assert_ne!( + point.overlay_node(targets[0]), + point.overlay_node(targets[1]), + "two devices of one member are two nodes" + ); + } + #[test] fn the_accelerator_is_selected_and_never_falls_back_to_emulation() { let argv = argv(&spec(manifest(3, 3072, &[]))); @@ -1238,8 +1788,13 @@ mod tests { let argv = argv(&spec(declared)); assert!(!argv.iter().any(|argument| argument == "-kernel")); assert!(!argv.iter().any(|argument| argument == "-append")); - let drive = value_of(&argv, "-drive"); - assert!(drive.contains("cache=unsafe"), "{drive}"); + let node = value_of(&argv, "-blockdev"); + assert!(node.contains("\"no-flush\":true"), "cache=unsafe: {node}"); + assert!(node.contains("\"direct\":false"), "cache=unsafe: {node}"); + // `-blockdev` has no `werror` and refuses the whole command line + // over one, so a node that declared the report default carries + // nothing for it. See `render_blockdev`. + assert!(!node.contains("werror"), "{node}"); let device = argv .iter() .find(|argument| argument.starts_with("virtio-blk-pci,")) diff --git a/packages/d2b-vm-harness/src/legacy.rs b/packages/d2b-vm-harness/src/legacy.rs index 62cce7b84..a010dc080 100644 --- a/packages/d2b-vm-harness/src/legacy.rs +++ b/packages/d2b-vm-harness/src/legacy.rs @@ -370,6 +370,13 @@ impl Console { ))) } + /// Wait for the guest's shell to announce itself again, discarding + /// whatever the console is holding. + #[allow(clippy::disallowed_methods, reason = "synchronous path")] + fn resync(&mut self, bound: Duration) -> Result<()> { + self.await_shell(bound) + } + /// Run one command in the guest and read back its status and output. /// /// The wire form is the driver's, unchanged: the command is run under @@ -900,6 +907,20 @@ impl LegacyGuest { }) } + /// Take the console back after the guest was restarted onto a restored + /// disk. + /// + /// The channel is one connection for the life of the emulator process, so + /// the bytes the previous guest wrote as it shut down are still in it when + /// the new guest's shell greets. Reading a command's output from that + /// position decodes the old guest's leftovers as the new one's answer - + /// which reads as a command that returned nonsense, not as a console that + /// needed resynchronising. Waiting for the greeting again is what puts + /// the reader back on a command boundary. + pub fn resync(&mut self) -> Result<()> { + self.control.console.resync(SHELL_GREETING_TIMEOUT) + } + /// The working directory this guest's check scripts are written to. pub fn work_dir(&self) -> &Path { &self.work_dir diff --git a/packages/d2b-vm-harness/src/lib.rs b/packages/d2b-vm-harness/src/lib.rs index 424bbdc95..4bc1518aa 100644 --- a/packages/d2b-vm-harness/src/lib.rs +++ b/packages/d2b-vm-harness/src/lib.rs @@ -23,8 +23,8 @@ pub mod monitor; pub mod legacy; pub use error::{HarnessError, Result, UnsnapshottableDevice}; -pub use guest::{ActiveGuest, GuestSpec, boot, report, reserve_loopback_port}; +pub use guest::{ActiveGuest, GuestSpec, RestoreTarget, SnapshotPoint, boot, report, reserve_loopback_port}; pub use legacy::{LegacyCheck, LegacyError, LegacyGuest, LegacyOutcome}; pub use host::{Capability, HostFacts, require_this_host}; -pub use manifest::{CheckRecord, Footprint, GuestManifest, PoolBudget}; -pub use monitor::{BlockDevice, Monitor}; +pub use manifest::{CheckRecord, EphemeralDrive, Footprint, GuestManifest, PoolBudget}; +pub use monitor::{BlockDevice, Cache, Monitor}; diff --git a/packages/d2b-vm-harness/src/manifest.rs b/packages/d2b-vm-harness/src/manifest.rs index b5f7a7220..fc443e89b 100644 --- a/packages/d2b-vm-harness/src/manifest.rs +++ b/packages/d2b-vm-harness/src/manifest.rs @@ -191,6 +191,28 @@ pub struct Drive { pub interface: String, } +/// A drive the guest node attached through its option list and asked the +/// emulator to shadow with a throwaway overlay. +/// +/// The node declares one of these as a rendered `-drive` with `snapshot=on` +/// rather than as a drive of its own, so it arrives as text rather than as +/// structure. What the lane needs from it is the store image the overlay +/// sits on, because that image is what a member's snapshot restores the +/// device to: the overlay the emulator created is named after the emulator's +/// own temporary directory and is gone on the next run, while the store +/// image behind it is the same for every member and every check. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EphemeralDrive { + /// The store image the overlay sits on. + pub file: String, + /// The format that image is in. + pub format: String, + /// The cache mode the node declared for the drive. + pub cache: String, + /// The bus the device is attached to. + pub interface: String, +} + /// One host directory the guest mounts over 9p. #[derive(Debug, Clone, PartialEq, Eq, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] @@ -365,6 +387,53 @@ impl GuestManifest { } refused } + + /// The drives the node attached through its option list rather than + /// through its drive list, as drives the lane can reason about. + /// + /// The state disk the re-homed node mounts at `/var/lib/d2b` is declared + /// exactly this way: a `-drive` rendered by the VM module with + /// `snapshot=on`, which asks the emulator for a throwaway overlay over + /// the store image. That overlay is a real device with a real node, and a + /// member's snapshot has to cover it: `/var/lib/d2b` holds the daemon's + /// store, so a snapshot that left it out would hand the second check on a + /// member the first check's rows. + pub fn ephemeral_drives(&self) -> Vec { + let mut drives = Vec::new(); + for (index, option) in self.extra_options.iter().enumerate() { + if !option.starts_with("file=") + || !self + .extra_options + .get(index.wrapping_sub(1)) + .is_some_and(|previous| previous == "-drive") + { + continue; + } + if !option.contains("snapshot=on") { + continue; + } + let read = |key: &str| { + option + .split(',') + .find_map(|field| field.strip_prefix(&format!("{key}="))) + .map(str::to_owned) + }; + let (Some(file), Some(format)) = (read("file"), read("format")) else { + continue; + }; + drives.push(EphemeralDrive { + file, + format, + cache: read("cache").unwrap_or_else(|| "writeback".to_owned()), + interface: match read("if").as_deref() { + Some("scsi") => "scsi".to_owned(), + Some("ide") => "ide".to_owned(), + _ => "virtio".to_owned(), + }, + }); + } + drives + } } #[cfg(test)] diff --git a/packages/d2b-vm-harness/src/monitor.rs b/packages/d2b-vm-harness/src/monitor.rs index bc48b2502..94364b748 100644 --- a/packages/d2b-vm-harness/src/monitor.rs +++ b/packages/d2b-vm-harness/src/monitor.rs @@ -1,16 +1,26 @@ //! The emulator's control monitor. //! //! The lane needs three things from the emulator that the console cannot -//! give it: to read the block graph the guest attached, to read the guest's -//! own snapshot list, and to ask the emulator to stop. All three are QMP -//! commands, spoken over the unix socket the launcher creates - the same -//! `qmp-socket` readiness the repository's own service-capability table -//! names. +//! give it: to read the block graph the guest attached, to move the guest's +//! writable disks between an external snapshot and a scratch layer, and to +//! ask the emulator to stop. All three are QMP commands, spoken over the unix +//! socket the launcher creates - the same `qmp-socket` readiness the +//! repository's own service-capability table names. +//! +//! The snapshot the pool takes is an *external* one: a fresh qcow2 overlay +//! taken with `blockdev-snapshot-sync` against the node the device is +//! writing, which leaves the previous node read-only and makes the overlay +//! the device's new top. An internal snapshot (`savevm`, or the QMP +//! `snapshot-save` behind it) is not available to this lane: the emulator +//! refuses to save a guest whose VirtFS export is mounted in the guest, and +//! every lane guest mounts one, because the guest is booted from a host store +//! path and panics at activation without it. use std::{ io::{BufRead, BufReader, Write}, os::unix::net::UnixStream, path::Path, + time::{Duration, Instant}, }; use serde_json::{Value, json}; @@ -27,14 +37,83 @@ pub struct Monitor { /// One writable block device the guest attached, as the emulator reports it. #[derive(Debug, Clone, PartialEq, Eq)] pub struct BlockDevice { - /// The device's id on the monitor. + /// The device's id on the monitor, or the path it is named by instead. pub id: String, + /// The device's own path in the machine's device tree, which is what + /// detaching and re-attaching it takes. + pub qdev: String, + /// The block node the device is reading and writing. + pub node: String, /// The backing file the emulator resolved. pub file: String, /// The image format that file is in. pub format: String, - /// Whether the emulator can store a snapshot inside that file. - pub snapshottable: bool, + /// Whether the node is open read-only. + pub read_only: bool, + /// Whether a restore can be made of this device. + pub rotatable: bool, +} + +impl BlockDevice { + /// The device as the emulator's own diagnostics name it. + pub fn describe(&self) -> String { + format!("{} ({} on {})", self.id, self.format, self.qdev) + } +} +/// How far down a node's backing chain the lane walks. A guest's chain is one +/// or two layers deep - an overlay over a store path, or an overlay over an +/// overlay - and the bound is what keeps a cycle the emulator should not +/// produce from becoming a loop. +const BACKING_DEPTH: usize = 8; + +/// How often the monitor polls while it waits for an event the emulator +/// delivers asynchronously. +const POLL_INTERVAL: Duration = Duration::from_millis(100); + +/// A block node's caching mode, as the guest's configuration declared it and +/// as a restored node has to be opened with again. +/// +/// A restore re-opens the frozen image as a node, and a node opened with a +/// different caching mode than it was written with is a different device as +/// far as the guest is concerned. The lane's guests declare `writeback` on the +/// disks they boot from and `unsafe` on the ephemeral ones, and both have to +/// survive a restore unchanged. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Cache { + /// The host's write cache, flushes honored. + Writeback, + /// The host's write cache, flushes ignored. + Unsafe, + /// No host write cache, flushes honored. + None, + /// The host's write cache, every write also written through. + Writethrough, +} + +impl Cache { + /// The mode a guest configuration's own spelling names. + pub fn parse(declared: &str) -> Result { + match declared { + "writeback" => Ok(Self::Writeback), + "unsafe" => Ok(Self::Unsafe), + "none" => Ok(Self::None), + "writethrough" => Ok(Self::Writethrough), + other => Err(HarnessError::Configuration(format!( + "the guest declares the unknown drive cache mode {other:?}" + ))), + } + } + + /// The options a block node carrying this mode is opened with. + pub fn as_options(self) -> Value { + let (direct, no_flush) = match self { + Self::Writeback => (false, false), + Self::Unsafe => (false, true), + Self::None => (true, false), + Self::Writethrough => (false, true), + }; + json!({ "direct": direct, "no-flush": no_flush }) + } } impl Monitor { @@ -63,6 +142,51 @@ impl Monitor { monitor.execute("qmp_capabilities")?; Ok(monitor) } + /// The files underneath one node, nearest first. + /// + /// A restore has to recognise a device the emulator named for itself - + /// the throwaway overlay it puts over an ephemeral drive - by the store + /// image that overlay is on, because the overlay's own name changes every + /// time the emulator starts and the store image is what the guest's + /// configuration declared. + pub fn backing_files(&mut self, node: &str) -> Result> { + let report = self.execute("query-named-block-nodes")?; + let nodes: Vec<&Value> = report + .as_array() + .into_iter() + .flatten() + .collect(); + let mut files = Vec::new(); + let mut current = Some(node.to_owned()); + let mut depth = 0; + while let Some(name) = current { + let Some(entry) = nodes + .iter() + .find(|entry| entry.get("node-name").and_then(Value::as_str) == Some(name.as_str())) + else { + break; + }; + if let Some(file) = entry.get("file").and_then(Value::as_str) { + files.push(file.to_owned()); + } + current = entry + .get("children") + .and_then(Value::as_array) + .and_then(|children| { + children + .iter() + .find(|child| child.get("child").and_then(Value::as_str) == Some("backing")) + }) + .and_then(|child| child.get("node-name")) + .and_then(Value::as_str) + .map(str::to_owned); + depth += 1; + if depth > BACKING_DEPTH { + break; + } + } + Ok(files) + } /// Run one QMP command and return its `return` value. pub fn execute(&mut self, command: &str) -> Result { @@ -112,82 +236,229 @@ impl Monitor { Ok(parse_block_devices(&report)) } - /// Save the guest's whole state - every device, plus the machine state - - /// under a tag. + /// Take the pool's snapshot of one writable device: a fresh qcow2 + /// overlay, written to `overlay`, becomes the device's top, and the node + /// it was writing goes read-only underneath it. /// - /// `savevm` is the human-monitor spelling of it rather than the QMP - /// `snapshot-save` command, and the difference matters: `snapshot-save` - /// takes the list of devices to capture, and a list that is wrong in - /// either direction produces a snapshot that restores a guest missing - /// state or a guest whose extra state was never captured, both of which - /// look like a healthy restore. `savevm` captures what the guest has, and - /// refuses the whole save if any single device refuses - which is the - /// property the lane depends on, since one raw attached device would - /// otherwise be the one thing a later `loadvm` cannot undo. - pub fn save_snapshot(&mut self, tag: &str) -> Result<()> { - self.human(&format!("savevm {}", Self::quote(tag))) - .map(|_| ()) + /// This is the point the pool snapshots at: after activation has + /// completed, after every attached writable device has been proven + /// rotatable, and before any check has touched the guest. A snapshot + /// taken later would be a snapshot of whatever the last check left + /// behind. + /// + /// The snapshot is named by the node the device is writing rather than + /// by its drive id, because the node is the thing the snapshot is taken + /// *of*: a drive id resolves to whichever node is currently on top, and + /// after a restore that is not the node the caller means. + pub fn take_overlay( + &mut self, + device: &BlockDevice, + overlay: &Path, + overlay_node: &str, + ) -> Result<()> { + self.execute_with( + "blockdev-snapshot-sync", + json!({ + "node-name": device.node, + "snapshot-file": overlay.to_string_lossy(), + "snapshot-node-name": overlay_node, + "format": "qcow2", + // The overlay records its backing by absolute path, so a + // restore can re-open the frozen image from wherever the + // member's working directory is when it happens. + "mode": "absolute-paths", + }), + ) + .map(|_| ()) + } + + /// Re-open a frozen image as a block node, so a fresh overlay can be + /// taken over it. + /// + /// A restore does not keep the snapshot in the block graph: the emulator + /// releases a node as soon as the last device drops it, and a restore + /// drops every device. The snapshot is the *file*, and this is how it + /// becomes a node again. + pub fn add_frozen_image( + &mut self, + node: &str, + file: &Path, + format: &str, + cache: Cache, + ) -> Result<()> { + self.execute_with( + "blockdev-add", + json!({ + "node-name": node, + "driver": format, + "file": { "driver": "file", "filename": file.to_string_lossy() }, + "cache": cache.as_options(), + // Read-only, because nothing may write into a snapshot. A + // restore that leaked a write into the frozen image would + // make the *next* restore show the check before it, which is + // exactly the failure a member reused across checks cannot + // have. + "read-only": true, + }), + ) + .map(|_| ()) } - /// Restore the guest from a snapshot it saved itself. - pub fn load_snapshot(&mut self, tag: &str) -> Result<()> { - self.human(&format!("loadvm {}", Self::quote(tag))) + /// Re-open a layer file as a node of the caller's own. + /// + /// The layer a snapshot writes and the node a device is attached to are + /// two different things here. A node `blockdev-snapshot-sync` creates is + /// dropped as soon as nothing references it, and a restore builds its + /// chain with no device on the graph at all - so the node the device goes + /// back on is opened by the lane over the file the snapshot wrote. The + /// file's own header names the frozen image underneath it, so the chain + /// is rebuilt from the file alone, and the node is writable where the + /// snapshot's own node would have inherited the read-only image beneath + /// it. + pub fn add_overlay_node(&mut self, node: &str, layer: &Path, cache: Cache) -> Result<()> { + self.execute_with( + "blockdev-add", + json!({ + "node-name": node, + "driver": "qcow2", + "file": { "driver": "file", "filename": layer.to_string_lossy() }, + "cache": cache.as_options(), + }), + ) + .map(|_| ()) + } + + /// Detach a device from the block graph and wait for the emulator to + /// finish detaching it. + /// + /// The wait is the point: `device_del` returns as soon as the request is + /// accepted, and the device keeps its node until the guest acknowledges + /// the unplug. A restore that dropped the dirty layer before that + /// happened is refused by the emulator, and one that re-attached the + /// device first would put two devices on one disk. + pub fn detach_device(&mut self, device: &BlockDevice, bound: Duration) -> Result<()> { + self.execute_with("device_del", json!({ "id": device.qdev }))?; + self.wait_for_event("DEVICE_DELETED", bound, &device.qdev) + } + + /// Attach a device to a node, with the properties its configuration + /// declared. + /// + /// The properties go in as arguments of their own rather than flattened + /// into the driver string. `device_add` reads that string as a model name + /// and nothing else, and refuses the whole call over a comma in it + /// (`... is not a valid device model name`), so the spelling the launch + /// uses on the command line is not the spelling the monitor takes. + /// + /// `bootindex` is the one property whose type is not a string: the + /// emulator takes the integer the guest's configuration wrote, and + /// answers a string with `Invalid parameter type for 'bootindex', expected: + /// integer`. + pub fn attach_device( + &mut self, + model: &str, + node: &str, + id: &str, + properties: &[(String, String)], + ) -> Result<()> { + let mut arguments = serde_json::Map::new(); + arguments.insert("driver".to_owned(), json!(model)); + arguments.insert("drive".to_owned(), json!(node)); + arguments.insert("id".to_owned(), json!(id)); + for (key, value) in properties { + let value = if key == "bootindex" { + json!(value.parse::().map_err(|_| { + HarnessError::Configuration(format!( + "the guest's drive declares the boot index {value:?}, which is not a \ + number, and the monitor takes that property as one" + )) + })?) + } else { + json!(value) + }; + arguments.insert(key.clone(), value); + } + self.execute_with("device_add", Value::Object(arguments)) .map(|_| ()) } - /// Remove a snapshot, which is what retiring a member frees. - pub fn delete_snapshot(&mut self, tag: &str) -> Result<()> { - self.human(&format!("delvm {}", Self::quote(tag))) + /// Drop a block node, which is how the layer a check dirtied is freed. + pub fn delete_node(&mut self, node: &str) -> Result<()> { + self.execute_with("blockdev-del", json!({ "node-name": node })) .map(|_| ()) } - /// The tags this guest currently holds a snapshot under. - pub fn snapshot_tags(&mut self) -> Result> { - let report = self.execute("snapshot-list")?; + /// Whether the emulator still holds a node under this name. + pub fn node_present(&mut self, node: &str) -> Result { + let report = self.execute("query-named-block-nodes")?; Ok(report - .get("snapshots") - .and_then(|snapshots| snapshots.as_array()) - .map(|snapshots| { - snapshots - .iter() - .filter_map(|snapshot| { - snapshot - .get("tag") - .and_then(|tag| tag.as_str()) - .map(str::to_owned) - }) - .collect() - }) - .unwrap_or_default()) + .as_array() + .into_iter() + .flatten() + .any(|entry| entry.get("node-name").and_then(Value::as_str) == Some(node))) } - /// Run one human-monitor command, treating anything the monitor printed - /// on stderr as a refusal. + /// Free the layer a check dirtied, if it is still in the graph. /// - /// The human monitor reports a failure in the text it returns rather than - /// in the QMP envelope, so an empty return would turn a refused - /// `loadvm` into a silent success - and a lane that believed it had - /// restored a guest, onto a guest still in whatever state the last check - /// left it in, is the failure this whole layer exists to prevent. - fn human(&mut self, command: &str) -> Result { - let output = self.execute_with( - "human-monitor-command", - json!({ "command-line": command }), - )?; - let text = output.as_str().unwrap_or_default().trim().to_owned(); - if text.is_empty() { - Ok(text) - } else { - Err(HarnessError::Monitor { - command: command.to_owned(), - detail: text, - }) + /// The emulator drops an unused node's whole chain as soon as the last + /// device on it goes away, so a restore that detaches every device first + /// usually finds the layer already gone - which is the state the restore + /// wants, not a failure. Asking for it is what keeps the two apart: a + /// delete that fails because the node is not there would otherwise read + /// as a restore that could not roll back, and the writes it carried are + /// gone either way. + pub fn drop_node_if_present(&mut self, node: &str) -> Result<()> { + if self.node_present(node)? { + self.delete_node(node)?; } + Ok(()) } - /// A snapshot tag, quoted for the human monitor's own parser. - fn quote(tag: &str) -> String { - format!("'{}'", tag.replace('\'', "'\\''")) + /// Restart the guest, so it runs again from the disk it is attached to. + /// + /// A disk-only restore replaces what is under the guest without + /// replacing what is in it: the page cache, the mounted filesystems and + /// every service's state would still be the ones the check before it left + /// behind. Resetting is what makes the restored disk the state the guest + /// is actually running on, which is the state a check's assertions were + /// written against. + pub fn reset_machine(&mut self) -> Result<()> { + self.execute("system_reset").map(|_| ()) + } + + /// Wait for one asynchronous event, bounded. + /// + /// The monitor's socket is a blocking stream, so the wait is a poll: a + /// command whose reply carries the event with it, and a clock. Reading + /// the socket under a read timeout instead would leave the reader + /// positioned mid-line, and the next command would read the tail of this + /// one. + fn wait_for_event(&mut self, event: &str, bound: Duration, about: &str) -> Result<()> { + // Only an event that arrives *after* this wait began can be evidence + // for the call that was supposed to produce it: the emulator's events + // are a stream, and the one a previous detach left here would satisfy + // this detach without the device having moved at all - which is how a + // device that is still attached reaches the command after it, in the + // shape of `blockdev-del: Node ... is in use`. + self.events.clear(); + let deadline = Instant::now() + bound; + loop { + if self + .events + .iter() + .any(|message| message.get("event").and_then(Value::as_str) == Some(event)) + { + return Ok(()); + } + if Instant::now() >= deadline { + return Err(HarnessError::Configuration(format!( + "the emulator did not report {event} for {about} within {}s", + bound.as_secs() + ))); + } + self.execute("query-status")?; + std::thread::sleep(POLL_INTERVAL); + } } #[allow(clippy::disallowed_methods, reason = "synchronous path")] @@ -221,12 +492,19 @@ impl Monitor { /// /// A backend with no image behind it - an empty drive, or a device whose /// medium is not plugged - is skipped: there is nothing to snapshot and -/// nothing to refuse. A read-only node is snapshottable by construction: -/// the emulator excludes it from a snapshot's device set. Every other node -/// has to be qcow2, because in the current emulator qcow2 is the only format -/// that implements the snapshot vtable, and a node that does not is one a -/// later `snapshot-save` refuses - refusing the whole save, not just that -/// node. +/// nothing to refuse. A read-only node is skipped for the same reason: the +/// emulator never writes through it, so there is nothing to roll back. Every +/// other node has to be qcow2, because an external snapshot is a qcow2 +/// overlay and a node that cannot be backed by one is a device a member +/// cannot be restored from. +/// +/// Two paths are read out of the report because a restore needs both: the +/// node, which is what the snapshot is taken of and what has to be dropped +/// afterwards, and the device's own path, which is what detaching it takes. +/// The emulator names a device attached after the guest started by its qdev +/// path rather than by a drive id - carrying `"device": ""` rather than +/// omitting the key - so an empty id has to read as no id, or the refusal +/// names nothing. fn parse_block_devices(report: &Value) -> Vec { report .as_array() @@ -243,33 +521,61 @@ fn parse_block_devices(report: &Value) -> Vec { .and_then(Value::as_str) .unwrap_or("unknown") .to_owned(); + if read_only { + return None; + } + // The device's own path is the parent of the path the report + // carries: the report names the block *backend*, and the backend + // hangs off the device it feeds. + let qdev = entry + .get("qdev") + .and_then(Value::as_str) + .map(device_path) + .unwrap_or_default(); + let node = inserted + .get("node-name") + .and_then(Value::as_str) + .unwrap_or("") + .to_owned(); + let id = entry + .get("device") + .and_then(Value::as_str) + .filter(|id| !id.is_empty()) + .map_or_else(|| qdev.clone(), str::to_owned); Some(BlockDevice { - // An entry the emulator names by qdev path rather than by - // drive id - a device attached after the guest started, for - // one - carries `"device": ""` rather than omitting the key, - // so an empty id has to read as no id, or the refusal names - // nothing. - id: entry - .get("device") - .and_then(Value::as_str) - .filter(|id| !id.is_empty()) - .or_else(|| entry.get("qdev").and_then(Value::as_str)) - .unwrap_or("unnamed") - .to_owned(), + qdev, + id, + node: node.clone(), file: inserted .get("file") .and_then(Value::as_str) .unwrap_or("") .to_owned(), - snapshottable: read_only || format == "qcow2", + read_only, + // A node the emulator named is a node a restore can name + // too; a device that reports none cannot be detached and + // re-attached, so it is refused rather than silently carried + // over to the next check. + rotatable: format == "qcow2" && !node.is_empty(), format, }) }) .collect() } +/// The device's own path in the machine's device tree, given the path a block +/// report names for it. +fn device_path(reported: &str) -> String { + match reported.rsplit_once('/') { + Some((parent, leaf)) if leaf.ends_with("-backend") => parent.to_owned(), + _ => reported.to_owned(), + } +} + #[cfg(test)] mod tests { + use std::sync::{Arc, Mutex}; + use super::*; const GREETING: &str = r#"{"QMP": {"version": {"qemu": {"major": 10, "minor": 2, "micro": 2}}, "capabilities": []}}"#; @@ -279,52 +585,128 @@ mod tests { /// drives. The canned side writes the greeting and every reply up front /// and then drains, which is what lets a test deliver an unsolicited /// event - the case a request/response fake cannot express. - fn monitor_answering(replies: &'static [&'static str]) -> Monitor { + /// + /// The requests are kept, because a restore's whole argument is in the + /// command the lane writes: a reply the canned side sends proves nothing + /// about which node an overlay was taken of, or how a device was + /// re-attached. + fn monitor_answering(replies: &'static [&'static str]) -> Wired { let (lane_end, emulator_end) = UnixStream::pair().expect("a socket pair"); + let requests: Arc>> = Arc::new(Mutex::new(Vec::new())); + let seen = Arc::clone(&requests); std::thread::spawn(move || { let mut reader = BufReader::new(emulator_end.try_clone().expect("clone")); let mut writer = emulator_end; for reply in std::iter::once(GREETING).chain(replies.iter().copied()) { writeln!(writer, "{reply}").expect("write a reply"); } - // Keep reading so the lane never writes into a closed socket. + // Keep reading so the lane never writes into a closed socket, + // and keep what it wrote: that is the command under test. let mut line = String::new(); while reader.read_line(&mut line).expect("read a request") > 0 { + if let Ok(request) = serde_json::from_str(line.trim()) { + seen.lock().expect("the request log").push(request); + } line.clear(); } }); - Monitor::from_stream(lane_end).expect("the greeting and handshake succeed") + Wired { + monitor: Monitor::from_stream(lane_end).expect("the greeting and handshake succeed"), + requests, + } + } + + /// A wired monitor and the requests the canned emulator saw. + struct Wired { + monitor: Monitor, + requests: Arc>>, + } + + impl std::ops::Deref for Wired { + type Target = Monitor; + + fn deref(&self) -> &Monitor { + &self.monitor + } + } + + impl std::ops::DerefMut for Wired { + fn deref_mut(&mut self) -> &mut Monitor { + &mut self.monitor + } + } + + impl Wired { + /// The last request the lane wrote, which is the command a test is + /// asserting about. + /// + /// The canned side writes its replies before it reads, so the + /// request that earned the reply being asserted on may not be logged + /// yet; the wait is for that, not for the command. + fn sent(&self) -> Value { + for _ in 0..200 { + let last = self + .requests + .lock() + .ok() + .and_then(|log| log.last().cloned()) + .unwrap_or(Value::Null); + if last.get("execute").is_some_and(|command| command != "qmp_capabilities") { + return last; + } + std::thread::sleep(Duration::from_millis(10)); + } + panic!("the lane wrote no command beyond the handshake"); + } } + /// The report the pinned emulator produces for a booted lane guest: a + /// read-only host share, a raw store-backed node, a qcow2 root, and the + /// qcow2 overlay the emulator put over an ephemeral state disk. const BLOCK_REPORT: &str = concat!( r#"{"return": ["#, r#"{"device": "file-nix-store", "inserted": {"drv": "raw", "file": "/nix/store/s.img", "ro": true}},"#, - r#"{"device": "virtio0", "inserted": {"drv": "raw", "file": "/run/lane/disk.qcow2", "ro": false}},"#, - r#"{"device": "virtio1", "inserted": {"drv": "qcow2", "file": "/run/lane/state.qcow2", "ro": false}},"#, + r#"{"device": "virtio0", "qdev": "/machine/peripheral-anon/device[10]/virtio-backend", "inserted": {"drv": "qcow2", "file": "/run/lane/vl.ABC123", "node-name": "lane_state.overlay", "ro": false}},"#, + r#"{"device": "lane_drive_0", "qdev": "/machine/peripheral-anon/device[4]/virtio-backend", "inserted": {"drv": "qcow2", "file": "/run/lane/disk.qcow2", "node-name": "lane_root", "ro": false}},"#, + r#"{"device": "lane_bad", "qdev": "/machine/peripheral-anon/device[6]/virtio-backend", "inserted": {"drv": "raw", "file": "/run/lane/refusal.img", "node-name": "lane_refusal", "ro": false}},"#, r#"{"qdev": "ide0-cd0"}"#, r#"]}"# ); #[test] - fn a_writable_raw_device_is_reported_as_unsnapshottable() { + fn every_writable_device_carries_the_node_and_the_path_a_restore_needs() { let mut monitor = monitor_answering(&[r#"{"return": {}}"#, BLOCK_REPORT]); let devices = monitor.block_devices().expect("the report parses"); assert_eq!( - devices - .iter() - .filter(|device| !device.snapshottable) - .map(|device| device.id.as_str()) - .collect::>(), - vec!["virtio0"], - "a writable non-qcow2 node is the one a snapshot-save would refuse" + devices.iter().map(|device| device.id.as_str()).collect::>(), + vec!["virtio0", "lane_drive_0", "lane_bad"], + "a read-only share and a backend with no medium are both skipped" + ); + assert_eq!( + devices[1].node, "lane_root", + "the snapshot is taken of the node the device is writing" + ); + assert_eq!( + devices[1].qdev, "/machine/peripheral-anon/device[4]", + "the path to detach is the device's own, not the backend's" ); - assert_eq!(devices.len(), 3, "a backend with no image is skipped"); - assert_eq!(devices[0].format, "raw", "a read-only backing is judged raw"); assert!( - devices[0].snapshottable, - "a read-only backing is excluded from a snapshot, so it never blocks one" + devices[0].rotatable && devices[1].rotatable, + "a named qcow2 node is one a restore can take and put back" ); - assert!(devices[2].snapshottable, "the qcow2 overlay carries the snapshot"); + } + + #[test] + fn a_writable_node_that_is_not_qcow2_is_refused_before_the_pool_is_built() { + let mut monitor = monitor_answering(&[r#"{"return": {}}"#, BLOCK_REPORT]); + let devices = monitor.block_devices().expect("the report parses"); + let refused = devices + .iter() + .find(|device| !device.rotatable) + .expect("a writable raw node is refused"); + assert_eq!(refused.id, "lane_bad"); + assert_eq!(refused.file, "/run/lane/refusal.img"); + assert_eq!(refused.format, "raw"); } /// The shape the real emulator reports for a device attached after the @@ -333,8 +715,8 @@ mod tests { /// the lane's refusal names a device with no name. const HOTPLUGGED_BLOCK_REPORT: &str = concat!( r#"{"return": ["#, - r#"{"device": "lane_drive_0", "inserted": {"drv": "qcow2", "file": "/run/lane/disk.qcow2", "ro": false}},"#, - r#"{"device": "", "qdev": "/machine/peripheral/lane-refusal/virtio-backend", "inserted": {"drv": "raw", "file": "/run/lane/refusal.img", "ro": false}}"#, + r#"{"device": "lane_drive_0", "qdev": "/machine/peripheral-anon/device[4]/virtio-backend", "inserted": {"drv": "qcow2", "file": "/run/lane/disk.qcow2", "node-name": "lane_root", "ro": false}},"#, + r#"{"device": "", "qdev": "/machine/peripheral/lane-refusal/virtio-backend", "inserted": {"drv": "raw", "file": "/run/lane/refusal.img", "node-name": "lane_refusal", "ro": false}}"#, r#"]}"# ); @@ -344,20 +726,166 @@ mod tests { let devices = monitor.block_devices().expect("the report parses"); let refused = devices .iter() - .find(|device| !device.snapshottable) + .find(|device| !device.rotatable) .expect("a writable raw node is refused"); assert_eq!( - refused.id, "/machine/peripheral/lane-refusal/virtio-backend", - "the refusal names the device by its qdev path, not by an empty id" + refused.id, "/machine/peripheral/lane-refusal", + "the refusal names the device by its own qdev path, not by an empty id" ); - assert_eq!(refused.file, "/run/lane/refusal.img"); - assert_eq!(refused.format, "raw"); - assert!( - devices[0].snapshottable, - "the qcow2 root drive is not what failed the lane" + assert_eq!(refused.qdev, "/machine/peripheral/lane-refusal"); + assert!(devices[0].rotatable, "the qcow2 root drive is not what failed"); + } + + #[test] + fn an_overlay_is_taken_against_the_node_rather_than_the_drive() { + let mut monitor = monitor_answering(&[ + r#"{"return": {}}"#, + r#"{"return": {}}"#, + r#"{"return": {}}"#, + ]); + let device = BlockDevice { + id: "lane_drive_0".to_owned(), + qdev: "/machine/peripheral-anon/device[4]".to_owned(), + node: "lane_root.overlay".to_owned(), + file: "/run/lane/lane_root.overlay.qcow2".to_owned(), + format: "qcow2".to_owned(), + read_only: false, + rotatable: true, + }; + monitor + .take_overlay(&device, Path::new("/run/lane/lane_root.overlay-2.qcow2"), "lane_root.overlay2") + .expect("the overlay is taken"); + let sent = monitor.sent(); + assert_eq!(sent["execute"], "blockdev-snapshot-sync"); + assert_eq!( + sent["arguments"]["node-name"], "lane_root.overlay", + "the overlay is taken of the node, which a drive id would not name after a restore" + ); + assert_eq!(sent["arguments"]["format"], "qcow2"); + assert_eq!( + sent["arguments"]["mode"], "absolute-paths", + "the overlay has to record its backing by a path the restore can re-open" + ); + } + + #[test] + fn a_frozen_image_is_re_opened_read_only() { + let mut monitor = monitor_answering(&[r#"{"return": {}}"#, r#"{"return": {}}"#]); + monitor + .add_frozen_image( + "lane_root", + Path::new("/run/lane/disk.qcow2"), + "qcow2", + Cache::Writeback, + ) + .expect("the frozen image re-opens"); + let sent = monitor.sent(); + assert_eq!(sent["execute"], "blockdev-add"); + assert_eq!(sent["arguments"]["read-only"], true); + assert_eq!(sent["arguments"]["cache"]["no-flush"], false); + } + + #[test] + fn a_cache_mode_the_guest_did_not_declare_is_refused() { + let error = Cache::parse("writaback").expect_err("a misspelt mode is refused"); + assert!(error.to_string().contains("writaback"), "{error}"); + assert_eq!(Cache::parse("unsafe").expect("a declared mode parses"), Cache::Unsafe); + assert_eq!(Cache::Unsafe.as_options()["no-flush"], true); + assert_eq!(Cache::None.as_options()["direct"], true); + } + + #[test] + fn detaching_a_device_waits_for_the_emulator_to_finish_it() { + let mut monitor = monitor_answering(&[ + r#"{"return": {}}"#, + r#"{"event": "DEVICE_DELETED", "data": {"device": "/machine/peripheral-anon/device[4]"}}"#, + r#"{"return": {"status": "running"}}"#, + ]); + let device = BlockDevice { + id: "lane_drive_0".to_owned(), + qdev: "/machine/peripheral-anon/device[4]".to_owned(), + node: "lane_root.overlay".to_owned(), + file: "/run/lane/lane_root.overlay.qcow2".to_owned(), + format: "qcow2".to_owned(), + read_only: false, + rotatable: true, + }; + monitor + .detach_device(&device, Duration::from_secs(5)) + .expect("the event arrives with the next command's reply"); + } + + #[test] + fn a_detach_the_emulator_never_reports_fails_with_the_device_named() { + // The handshake's reply, the detach's, and one per poll the bound + // allows before it trips. + let mut monitor = monitor_answering(&[ + r#"{"return": {}}"#, + r#"{"return": {}}"#, + r#"{"return": {"status": "running"}}"#, + r#"{"return": {"status": "running"}}"#, + r#"{"return": {"status": "running"}}"#, + r#"{"return": {"status": "running"}}"#, + r#"{"return": {"status": "running"}}"#, + r#"{"return": {"status": "running"}}"#, + ]); + let device = BlockDevice { + id: "lane_drive_0".to_owned(), + qdev: "/machine/peripheral-anon/device[4]".to_owned(), + node: "lane_root.overlay".to_owned(), + file: "/run/lane/lane_root.overlay.qcow2".to_owned(), + format: "qcow2".to_owned(), + read_only: false, + rotatable: true, + }; + let error = monitor + .detach_device(&device, Duration::from_millis(300)) + .expect_err("an unplug the emulator never confirms is a failure"); + let rendered = error.to_string(); + assert!(rendered.contains("DEVICE_DELETED"), "{rendered}"); + assert!(rendered.contains("device[4]"), "{rendered}"); + } + + #[test] + fn a_device_comes_back_with_the_properties_its_configuration_declared() { + let mut monitor = monitor_answering(&[r#"{"return": {}}"#, r#"{"return": {}}"#]); + monitor + .attach_device( + "virtio-blk-pci", + "lane_root.overlay2", + "lane_drive_0", + &[("serial".to_owned(), "root".to_owned()), ("bootindex".to_owned(), "1".to_owned())], + ) + .expect("the device is attached"); + let sent = monitor.sent(); + // The properties are the emulator's own arguments and not a driver + // string: it reads that string as a model name and refuses a comma in + // it, and it takes `bootindex` as the integer the guest's + // configuration wrote rather than as the text of one. + assert_eq!(sent["arguments"]["driver"], "virtio-blk-pci"); + assert_eq!(sent["arguments"]["drive"], "lane_root.overlay2"); + assert_eq!(sent["arguments"]["id"], "lane_drive_0"); + assert_eq!(sent["arguments"]["serial"], "root"); + assert_eq!( + sent["arguments"]["bootindex"], 1, + "the emulator answers a string with `Invalid parameter type for 'bootindex'`" ); } + #[test] + fn a_boot_index_that_is_not_a_number_is_refused_before_the_monitor() { + let mut monitor = monitor_answering(&[r#"{"return": {}}"#]); + let error = monitor + .attach_device( + "virtio-blk-pci", + "lane_root.overlay2", + "lane_drive_0", + &[("bootindex".to_owned(), "first".to_owned())], + ) + .expect_err("a boot index the emulator cannot take is a configuration failure"); + assert!(error.to_string().contains("boot index"), "{error}"); + } + #[test] fn a_monitor_error_becomes_a_named_failure() { let mut monitor = monitor_answering(&[ From 1c4c3ccbaf58134c739cdc60ef7cf1a9deb558c1 Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sat, 26 Sep 2026 20:45:56 -0700 Subject: [PATCH 08/51] refactor(vm): rename d2b-vm-harness to d2b-test-vm-harness The package, its crate and Bazel target names, the binary, and the D2B_VM_HARNESS_* environment prefix are renamed to d2b-test-vm-harness and D2B_TEST_VM_HARNESS_*. The prefix is not cosmetic: it was observed set in both generated lane runners and read by the live harness process in /proc/environ, because an unread variable yields a default rather than an error, so a passing lane could not otherwise have distinguished a working rename from a cosmetic one. Not renamed, because they are a different concept that shares the prefix: d2b_vm_* are per-VM-instance Prometheus metric names in d2bd-runtime, and the same names key four Grafana dashboards. Renaming those would silently break live metrics and every dashboard, with nothing in CI catching it before a release. D2B_VM_CHECK in the Makefile selects VM checks and is not this package; the legacy lane's D2B_VM_SSH_KEY_* are per-VM as well. CHANGELOG.md is released history and docs/plans/ are prior artifacts. The lane's per-check table was not re-measured: the verification run was damaged by a concurrent cleanup that removed its working directories underneath running guests. The build, clippy and crate policy gates are green, the rename is complete across every tracked file, and the two failing checks are a separate known defect in the restore path - one that this table would not have exonerated. --- BUILD.bazel | 2 +- Cargo.lock | 16 ++++++++-------- Cargo.toml | 2 +- bazel/checks/vm/BUILD.bazel | 10 +++++----- bazel/checks/vm/defs.bzl | 18 +++++++++--------- .../bazel-owned-legacy-check-surface.md | 2 +- changelog.d/bazel-owned-vm-harness.md | 2 +- changelog.d/v3.md | 3 +++ .../BUILD.bazel | 12 ++++++------ .../Cargo.toml | 6 +++--- .../src/bin/d2b-test-vm-harness.rs} | 14 +++++++------- .../src/diagnostics.py | 2 +- .../src/error.rs | 0 .../src/guest.rs | 0 .../src/host.rs | 0 .../src/legacy.rs | 2 +- .../src/legacy_bridge.py | 0 .../src/lib.rs | 0 .../src/manifest.rs | 0 .../src/monitor.rs | 0 packages/xtask/src/provider_crate_policy.rs | 2 +- tests/host-integration/lib.nix | 4 ++-- 22 files changed, 50 insertions(+), 47 deletions(-) create mode 100644 changelog.d/v3.md rename packages/{d2b-vm-harness => d2b-test-vm-harness}/BUILD.bazel (91%) rename packages/{d2b-vm-harness => d2b-test-vm-harness}/Cargo.toml (90%) rename packages/{d2b-vm-harness/src/bin/d2b-vm-harness.rs => d2b-test-vm-harness/src/bin/d2b-test-vm-harness.rs} (99%) rename packages/{d2b-vm-harness => d2b-test-vm-harness}/src/diagnostics.py (98%) rename packages/{d2b-vm-harness => d2b-test-vm-harness}/src/error.rs (100%) rename packages/{d2b-vm-harness => d2b-test-vm-harness}/src/guest.rs (100%) rename packages/{d2b-vm-harness => d2b-test-vm-harness}/src/host.rs (100%) rename packages/{d2b-vm-harness => d2b-test-vm-harness}/src/legacy.rs (99%) rename packages/{d2b-vm-harness => d2b-test-vm-harness}/src/legacy_bridge.py (100%) rename packages/{d2b-vm-harness => d2b-test-vm-harness}/src/lib.rs (100%) rename packages/{d2b-vm-harness => d2b-test-vm-harness}/src/manifest.rs (100%) rename packages/{d2b-vm-harness => d2b-test-vm-harness}/src/monitor.rs (100%) diff --git a/BUILD.bazel b/BUILD.bazel index ea15be787..7f2b28011 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -353,7 +353,7 @@ filegroup( "//packages/d2b-provider-host:cargo_workspace_sources", "//packages/d2b-provider-user:cargo_workspace_sources", "//packages/d2b-provider-test-controller:cargo_workspace_sources", - "//packages/d2b-vm-harness:cargo_workspace_sources", + "//packages/d2b-test-vm-harness:cargo_workspace_sources", "//packages/d2b-resource-runtime:cargo_workspace_sources", "//packages/d2b-audit:BUILD.bazel", "//packages/d2b-broker-composition:BUILD.bazel", diff --git a/Cargo.lock b/Cargo.lock index 6900ab9e0..90c27c040 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2154,6 +2154,14 @@ dependencies = [ "sha2", ] +[[package]] +name = "d2b-test-vm-harness" +version = "0.0.0-bootstrap" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "d2b-unsafe-local-helper" version = "0.0.0-bootstrap" @@ -2173,14 +2181,6 @@ dependencies = [ "zbus", ] -[[package]] -name = "d2b-vm-harness" -version = "0.0.0-bootstrap" -dependencies = [ - "serde", - "serde_json", -] - [[package]] name = "d2b-zone-routing" version = "0.0.0-bootstrap" diff --git a/Cargo.toml b/Cargo.toml index 0ca9e6e6d..e0670aab6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -95,7 +95,7 @@ members = [ "packages/d2b-provider-command", "packages/d2b-provider-operation", "packages/d2b-provider-seccomp-profile", - "packages/d2b-vm-harness", + "packages/d2b-test-vm-harness", ] [workspace.package] diff --git a/bazel/checks/vm/BUILD.bazel b/bazel/checks/vm/BUILD.bazel index c3aa045ff..46f7c2c4e 100644 --- a/bazel/checks/vm/BUILD.bazel +++ b/bazel/checks/vm/BUILD.bazel @@ -39,7 +39,7 @@ _GUEST_SOURCES = [ "//tests/fixtures:fixture_sources", # The diagnostics prelude the fixtures interpolate and the lane's own # guest-control surface embeds, read out of the tree by both. - "//packages/d2b-vm-harness:src/diagnostics.py", + "//packages/d2b-test-vm-harness:src/diagnostics.py", # The provider manifests the acceptance fixtures sign their artifacts # against, read by path out of the tree by the fixtures' own helpers. "//packages/d2b-provider-guest-cloud-hypervisor:provider-manifest.json", @@ -162,7 +162,7 @@ _CHECK_IMAGES = [":guest_image_" + check for check in _CHECKS] _EMULATOR = "@qemu_kvm//:bin/qemu-kvm" # The lane's harness, which is the thing that spawns a guest. -_HARNESS = "//packages/d2b-vm-harness:d2b-vm-harness" +_HARNESS = "//packages/d2b-test-vm-harness:d2b-test-vm-harness" guest_boot_test( name = "guest_boot_daemon", @@ -199,9 +199,9 @@ lane_test( test_suite( name = "host_integration_lane", tests = [ - "//packages/d2b-vm-harness:d2b-vm-harness_clippy", - "//packages/d2b-vm-harness:d2b_vm_harness_clippy", - "//packages/d2b-vm-harness:d2b_vm_harness_test", + "//packages/d2b-test-vm-harness:d2b-test-vm-harness_clippy", + "//packages/d2b-test-vm-harness:d2b_test_vm_harness_clippy", + "//packages/d2b-test-vm-harness:d2b_test_vm_harness_test", ":guest_boot_daemon", ":guest_boot_writable_store", ":host_integration_lane_run", diff --git a/bazel/checks/vm/defs.bzl b/bazel/checks/vm/defs.bzl index 12405ca27..61d2ca412 100644 --- a/bazel/checks/vm/defs.bzl +++ b/bazel/checks/vm/defs.bzl @@ -302,14 +302,14 @@ set -eu # assumed - a wrong guess here is a guest that never boots, with a path error # instead of a boot error. runfiles="$(CDPATH= cd -- "$(dirname -- "$0")/../../../.." && pwd -P)" -export D2B_VM_HARNESS_CYCLES="{cycles}" -export D2B_VM_HARNESS_EMULATOR="$runfiles/__EMULATOR__" -export D2B_VM_HARNESS_IMAGE="$runfiles/__IMAGE__" +export D2B_TEST_VM_HARNESS_CYCLES="{cycles}" +export D2B_TEST_VM_HARNESS_EMULATOR="$runfiles/__EMULATOR__" +export D2B_TEST_VM_HARNESS_IMAGE="$runfiles/__IMAGE__" # A lane-scoped working directory that outlives each individual guest, and is # this test's own rather than the sandboxed temporary directory the current # Bazel release does not expose to a sandboxed action. The harness resolves a # relative one against its own working directory. -export D2B_VM_HARNESS_WORK_ROOT="{work_root}" +export D2B_TEST_VM_HARNESS_WORK_ROOT="{work_root}" exec "$runfiles/__HARNESS__" "$@" """ @@ -404,23 +404,23 @@ set -eu # instead of a boot error. runfiles="$(CDPATH= cd -- "$(dirname -- "$0")/../../../.." && pwd -P)" -export D2B_VM_HARNESS_EMULATOR="$runfiles/__EMULATOR__" -export D2B_VM_HARNESS_IMAGES="$runfiles/__IMAGES__" +export D2B_TEST_VM_HARNESS_EMULATOR="$runfiles/__EMULATOR__" +export D2B_TEST_VM_HARNESS_IMAGES="$runfiles/__IMAGES__" # A check that has not been ported yet is a Python script, and the # interpreter it runs under is part of what the lane is. Left to itself the # surface resolves whatever `python3` a developer's shell happens to find, so # the interpreter is a declared runfile from the same pinned nix package set # as the guest it drives. -export D2B_VM_HARNESS_PYTHON="$runfiles/__PYTHON__" +export D2B_TEST_VM_HARNESS_PYTHON="$runfiles/__PYTHON__" # A lane-scoped working directory that outlives each individual guest, and is # this test's own rather than the sandboxed temporary directory the current # Bazel release does not expose to a sandboxed action. The harness resolves a # relative one against its own working directory. -export D2B_VM_HARNESS_WORK_ROOT="d2b-vm-lane-work/{name}" +export D2B_TEST_VM_HARNESS_WORK_ROOT="d2b-vm-lane-work/{name}" # The harness is asked for the `lane` subcommand rather than being left to its # own default. With no argument it runs the single-guest self-check, which is -# the other target's job and which asks for `D2B_VM_HARNESS_IMAGE` - a variable +# the other target's job and which asks for `D2B_TEST_VM_HARNESS_IMAGE` - a variable # the lane has no reason to set, because the lane reads the whole image list # instead. Anything a contributor passes on the command line reaches the # lane's own selection, which reads `--check`. diff --git a/changelog.d/bazel-owned-legacy-check-surface.md b/changelog.d/bazel-owned-legacy-check-surface.md index f7e61e56c..af4d0f97f 100644 --- a/changelog.d/bazel-owned-legacy-check-surface.md +++ b/changelog.d/bazel-owned-legacy-check-surface.md @@ -6,4 +6,4 @@ ### Changed -- The fixture diagnostics prelude (issue #513) moved out of `tests/host-integration/lib.nix` into `packages/d2b-vm-harness/src/diagnostics.py`, which `lib.nix` now reads and the lane's assertion surface carries. It is the one copy both lanes interpolate: the nix lane reaches it through the fixture's evaluated `testScript`, and the Bazel lane runs that same script, so a failing unported check reports its stage, its row dumps, its unit journals and its zone debug dump through the same text under either lane. A second copy would be a second dialect of the same diagnostics, and the two would drift the first time one of them gained a helper the other did not. +- The fixture diagnostics prelude (issue #513) moved out of `tests/host-integration/lib.nix` into `packages/d2b-test-vm-harness/src/diagnostics.py`, which `lib.nix` now reads and the lane's assertion surface carries. It is the one copy both lanes interpolate: the nix lane reaches it through the fixture's evaluated `testScript`, and the Bazel lane runs that same script, so a failing unported check reports its stage, its row dumps, its unit journals and its zone debug dump through the same text under either lane. A second copy would be a second dialect of the same diagnostics, and the two would drift the first time one of them gained a helper the other did not. diff --git a/changelog.d/bazel-owned-vm-harness.md b/changelog.d/bazel-owned-vm-harness.md index 97c8fb9aa..4c8c7e70d 100644 --- a/changelog.d/bazel-owned-vm-harness.md +++ b/changelog.d/bazel-owned-vm-harness.md @@ -1,6 +1,6 @@ ### Added -- Added `packages/d2b-vm-harness`, the Bazel-owned host-integration lane's guest launcher. It reproduces the emulator invocation a check's guest configuration declared - memory, vCPU count, the drive layout including the writable-store root drive, and per-check device options such as a vsock device - reads that invocation off the guest image's manifest rather than off a uniform shape of its own, boots the guest with hardware virtualization selected explicitly, waits for the guest's own activation contract on its serial console, and tears it down. A guest that never activates fails the lane inside a bound with the console tail attached, rather than hanging. +- Added `packages/d2b-test-vm-harness`, the Bazel-owned host-integration lane's guest launcher. It reproduces the emulator invocation a check's guest configuration declared - memory, vCPU count, the drive layout including the writable-store root drive, and per-check device options such as a vsock device - reads that invocation off the guest image's manifest rather than off a uniform shape of its own, boots the guest with hardware virtualization selected explicitly, waits for the guest's own activation contract on its serial console, and tears it down. A guest that never activates fails the lane inside a bound with the console tail attached, rather than hanging. - The lane asserts the host's virtualization capabilities before it boots anything, and names every one that is missing: `/dev/kvm`, nested virtualization, and nested-state save support. The lane does not fall back to emulation, so a host that cannot run it stops with a message naming the knob to turn rather than running the suite several times slower without saying so. - The lane verifies at boot that every writable device the guest attached can carry an internal snapshot, and refuses the guest when one cannot - before any check runs, rather than at the first restore after a suite has already executed against a guest that could not be rolled back. - Every reusable-pool guest carries a machine identity device and a hardware random source, and the guest's activation marker is written only once its random pool is initialised, so a snapshot can never be taken of a guest whose `getrandom()` would block after a restore. diff --git a/changelog.d/v3.md b/changelog.d/v3.md new file mode 100644 index 000000000..a46538c25 --- /dev/null +++ b/changelog.d/v3.md @@ -0,0 +1,3 @@ +### Changed + +- Renamed `packages/d2b-vm-harness` to `packages/d2b-test-vm-harness`, with the crate, its `Cargo.lock` entry, its Bazel targets, and its `D2B_VM_HARNESS_*` environment contract renamed to match (`D2B_TEST_VM_HARNESS_*`). The name now reads as one name wherever it appears: the package, the binary, the lib, and every variable the lane hands the harness. No behaviour changed; the lane boots the same guests and runs the same checks. diff --git a/packages/d2b-vm-harness/BUILD.bazel b/packages/d2b-test-vm-harness/BUILD.bazel similarity index 91% rename from packages/d2b-vm-harness/BUILD.bazel rename to packages/d2b-test-vm-harness/BUILD.bazel index 53f0c01a0..4f5d720a8 100644 --- a/packages/d2b-vm-harness/BUILD.bazel +++ b/packages/d2b-test-vm-harness/BUILD.bazel @@ -26,7 +26,7 @@ exports_files( ) d2b_rust_library( - name = "d2b_vm_harness", + name = "d2b_test_vm_harness", srcs = glob( ["src/**/*.rs"], exclude = ["src/bin/**/*.rs"], @@ -47,17 +47,17 @@ d2b_rust_library( ) d2b_rust_binary( - name = "d2b-vm-harness", - srcs = ["src/bin/d2b-vm-harness.rs"], + name = "d2b-test-vm-harness", + srcs = ["src/bin/d2b-test-vm-harness.rs"], compile_data = ["Cargo.toml"], deps = [ - ":d2b_vm_harness", + ":d2b_test_vm_harness", ] + all_crate_deps(normal = True, cargo_only = True), ) d2b_rust_test( - name = "d2b_vm_harness_test", - crate = ":d2b_vm_harness", + name = "d2b_test_vm_harness_test", + crate = ":d2b_test_vm_harness", compile_data = ["Cargo.toml"], deps = all_crate_deps(normal = True, normal_dev = True, cargo_only = True), ) diff --git a/packages/d2b-vm-harness/Cargo.toml b/packages/d2b-test-vm-harness/Cargo.toml similarity index 90% rename from packages/d2b-vm-harness/Cargo.toml rename to packages/d2b-test-vm-harness/Cargo.toml index 7dc12d890..951787bb6 100644 --- a/packages/d2b-vm-harness/Cargo.toml +++ b/packages/d2b-test-vm-harness/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "d2b-vm-harness" +name = "d2b-test-vm-harness" version = "0.0.0-bootstrap" edition = "2024" publish = false @@ -28,5 +28,5 @@ serde = { workspace = true } serde_json = { workspace = true } [[bin]] -name = "d2b-vm-harness" -path = "src/bin/d2b-vm-harness.rs" +name = "d2b-test-vm-harness" +path = "src/bin/d2b-test-vm-harness.rs" diff --git a/packages/d2b-vm-harness/src/bin/d2b-vm-harness.rs b/packages/d2b-test-vm-harness/src/bin/d2b-test-vm-harness.rs similarity index 99% rename from packages/d2b-vm-harness/src/bin/d2b-vm-harness.rs rename to packages/d2b-test-vm-harness/src/bin/d2b-test-vm-harness.rs index e45f57088..2924085b8 100644 --- a/packages/d2b-vm-harness/src/bin/d2b-vm-harness.rs +++ b/packages/d2b-test-vm-harness/src/bin/d2b-test-vm-harness.rs @@ -24,7 +24,7 @@ use std::{ time::{Duration, Instant}, }; -use d2b_vm_harness::{ +use d2b_test_vm_harness::{ ActiveGuest, Footprint, GuestSpec, HarnessError, HostFacts, LegacyCheck, LegacyGuest, SnapshotPoint, boot, host, manifest::GuestManifest, report, }; @@ -32,16 +32,16 @@ use serde_json::json; /// The image the lane was pointed at. Supplied by the Bazel test target as a /// runfile path. -const IMAGE: &str = "D2B_VM_HARNESS_IMAGE"; +const IMAGE: &str = "D2B_TEST_VM_HARNESS_IMAGE"; /// The emulator binary, from the pinned nix package set the guest closure was /// realized from. -const EMULATOR: &str = "D2B_VM_HARNESS_EMULATOR"; +const EMULATOR: &str = "D2B_TEST_VM_HARNESS_EMULATOR"; /// The lane's working directory, which outlives each individual guest. -const WORK_ROOT: &str = "D2B_VM_HARNESS_WORK_ROOT"; +const WORK_ROOT: &str = "D2B_TEST_VM_HARNESS_WORK_ROOT"; /// How long the guest has to activate. -const ACTIVATION_TIMEOUT: &str = "D2B_VM_HARNESS_ACTIVATION_TIMEOUT_SECS"; +const ACTIVATION_TIMEOUT: &str = "D2B_TEST_VM_HARNESS_ACTIVATION_TIMEOUT_SECS"; /// How many boot and teardown cycles to run. -const CYCLES: &str = "D2B_VM_HARNESS_CYCLES"; +const CYCLES: &str = "D2B_TEST_VM_HARNESS_CYCLES"; /// The block-graph node name the boot-time snapshot-capability proof attaches /// its unsnapshottable device under. @@ -103,7 +103,7 @@ fn main() -> ExitCode { // restored. /// The image list the lane's target generated, one guest image per line. -const IMAGES: &str = "D2B_VM_HARNESS_IMAGES"; +const IMAGES: &str = "D2B_TEST_VM_HARNESS_IMAGES"; /// The checks a contributor selected, as the existing selection variables /// carry them: a whitespace- or comma-separated list of check names. const CHECKS: &str = "D2B_VM_CHECK"; diff --git a/packages/d2b-vm-harness/src/diagnostics.py b/packages/d2b-test-vm-harness/src/diagnostics.py similarity index 98% rename from packages/d2b-vm-harness/src/diagnostics.py rename to packages/d2b-test-vm-harness/src/diagnostics.py index b998d4398..00d687b10 100644 --- a/packages/d2b-vm-harness/src/diagnostics.py +++ b/packages/d2b-test-vm-harness/src/diagnostics.py @@ -13,7 +13,7 @@ # # The helpers are diagnostics only: no assertion and no timeout declared # here changes any of them. The lane's guest-control surface -# (`packages/d2b-vm-harness/src/legacy.rs`) owns what `machine.*` means, and +# (`packages/d2b-test-vm-harness/src/legacy.rs`) owns what `machine.*` means, and # the failure path below is what makes a failed check legible - the stage it # was in, the rows it was asserting on, the journal lines that explain them, # and the zone's own account of the row that did not settle. diff --git a/packages/d2b-vm-harness/src/error.rs b/packages/d2b-test-vm-harness/src/error.rs similarity index 100% rename from packages/d2b-vm-harness/src/error.rs rename to packages/d2b-test-vm-harness/src/error.rs diff --git a/packages/d2b-vm-harness/src/guest.rs b/packages/d2b-test-vm-harness/src/guest.rs similarity index 100% rename from packages/d2b-vm-harness/src/guest.rs rename to packages/d2b-test-vm-harness/src/guest.rs diff --git a/packages/d2b-vm-harness/src/host.rs b/packages/d2b-test-vm-harness/src/host.rs similarity index 100% rename from packages/d2b-vm-harness/src/host.rs rename to packages/d2b-test-vm-harness/src/host.rs diff --git a/packages/d2b-vm-harness/src/legacy.rs b/packages/d2b-test-vm-harness/src/legacy.rs similarity index 99% rename from packages/d2b-vm-harness/src/legacy.rs rename to packages/d2b-test-vm-harness/src/legacy.rs index a010dc080..34d609975 100644 --- a/packages/d2b-vm-harness/src/legacy.rs +++ b/packages/d2b-test-vm-harness/src/legacy.rs @@ -100,7 +100,7 @@ const ACCEPT_POLL: Duration = Duration::from_millis(20); /// under, for a lane that wants a specific one. The lane's test target sets /// it to the interpreter declared as a runfile; without it the interpreter is /// resolved from the runfiles tree, and failing that from `PATH`. -const PYTHON: &str = "D2B_VM_HARNESS_PYTHON"; +const PYTHON: &str = "D2B_TEST_VM_HARNESS_PYTHON"; /// The smallest read bound the console is given while it waits for its /// shell. A bound of zero is not a bound at all on a socket: it means wait diff --git a/packages/d2b-vm-harness/src/legacy_bridge.py b/packages/d2b-test-vm-harness/src/legacy_bridge.py similarity index 100% rename from packages/d2b-vm-harness/src/legacy_bridge.py rename to packages/d2b-test-vm-harness/src/legacy_bridge.py diff --git a/packages/d2b-vm-harness/src/lib.rs b/packages/d2b-test-vm-harness/src/lib.rs similarity index 100% rename from packages/d2b-vm-harness/src/lib.rs rename to packages/d2b-test-vm-harness/src/lib.rs diff --git a/packages/d2b-vm-harness/src/manifest.rs b/packages/d2b-test-vm-harness/src/manifest.rs similarity index 100% rename from packages/d2b-vm-harness/src/manifest.rs rename to packages/d2b-test-vm-harness/src/manifest.rs diff --git a/packages/d2b-vm-harness/src/monitor.rs b/packages/d2b-test-vm-harness/src/monitor.rs similarity index 100% rename from packages/d2b-vm-harness/src/monitor.rs rename to packages/d2b-test-vm-harness/src/monitor.rs diff --git a/packages/xtask/src/provider_crate_policy.rs b/packages/xtask/src/provider_crate_policy.rs index 2aae8b4dc..72d338d44 100644 --- a/packages/xtask/src/provider_crate_policy.rs +++ b/packages/xtask/src/provider_crate_policy.rs @@ -8984,7 +8984,7 @@ const COMMITTED_SCOPE: &[CommittedScopeEntry] = &[ reason: "the daemon composition root (and its runtime)" }, CommittedScopeEntry { crate_name: "xtask", class: CommittedScopeClass::Tooling, reason: "the check's own home; every U-unit touches the tooling" }, - CommittedScopeEntry { crate_name: "d2b-vm-harness", class: CommittedScopeClass::Tooling, + CommittedScopeEntry { crate_name: "d2b-test-vm-harness", class: CommittedScopeClass::Tooling, reason: "the host-integration lane's own harness; the check's tooling, booted against a lane-owned guest" }, CommittedScopeEntry { crate_name: "d2b-broker", class: CommittedScopeClass::Broker, reason: "the broker binary and its composition/fixture support crates" }, diff --git a/tests/host-integration/lib.nix b/tests/host-integration/lib.nix index c02787719..79269bb2f 100644 --- a/tests/host-integration/lib.nix +++ b/tests/host-integration/lib.nix @@ -509,14 +509,14 @@ rec { # timeout is passed through unchanged. # # The text itself lives with the lane's own assertion surface, in - # `packages/d2b-vm-harness/src/diagnostics.py`, and is read from there + # `packages/d2b-test-vm-harness/src/diagnostics.py`, and is read from there # rather than kept here. The Bazel lane runs these very same evaluated # scripts, so a check that has not been ported yet reports its failure # through this text under either lane; a second copy of it would be a # second dialect of the same diagnostics, and the two would drift the first # time one of them gained a helper the other did not. fixtureDiagnostics = - builtins.readFile ../../packages/d2b-vm-harness/src/diagnostics.py; + builtins.readFile ../../packages/d2b-test-vm-harness/src/diagnostics.py; # Re-exported so tests can assert against the shared declaration. inherit mkGuestSystem mkRuntimeCloudHypervisorArtifact From 03684ed0b73df640db2ef57ac8417543ec2a5c52 Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sat, 26 Sep 2026 21:43:27 -0700 Subject: [PATCH 09/51] fix(vm): re-attach a restored guest's devices in launch order ActiveGuest::restore detached and re-attached every writable device in QEMU's query-block order, which is not the order the launch attached them. The kernel names a block device by creation order, so on every restore a two-disk member's root disk and its option-list state disk swapped /dev/vda and /dev/vdb. The guest's volume-local markers anchor a volume root by (device, inode) and fail closed on mismatch, so after any restore every marker disagreed with the live tree - RootReplaced, then volume-layout-effect-failed forever, the Volume row never reaching Ready, and the waits timing out at 180s. The fresh boot wrote the markers while /var/lib/d2b was /dev/vda; the restored boot saw the same tree at /dev/vdb. daemon-smoke carried the same swap in its console log and passed only because it never waits on a Volume, and single-disk members cannot swap at all, which is why two checks failed while the lane looked mostly green. Restore now ranks each device by SnapshotPoint::launch_rank - declared drives in manifest order, then the node's own option drives - and orders them with reattachment_order, a stable sort, so detach, drop, freeze, overlay and re-attach all happen in that one order. Two unit tests pin the ranking and the seam against a deliberately mismatched report. The equivalence marker now reports each /sys/block/vd* name and size, so the gate compares the guest's device identity between a fresh boot and a boot from the restored disk. That is what makes a regression of this defect fail at the gate in milliseconds instead of as a 180s wait; the marker grew from 307-319 bytes to 350-375 and all eleven members matched in two post-fix runs. What the gate still does not prove is written in its doc comment: it compares the configuration surface as a booted guest reports it, not RAM or in-flight state, because the restore is disk-only. The stale monitor test fake no longer drops device_del's ack, so wait_for_event no longer blocks on an event it already consumed. The clear() it depends on is untouched - discarding events that predate the wait is what stopped a restore matching a stale DEVICE_DELETED. virtiofsd-volume-runtime goes from a 240.9s failure to 68-110s passing. device-worker-launch still fails, deterministically, at the same TPM assertion with its Volume now Ready - a second, independent cause this commit does not address - and one nested-guest member failed in one of two runs. Neither is papered over here. --- .../src/bin/d2b-test-vm-harness.rs | 18 ++ packages/d2b-test-vm-harness/src/guest.rs | 158 +++++++++++++++++- packages/d2b-test-vm-harness/src/monitor.rs | 53 ++++++ 3 files changed, 226 insertions(+), 3 deletions(-) diff --git a/packages/d2b-test-vm-harness/src/bin/d2b-test-vm-harness.rs b/packages/d2b-test-vm-harness/src/bin/d2b-test-vm-harness.rs index 2924085b8..4b241659c 100644 --- a/packages/d2b-test-vm-harness/src/bin/d2b-test-vm-harness.rs +++ b/packages/d2b-test-vm-harness/src/bin/d2b-test-vm-harness.rs @@ -177,6 +177,23 @@ struct CheckResult { /// mounted where it mounted it, and the guest's random pool is out of its /// initialising state. A restore that drops any of those is a restore that /// would break a check, and that is the failure worth catching. +/// +/// The guest's block devices are among them, by the name the guest's kernel +/// gave each one and its size: the kernel names a disk by the order the +/// devices were created, so a restore that put this member's disks back in +/// the emulator's report order rather than the launch's would hand the guest +/// a different `/dev/vda` than the fresh boot it is compared with - and every +/// fact a check anchors to a device identity (a volume-local marker anchors +/// its root by `(device, inode)`, and an inode means nothing without the +/// device it was read from) would be read against the wrong disk. +/// +/// What this gate does *not* prove, and must not be read as proving: it +/// observes the guest's configuration surface as a booted guest reports it, +/// so it says nothing about RAM or in-flight state. The restore is disk-only - +/// an internal snapshot is refused while a VirtFS export is mounted in the +/// guest, and every lane guest mounts one - so a guest restored onto its disk +/// and reset is compared against the fresh boot at the same surface, not +/// across the memory each had. const EQUIVALENCE_MARKER: &str = r#" import subprocess import sys @@ -208,6 +225,7 @@ report("hostTools", "ls /run/d2b-host-tools 2>/dev/null || ls /nix/var/nix/profi report("activation", "systemctl show d2b-lane-activation --property=ActiveState --property=Result --no-pager") report("crng", "journalctl -b --no-pager -o cat | grep -c 'crng init done' || true") report("backdoor", "systemctl is-active backdoor.service") +report("blockDevices", "for device in /sys/block/vd*; do printf '%s=%s ' \"$(basename \"$device\")\" \"$(cat \"$device/size\")\"; done; echo") print("d2b-lane-marker complete") "#; diff --git a/packages/d2b-test-vm-harness/src/guest.rs b/packages/d2b-test-vm-harness/src/guest.rs index 631de7c93..6ee2c080c 100644 --- a/packages/d2b-test-vm-harness/src/guest.rs +++ b/packages/d2b-test-vm-harness/src/guest.rs @@ -798,6 +798,54 @@ impl SnapshotPoint { ))) } + /// Where one device sits in the order the launch attached it. + /// + /// The drives the node declared come first, in the order it declared + /// them, then the drives the node attached through its own option list. + /// That is the order the launch creates the devices in, and the guest's + /// kernel names a block device by the order the devices are created: the + /// same fleet attached in a different order is a different set of + /// `/dev/vd*` names. A device the launch never attached sorts after both, + /// and the sort that consumes this ranking is stable, so such a device + /// keeps the emulator's own order among its peers. + fn launch_rank(&self, target: &RestoreTarget) -> usize { + if let Some(index) = self + .declared + .iter() + .position(|(_, declared)| declared.label == target.label) + { + return index; + } + if let Some(index) = self + .ephemeral + .iter() + .position(|(_, ephemeral)| ephemeral.label == target.label) + { + return self.declared.len() + index; + } + usize::MAX + } + + /// The devices a restore puts back, in the order the launch attached + /// them. + /// + /// The devices arrive as the emulator reported them - `query-block` + /// order, which is not the launch's - and leave in the order the launch + /// created them, which is the order the guest's kernel names its disks + /// in. The sort is stable, so two devices the launch never attached keep + /// the emulator's order among themselves. + pub fn reattachment_order( + &self, + devices: Vec<(BlockDevice, RestoreTarget)>, + ) -> Vec<(BlockDevice, RestoreTarget)> { + let mut ranked: Vec<(usize, (BlockDevice, RestoreTarget))> = devices + .into_iter() + .map(|device| (self.launch_rank(&device.1), device)) + .collect(); + ranked.sort_by_key(|(rank, _)| *rank); + ranked.into_iter().map(|(_, device)| device).collect() + } + /// The layer one device's next writes go into. pub fn overlay_for(&self, target: &RestoreTarget) -> PathBuf { self.work_dir.join(format!( @@ -969,14 +1017,34 @@ impl ActiveGuest { /// writes a layer and a node name of its own, because the emulator refuses /// to open a node twice or to write over a file that holds an open /// image. + /// + /// The devices go back in the order the launch attached them, not in the + /// order the emulator reports them. The guest's kernel names a block + /// device by the order the devices are created, so a fleet re-attached in + /// a different order hands the guest a different `/dev/vda`: a check that + /// recorded a device identity - a volume-local marker anchors its root by + /// `(device, inode)`, and an inode is only meaningful on the device it + /// was read from - would then be run against a guest whose disks are + /// named differently than they were on the fresh boot it is compared + /// with. pub fn restore(&mut self, point: &mut SnapshotPoint) -> Result { let started = Instant::now(); let devices = self.rotatable_devices()?; let mut targets = Vec::with_capacity(devices.len()); - for device in &devices { - let target = point.target_for(self.monitor_mut()?, device)?.clone(); + for device in devices { + let target = point.target_for(self.monitor_mut()?, &device)?.clone(); + targets.push((device, target)); + } + // The devices go back on in the order the launch attached them: the + // guest's kernel names a block device by the order the devices are + // created, so the emulator's own report order hands it a different + // `/dev/vda`, and a check that recorded a device identity - a + // volume-local marker anchors its root by `(device, inode)` - would + // then run against a guest whose disks are named differently than on + // the fresh boot it is compared with. + let targets = point.reattachment_order(targets); + for (device, _) in &targets { self.monitor_mut()?.detach_device(device, DETACH_BOUND)?; - targets.push((device.clone(), target)); } point.next_layer(); for (device, _) in &targets { @@ -1743,6 +1811,90 @@ mod tests { ); } + #[test] + fn a_restore_orders_the_devices_the_launch_declared_them() { + // The launch attaches the node's declared drives first and the drives + // the node attached through its own option list after, and the + // guest's kernel names a block device by the order the devices are + // created. A restore that re-attached in the emulator's own report + // order instead handed the guest a different `/dev/vda` - the root + // disk and the state disk swapped - and the volume-local marker + // anchored to the root's `(device, inode)` then failed closed for the + // rest of the run. + let mut declared = manifest(3, 3072, &[]); + declared.extra_options = vec![ + "-drive".to_owned(), + "file=/nix/store/0123456789bcdefghijklmnopqrstuv-d2b-state.img,format=raw,if=virtio,snapshot=on" + .to_owned(), + ]; + let spec = GuestSpec::new( + declared, + "/run/lane/image", + "/nix/store/qemu/bin/qemu-kvm", + "/run/lane/work", + "device-worker-launch", + ); + let point = SnapshotPoint::new(&spec).expect("the member's snapshot reads out of its spec"); + + assert_eq!(point.declared.len(), 1, "the node declared one drive"); + assert_eq!(point.ephemeral.len(), 1, "the node attached one state disk"); + let root = &point.declared[0].1; + let state = &point.ephemeral[0].1; + assert_eq!(point.launch_rank(root), 0, "the declared drive launches first"); + assert_eq!( + point.launch_rank(state), + point.declared.len(), + "the node's own option drives follow the declared ones" + ); + assert!( + point.launch_rank(root) < point.launch_rank(state), + "the root disk is /dev/vda on the launch and must be again on a restore" + ); + } + + #[test] + fn a_restore_puts_the_devices_back_in_the_order_the_launch_attached_them() { + // The report comes back in `query-block` order - the state disk the + // node attached through its own option list first, the root drive the + // launch declared second - and the restore has to hand the emulator + // the opposite order, because that is the order the fresh boot + // created the devices in and therefore the order the guest's kernel + // named them in. Under the defect this is the swap that made every + // volume-local marker mismatch its own root. + let mut declared = manifest(3, 3072, &[]); + declared.extra_options = vec![ + "-drive".to_owned(), + "file=/nix/store/0123456789bcdefghijklmnopqrstuv-d2b-state.img,format=raw,if=virtio,snapshot=on" + .to_owned(), + ]; + let point = SnapshotPoint::new(&spec(declared)).expect("the member's snapshot reads out of its spec"); + let device = |id: &str, node: &str| BlockDevice { + id: id.to_owned(), + qdev: format!("/machine/peripheral-anon/{id}"), + node: node.to_owned(), + file: format!("/run/lane/{id}.qcow2"), + format: "qcow2".to_owned(), + read_only: false, + rotatable: true, + }; + let declared_target = point.declared[0].1.clone(); + let (ephemeral_node, ephemeral) = point.ephemeral[0].clone(); + let reported = vec![ + (device("state", &ephemeral_node), ephemeral), + (device("root", &point.declared[0].0), declared_target.clone()), + ]; + let ordered = point.reattachment_order(reported); + assert_eq!( + ordered.iter().map(|(device, _)| device.id.as_str()).collect::>(), + vec!["root", "state"], + "the launched-first drive is attached first whatever order the report came in" + ); + assert_eq!( + ordered[0].1, declared_target, + "the first device back on is the node's own declared drive" + ); + } + #[test] fn the_accelerator_is_selected_and_never_falls_back_to_emulation() { let argv = argv(&spec(manifest(3, 3072, &[]))); diff --git a/packages/d2b-test-vm-harness/src/monitor.rs b/packages/d2b-test-vm-harness/src/monitor.rs index 94364b748..0b489700f 100644 --- a/packages/d2b-test-vm-harness/src/monitor.rs +++ b/packages/d2b-test-vm-harness/src/monitor.rs @@ -590,6 +590,13 @@ mod tests { /// command the lane writes: a reply the canned side sends proves nothing /// about which node an overlay was taken of, or how a device was /// re-attached. + /// + /// A request that outruns the list is not a failure, it is a block: the + /// lane reads the socket for a line the canned side will never write. So + /// every command the code under test writes needs a reply of its own, in + /// the order the lane reads them - its handshake's, then one per command, + /// with any event the command is supposed to deliver after the reply that + /// command's own `execute` consumes. fn monitor_answering(replies: &'static [&'static str]) -> Wired { let (lane_end, emulator_end) = UnixStream::pair().expect("a socket pair"); let requests: Arc>> = Arc::new(Mutex::new(Vec::new())); @@ -796,7 +803,14 @@ mod tests { #[test] fn detaching_a_device_waits_for_the_emulator_to_finish_it() { + // The replies in the order the lane reads them: the handshake's, the + // detach's own ack, and then the poll that carries the event. The ack + // is not optional - without it `device_del` reads the event as its + // own reply across an empty `return` and the wait that follows has + // nothing left to poll, which is a test that blocks rather than + // fails. let mut monitor = monitor_answering(&[ + r#"{"return": {}}"#, r#"{"return": {}}"#, r#"{"event": "DEVICE_DELETED", "data": {"device": "/machine/peripheral-anon/device[4]"}}"#, r#"{"return": {"status": "running"}}"#, @@ -815,6 +829,45 @@ mod tests { .expect("the event arrives with the next command's reply"); } + #[test] + fn an_event_a_previous_detach_left_behind_does_not_confirm_this_one() { + // The emulator's events are a stream, and the DEVICE_DELETED a + // previous detach left on it arrives while this detach's own reply is + // on its way. The wait matches an event by its *name* - a + // DEVICE_DELETED is a DEVICE_DELETED - so that stale one would satisfy + // it without this device having moved at all, and the restore after it + // would re-attach a device the emulator still holds. Discarding what + // was on the stream before the wait began is what keeps them apart, + // and this is the case that pins it: the wait sees the stale event + // read, drops it, and goes on to poll until the bound trips. + let mut monitor = monitor_answering(&[ + r#"{"return": {}}"#, + r#"{"event": "DEVICE_DELETED", "data": {"device": "/machine/peripheral-anon/device[9]"}}"#, + r#"{"return": {}}"#, + r#"{"return": {"status": "running"}}"#, + r#"{"return": {"status": "running"}}"#, + r#"{"return": {"status": "running"}}"#, + ]); + let device = BlockDevice { + id: "lane_drive_0".to_owned(), + qdev: "/machine/peripheral-anon/device[4]".to_owned(), + node: "lane_root.overlay".to_owned(), + file: "/run/lane/lane_root.overlay.qcow2".to_owned(), + format: "qcow2".to_owned(), + read_only: false, + rotatable: true, + }; + let error = monitor + .detach_device(&device, Duration::from_millis(300)) + .expect_err("another device's event is not confirmation of this unplug"); + let rendered = error.to_string(); + assert!(rendered.contains("DEVICE_DELETED"), "{rendered}"); + assert!( + rendered.contains("device[4]"), + "the failure names the device that did not move: {rendered}" + ); + } + #[test] fn a_detach_the_emulator_never_reports_fails_with_the_device_named() { // The handshake's reply, the detach's, and one per poll the bound From ac31be73c89075f6e857d4a3505d5d106da12f1b Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sat, 26 Sep 2026 22:06:09 -0700 Subject: [PATCH 10/51] refactor(vm): assert daemon-smoke in Rust and retire its fixture The first of eleven ports, and the pattern the rest follow. The check's twenty-eight guest assertions move from the fixture's testScript into packages/d2b-test-vm-harness/src/checks/daemon_smoke.rs, in the same order, with the same commands and the same 30s and 180s bounds: 17 succeed, 1 wait_for_file, 4 diag_unit, 5 stage, plus the acceptance census as a set comparison and the two restart/cgroup-survival steps. GuestControl grew the stage and diag_unit primitives the fixture's diagnostics prelude used, so the log reads the way it read. Deleting the fixture is what forced the design. A fixture carried both a check's guest configuration and its testScript, and the lane identified a check by the check.py the image wrote for it - so a check whose assertions are Rust had no way to appear in the lane at all. The manifest's CheckRecord now carries an `assertions` field naming which side holds a check's assertions, the image action writes it, and the harness dispatches on it: Rust goes to the crate's table of ported checks, Python takes the unchanged check.py path. Both directions fail closed - an image that claims Rust with no module behind it fails the lane by name, and a manifest that claims Python with no script still fails on the read - so a half-finished port is a red lane, never a check that quietly ran nothing. daemon-smoke's guest configuration moves to host-integration-node.nix, next to the node shapes it was written against, so deleting the fixture loses no machine size, drive layout or device option. Counts the port must satisfy: 17 succeed, 1 wait_for_file, 4 diag_unit, 5 stage, before and after. The lane's own plan carried 78 em-dashes and failed the source-hygiene gate that make check-tier0 runs, which no other tracked document in the repository does; they are now the ASCII form its superseded sibling uses. --- bazel/checks/vm/BUILD.bazel | 27 +- bazel/checks/vm/defs.bzl | 6 + changelog.d/bazel-owned-ported-check.md | 13 + ...actor-bazel-owned-host-integration-plan.md | 110 +++--- nix/test-support/guest-image.nix | 88 ++++- nix/test-support/host-integration-node.nix | 56 +++ .../src/bin/d2b-test-vm-harness.rs | 36 +- .../src/checks/daemon_smoke.rs | 176 +++++++++ .../d2b-test-vm-harness/src/checks/mod.rs | 50 +++ packages/d2b-test-vm-harness/src/legacy.rs | 342 ++++++++++++++++-- packages/d2b-test-vm-harness/src/lib.rs | 5 +- packages/d2b-test-vm-harness/src/manifest.rs | 31 +- tests/host-integration/daemon-smoke.nix | 128 ------- 13 files changed, 826 insertions(+), 242 deletions(-) create mode 100644 changelog.d/bazel-owned-ported-check.md create mode 100644 packages/d2b-test-vm-harness/src/checks/daemon_smoke.rs create mode 100644 packages/d2b-test-vm-harness/src/checks/mod.rs delete mode 100644 tests/host-integration/daemon-smoke.nix diff --git a/bazel/checks/vm/BUILD.bazel b/bazel/checks/vm/BUILD.bazel index 46f7c2c4e..da360f408 100644 --- a/bazel/checks/vm/BUILD.bazel +++ b/bazel/checks/vm/BUILD.bazel @@ -107,11 +107,13 @@ guest_image( tags = _IMAGE_TAGS, ) -# One guest per check, built from that check's own fixture. +# One guest per check, built from that check's own fixture - or, for a check +# whose assertions have moved to the lane's own Rust, from the node the +# reusable node module declares for it by name. # -# The list is the lane's check inventory and nothing else: each entry names -# the fixture the guest is read out of, and the guest's memory, vCPU count, -# disk, drives, device options and assertions all come out of that file +# The list is the lane's check inventory and nothing else: an unported entry +# names the fixture the guest is read out of, and the guest's memory, vCPU +# count, disk, drives, device options and assertions all come out of that file # during the evaluation. Three of these checks boot a plain NixOS node with # no d2b daemon host at all, two turn nftables on, two install acceptance # artifacts, and two boot a nested guest on the writable-store shape; that @@ -135,10 +137,25 @@ _CHECKS = [ "wayland-proxy", ] +# The checks that assert in Rust. Each one's fixture is retired in the same +# change as its port, so there is no fixture left to read its guest out of: +# the node it boots is declared with the reusable nodes +# (`nix/test-support/host-integration-node.nix`), and the image carries no +# `check.py` - the lane runs the Rust module the check is declared in. Moving +# a check here and deleting its fixture is the whole of what a port costs this +# list; the image is built from the check's name, which is the shape its node +# is declared under. +_PORTED_CHECKS = [ + "daemon-smoke", +] + [ guest_image( name = "guest_image_" + check, - check = "tests/host-integration/%s.nix" % check, + # An unported check's guest and assertions are read out of its own + # fixture; a ported check has no fixture left, and the evaluation finds + # the node the check boots in the reusable node module instead. + check = "" if check in _PORTED_CHECKS else "tests/host-integration/%s.nix" % check, srcs = _GUEST_SOURCES, cloud_hypervisor_controller = _CONTROLLER, flake = "//:flake.nix", diff --git a/bazel/checks/vm/defs.bzl b/bazel/checks/vm/defs.bzl index 61d2ca412..d101f4e91 100644 --- a/bazel/checks/vm/defs.bzl +++ b/bazel/checks/vm/defs.bzl @@ -254,6 +254,12 @@ guest_image = rule( # layout and its assertions are all read out of that file during the # evaluation, so the lane carries no list of which check wants which # guest: the only place a check's guest is declared is the check. + # + # Left empty for a check whose assertions are the lane's own Rust: its + # fixture is gone, so there is nothing to read a guest out of, and the + # evaluation reads the check's node from the reusable node module's + # table of ported checks instead - by the `node_shape` this target + # declares, which is the check's own name. "check": attr.string(), # The name this guest reports on its console. With a `check` it is # the check's own name, which is what makes a launcher that booted diff --git a/changelog.d/bazel-owned-ported-check.md b/changelog.d/bazel-owned-ported-check.md new file mode 100644 index 000000000..20b8f1971 --- /dev/null +++ b/changelog.d/bazel-owned-ported-check.md @@ -0,0 +1,13 @@ +### Added + +- The host-integration lane can now run a check whose assertions are the lane's own Rust. A ported check is a function over the lane's guest-control surface (`packages/d2b-test-vm-harness/src/checks/`), declared in that crate's table of ported checks, and its guest image carries no `check.py`: the image's manifest says which side of the lane carries the check's assertions, so a check that has been ported and a check that has not are both ordinary checks to the pool, the inventory and the report. A ported check reports under its own result entry with its own diagnostics, and a check whose image says it asserts in Rust and which has no module in the lane is a lane failure rather than a check that quietly does not run. +- The guest-control surface gained the diagnostics primitives a ported check asserts through, in the same one place the fixtures' Python prelude provides them: `stage` (the phase a failure names), `diag` (a labelled diagnostic command that is never fatal), `diag_unit` (a unit wait that prints the unit's status, its journal and the zone's debug dump when it does not settle) and `diag_wait` (the same for a command wait, with the wait and the row set named). A ported check's failure therefore prints the stage it was in, the rows it was asserting on, and the lines that explain them, in the prelude's own order and wording. +- The lane tolerates a check with no fixture end to end. The guest-image action accepts a check declared by name (`ported_check`) as well as one declared by its fixture, and reads the guest such a check boots out of the reusable node module's table of ported checks (`nix/test-support/host-integration-node.nix`); the image is built from that node exactly as a fixture's guest was, and keeps the check's driver name as a filter alias. + +### Changed + +- Ported `daemon-smoke` to Rust: its assertions moved unchanged into `packages/d2b-test-vm-harness/src/checks/daemon_smoke.rs` - the same stages, the same commands, the same 30s and 180s bounds, in the same order - and the guest it boots (the reusable daemon node plus `jq`) is now declared in `nix/test-support/host-integration-node.nix` rather than in the fixture. + +### Removed + +- Removed `tests/host-integration/daemon-smoke.nix`. The check it declared is the same check, with the same guest and the same result in the lane; what is gone is the fixture the assertions used to be written in. diff --git a/docs/plans/2026-09-26-0021-refactor-bazel-owned-host-integration-plan.md b/docs/plans/2026-09-26-0021-refactor-bazel-owned-host-integration-plan.md index 61ce199b3..ea764d13f 100644 --- a/docs/plans/2026-09-26-0021-refactor-bazel-owned-host-integration-plan.md +++ b/docs/plans/2026-09-26-0021-refactor-bazel-owned-host-integration-plan.md @@ -15,8 +15,8 @@ execution: code - **Objective:** The d2b host-integration lane runs as Bazel tests, so a contributor gets check selection, results in the Bazel graph, and a faster local loop, and the repository stops maintaining two build systems for one test tier. - **Means:** Bazel owns the lane end to end; nix is reduced to a hermetic build action that produces the guest image. -- **Product authority:** This Product Contract owns behavior and scope. `tests/AGENTS.md` owns the type-10 classification; root `AGENTS.md` owns the profile and changelog rules. `docs/plans/2026-08-24-001-refactor-bazel-backed-host-integration-binaries-plan.md` is superseded on its R3, which reserved VM orchestration to nix. The cutover amends the repository instruction sentences that R3 reverses — the host-lane binary-injection rule in root `AGENTS.md`, the type-10 tier rows and host-lane bundle handoff in `tests/AGENTS.md`, and the handoff descriptions in `docs/contributing/critical-subsystems.md`, `docs/contributing/gates-and-lints.md`, `docs/reference/compatibility.md`, and `docs/reference/support-matrix.md` — in the same change that removes the handoff. The heavy-gate rule those files also describe is not carried forward: the guard was deleted and R16 does not require it. -- **Product Contract preservation:** changed at planning time — R3, R5, R6, R7, R9, R16, R17, F1, AE2, AE10. The two user-directed changes are the single lane test target and dropping the heavy-gate semaphore; the rest are research corrections to a snapshot-capability premise, a missing host precondition, the disk layout those two assume, and two contradictions a document review found between requirements that planning-time evidence resolved. +- **Product authority:** This Product Contract owns behavior and scope. `tests/AGENTS.md` owns the type-10 classification; root `AGENTS.md` owns the profile and changelog rules. `docs/plans/2026-08-24-001-refactor-bazel-backed-host-integration-binaries-plan.md` is superseded on its R3, which reserved VM orchestration to nix. The cutover amends the repository instruction sentences that R3 reverses - the host-lane binary-injection rule in root `AGENTS.md`, the type-10 tier rows and host-lane bundle handoff in `tests/AGENTS.md`, and the handoff descriptions in `docs/contributing/critical-subsystems.md`, `docs/contributing/gates-and-lints.md`, `docs/reference/compatibility.md`, and `docs/reference/support-matrix.md` - in the same change that removes the handoff. The heavy-gate rule those files also describe is not carried forward: the guard was deleted and R16 does not require it. +- **Product Contract preservation:** changed at planning time - R3, R5, R6, R7, R9, R16, R17, F1, AE2, AE10. The two user-directed changes are the single lane test target and dropping the heavy-gate semaphore; the rest are research corrections to a snapshot-capability premise, a missing host precondition, the disk layout those two assume, and two contradictions a document review found between requirements that planning-time evidence resolved. - **Execution profile:** Deep, local-only, delivered as a port of one check at a time rather than a single cutover. Within the port the branch stays green because each unported check keeps executing its existing assertions. - **Stop conditions:** Stop if the guest's attached writable devices prove not snapshot-capable, before the pool is built; if a restored run costs at least as much as a fresh boot for the same check; or, at the point in the cutover where the deferred check is retired, if no retained check is found to carry the host isolation from Gateway relay credentials it asserts. @@ -50,7 +50,7 @@ The cost of the carve-out is structural rather than episodic. The lane has no Ba - R7. The lane must require `/dev/kvm`, nested virtualization, and nested-state save support on the host; a host missing any of them must stop with a clear message instead of falling back to emulation. - R8. The pool's aggregate guest footprint must stay within a stated host budget covering memory, vCPU count, and the lane working directory, sized per guest shape with the pool size derived during Planning; checks assigned to one guest run sequentially while the pool itself runs concurrently. - R9. A contributor must be able to run one named check by filtering the lane target, without booting the checks that were not selected. -- R10. The lane must reproduce each check's emulator invocation — its memory, vCPU count, disk size, the drive layout including the writable-store root drive, and any per-check device options such as the vsock device — rather than booting a single uniform guest shape. +- R10. The lane must reproduce each check's emulator invocation - its memory, vCPU count, disk size, the drive layout including the writable-store root drive, and any per-check device options such as the vsock device - rather than booting a single uniform guest shape. **Assertion layer** @@ -71,14 +71,14 @@ The cost of the carve-out is structural rather than episodic. The lane has no Ba ### Key Decisions -- **Bazel owns the lane end to end.** (session-settled: user-directed — chosen over wrapping the existing nix lane and over consuming a prebuilt guest from outside the lane: one build system, not two.) Governs R1, R4, R14. -- **Nix is reduced to a hermetic guest-image build action.** (session-settled: user-directed — chosen over fetching a published guest image: the guest has to track source changes inside the Bazel graph.) Governs R1, R2, R3. -- **The lane stays contributor-local.** (session-settled: user-directed — chosen over a required PR gate on BuildBuddy: the lane is a local pre-PR surface, which is also what makes the virtualization precondition assertable rather than negotiable.) Governs R7, R8, R18. -- **A small pool of guests is reused through snapshot and restore.** (session-settled: user-directed — chosen over a guest per check and over a single sequential guest: cut total boots from one-per-check to one-per-pool-member while keeping a parallel wave.) Governs R5, R6, R8. -- **Virtualization is a precondition, not a fallback.** (session-settled: user-directed — the lane runs on the contributor's own machine, so emulation is a silent degradation rather than a needed capability.) Governs R7. -- **Assertions port to Rust one check at a time.** (session-settled: user-directed — chosen over keeping the Python testScripts permanently and over porting all eleven in one cutover: the lane is Bazel-native from day one without a coverage cliff.) Governs R11, R12, R13. -- **The pool runs concurrently; one guest runs its checks sequentially.** (session-settled: user-directed — chosen over a single sequential guest: snapshot and restore mutates one guest, so parallelism has to come from the pool.) Governs R8. -- **A check that cannot be snapshot-restored gets a single-use guest.** (session-settled: user-directed — chosen over leaving it on the legacy lane and over deciding at a de-risking spike: its coverage is preserved and the migration never blocks on proving snapshot safety.) Governs R5. +- **Bazel owns the lane end to end.** (session-settled: user-directed - chosen over wrapping the existing nix lane and over consuming a prebuilt guest from outside the lane: one build system, not two.) Governs R1, R4, R14. +- **Nix is reduced to a hermetic guest-image build action.** (session-settled: user-directed - chosen over fetching a published guest image: the guest has to track source changes inside the Bazel graph.) Governs R1, R2, R3. +- **The lane stays contributor-local.** (session-settled: user-directed - chosen over a required PR gate on BuildBuddy: the lane is a local pre-PR surface, which is also what makes the virtualization precondition assertable rather than negotiable.) Governs R7, R8, R18. +- **A small pool of guests is reused through snapshot and restore.** (session-settled: user-directed - chosen over a guest per check and over a single sequential guest: cut total boots from one-per-check to one-per-pool-member while keeping a parallel wave.) Governs R5, R6, R8. +- **Virtualization is a precondition, not a fallback.** (session-settled: user-directed - the lane runs on the contributor's own machine, so emulation is a silent degradation rather than a needed capability.) Governs R7. +- **Assertions port to Rust one check at a time.** (session-settled: user-directed - chosen over keeping the Python testScripts permanently and over porting all eleven in one cutover: the lane is Bazel-native from day one without a coverage cliff.) Governs R11, R12, R13. +- **The pool runs concurrently; one guest runs its checks sequentially.** (session-settled: user-directed - chosen over a single sequential guest: snapshot and restore mutates one guest, so parallelism has to come from the pool.) Governs R8. +- **A check that cannot be snapshot-restored gets a single-use guest.** (session-settled: user-directed - chosen over leaving it on the legacy lane and over deciding at a de-risking spike: its coverage is preserved and the migration never blocks on proving snapshot safety.) Governs R5. ### How This Work Fits Together @@ -86,10 +86,10 @@ The cost of the carve-out is structural rather than episodic. The lane has no Ba This plan covers the type-10 VM lane only. That split is the current understanding, not a committed roadmap. -- Remote execution of the lane on BuildBuddy, an executor pool, or in CI as a required gate — *Deferred*: a later plan may take it up once the lane has a stable local run and the virtualization precondition is settled. -- The type-9 container lane and the live-host scripts — *Can proceed independently of* this plan; they share the Make facade but not the VM harness. -- Porting the checks to Rust — in scope for this plan and sequenced inside it: the boot layer lands first, each check then ports per F2, and cutover fires when the last check asserts in Rust. -- Bumping the pinned nixpkgs Bazel ruleset — *Can proceed independently of* this plan; this work stays inside the existing pin. +- Remote execution of the lane on BuildBuddy, an executor pool, or in CI as a required gate - *Deferred*: a later plan may take it up once the lane has a stable local run and the virtualization precondition is settled. +- The type-9 container lane and the live-host scripts - *Can proceed independently of* this plan; they share the Make facade but not the VM harness. +- Porting the checks to Rust - in scope for this plan and sequenced inside it: the boot layer lands first, each check then ports per F2, and cutover fires when the last check asserts in Rust. +- Bumping the pinned nixpkgs Bazel ruleset - *Can proceed independently of* this plan; this work stays inside the existing pin. ### Key Flows @@ -189,7 +189,7 @@ flowchart TB - AE10. A guest built from the current disk layout - **Covers:** R5, R6. - - **Given:** a guest whose attached writable devices — the root drive and the shared state disk — are in a snapshot-capable configuration, and whose in-guest writable-store images are files inside that guest rather than attached devices. + - **Given:** a guest whose attached writable devices - the root drive and the shared state disk - are in a snapshot-capable configuration, and whose in-guest writable-store images are files inside that guest rather than attached devices. - **When:** the lane boots that guest and restores it between checks. - **Then:** the guest boots and restores, and the suite proceeds on the reused pool rather than the single-use tier. @@ -221,7 +221,7 @@ flowchart TB ### Dependencies / Assumptions - The contributor's host provides `/dev/kvm` with nested virtualization and nested-state save support, which the nested cloud-hypervisor check requires of the guest and which snapshotting an outer guest requires. -- The guest-image action needs a nix build that realizes the system closure into a Bazel-declared output from label inputs. The repository's existing nix-inside-Bazel test harness does not provide this — it runs against the host store over a working-tree flake reference inside an uncacheable, unsandboxed test action — and no nix Bazel ruleset version provides a cacheable nix-build action, so the action is authored here. The substitute reachability the current recipe's cache preflight and closure upload provide must be carried into the action, and the flake must arrive as a declared input rather than a working-tree reference, before R1 can hold. +- The guest-image action needs a nix build that realizes the system closure into a Bazel-declared output from label inputs. The repository's existing nix-inside-Bazel test harness does not provide this - it runs against the host store over a working-tree flake reference inside an uncacheable, unsandboxed test action - and no nix Bazel ruleset version provides a cacheable nix-build action, so the action is authored here. The substitute reachability the current recipe's cache preflight and closure upload provide must be carried into the action, and the flake must arrive as a declared input rather than a working-tree reference, before R1 can hold. - The guest's attached writable devices are already in a snapshot-capable configuration: the root drive is qcow2 and the shared state disk is attached with a writable overlay. This is a property to verify, not a conversion to perform; the two writable-store ext4 images are files inside the guest, not attached devices, and a single raw attached device would fail the snapshot outright. ### Outstanding Questions @@ -234,22 +234,22 @@ flowchart TB ### Sources / Research -- `tests/AGENTS.md:12-15, 23, 68` — the type-10 classification and the "push coverage down toward type 1" rule. -- Root `AGENTS.md:209-215, 229-230` — the no-profile-override rule that R18 answers, and the existing requirement that the guest consume Bazel-built binaries. -- `Makefile:8-10, 11-22, 158-159, 178-361` — the one-class-per-target dispatcher invariant, the lane's membership in the local-target class, the canned Bazel alias, and the serial default at `:295`. -- `flake.nix:606-681` — the `vmChecks` output, the non-recursive fixture discovery, and the two `builtins.getEnv` bundle reads at `:610-612`. -- `tests/host-integration/lib.nix:519-641, 653-796` — the shared node configuration and the diagnostics prelude the Rust assertion layer must reproduce, and the split between reusable configuration and driver-coupled diagnostics. -- `tests/host-integration/lib.nix:529-536, 627-630` — the shared state disk, attached with a writable overlay. -- `runtime-cloud-hypervisor-guest-preflight.nix:637-644` — the in-guest virtualization and vhost-net assertions behind the single-use tier. -- `bazel/checks/fixtures/defs.bzl:1-63` — the one existing Starlark rule that runs nix as a cacheable build action; the model for the guest-image action. -- `bazel/checks/nix/defs.bzl:3-9, 81-125` — the existing nix-inside-Bazel test harness, whose tags establish the non-cacheable convention the lane follows. -- `tests/unit/meta/rust-main-packages-suite-guard.sh:150-173` — the guard that force-registers any crate carrying a test aggregate and forbids positive tags on such aggregates; the reason the lane's targets live outside the main package suite. -- `nixos-modules/base.nix:67-68` and `nixos-modules/lib.nix:322, 421-429` — sshd enabled by default in the guest base, the guest's ssh capability, and the repository's existing QMP readiness vocabulary. -- `CHANGELOG.md:1086-1088` and commit `2c2f8149b` — the deletion of the heavy-gate orchestration, which R16 and four documentation sites previously described as current. -- `docs/plans/2026-08-24-001-refactor-bazel-backed-host-integration-binaries-plan.md` — the superseded plan; its R3 and its "Bazel does not become the scheduler for the NixOS VM test" boundary are what this contract reverses. -- `docs/plans/2026-08-19-002-refactor-build-test-ownership-cleanup-plan.md:266, 269-270` — the direction that tests expose Bazel only and receive binaries from Bazel rather than building at test runtime. +- `tests/AGENTS.md:12-15, 23, 68` - the type-10 classification and the "push coverage down toward type 1" rule. +- Root `AGENTS.md:209-215, 229-230` - the no-profile-override rule that R18 answers, and the existing requirement that the guest consume Bazel-built binaries. +- `Makefile:8-10, 11-22, 158-159, 178-361` - the one-class-per-target dispatcher invariant, the lane's membership in the local-target class, the canned Bazel alias, and the serial default at `:295`. +- `flake.nix:606-681` - the `vmChecks` output, the non-recursive fixture discovery, and the two `builtins.getEnv` bundle reads at `:610-612`. +- `tests/host-integration/lib.nix:519-641, 653-796` - the shared node configuration and the diagnostics prelude the Rust assertion layer must reproduce, and the split between reusable configuration and driver-coupled diagnostics. +- `tests/host-integration/lib.nix:529-536, 627-630` - the shared state disk, attached with a writable overlay. +- `runtime-cloud-hypervisor-guest-preflight.nix:637-644` - the in-guest virtualization and vhost-net assertions behind the single-use tier. +- `bazel/checks/fixtures/defs.bzl:1-63` - the one existing Starlark rule that runs nix as a cacheable build action; the model for the guest-image action. +- `bazel/checks/nix/defs.bzl:3-9, 81-125` - the existing nix-inside-Bazel test harness, whose tags establish the non-cacheable convention the lane follows. +- `tests/unit/meta/rust-main-packages-suite-guard.sh:150-173` - the guard that force-registers any crate carrying a test aggregate and forbids positive tags on such aggregates; the reason the lane's targets live outside the main package suite. +- `nixos-modules/base.nix:67-68` and `nixos-modules/lib.nix:322, 421-429` - sshd enabled by default in the guest base, the guest's ssh capability, and the repository's existing QMP readiness vocabulary. +- `CHANGELOG.md:1086-1088` and commit `2c2f8149b` - the deletion of the heavy-gate orchestration, which R16 and four documentation sites previously described as current. +- `docs/plans/2026-08-24-001-refactor-bazel-backed-host-integration-binaries-plan.md` - the superseded plan; its R3 and its "Bazel does not become the scheduler for the NixOS VM test" boundary are what this contract reverses. +- `docs/plans/2026-08-19-002-refactor-build-test-ownership-cleanup-plan.md:266, 269-270` - the direction that tests expose Bazel only and receive binaries from Bazel rather than building at test runtime. - Planning research dossiers, kept at `/tmp/compound-engineering-1000/ce-plan-research/d57f3743/`: repository patterns, QEMU and nix best practices, framework documentation, and flow analysis. -- Emulator snapshot semantics from the research dossier: internal snapshots are supported only by the qcow2 format, a single writable non-snapshot-capable device fails the whole snapshot, and restoring a guest that has a live nested guest is documented undefined behavior on one vendor while working on another — which is why a member that has run a nested guest is retired rather than restored. +- Emulator snapshot semantics from the research dossier: internal snapshots are supported only by the qcow2 format, a single writable non-snapshot-capable device fails the whole snapshot, and restoring a guest that has a live nested guest is documented undefined behavior on one vendor while working on another - which is why a member that has run a nested guest is retired rather than restored. - Bazel execution model from the research dossier: a test action reaches a resource created outside it only by opting out of the sandbox or by an explicit mount pair, a sandboxed test's only writable surface is its own temporary directory, a test result defaults to replaying a cached verdict, and the current Bazel release no longer exposes the host temporary directory to sandboxed actions. - Measured on the contributor's host: a hardware-virtualized boot reaches a running d2b daemon in 13.6s, against 84.0s under emulation, a 6.2x difference on the same guest image. This is the basis for treating virtualization as a precondition. It does not price the refactor, and it does not describe the whole suite: it is a single boot of the default guest shape, while the two writable-store checks replace the root drive with a bootable one and the repository's own comment says that path adds many minutes to startup and can hang. Both the speed criterion and the restore stop condition are therefore measured per guest shape, and the writable-store shape's cold-boot cost is recorded before the pool is sized. @@ -259,10 +259,10 @@ flowchart TB ### Key Technical Decisions -- KTD1. The heavy-gate semaphore is not reinstated. (session-settled: user-directed — chosen over rebuilding the guard: the repository deleted it deliberately and six documentation sites still describe it as current, so correcting the contract is cheaper than reviving dead infrastructure.) Lane-local teardown and the lane's own stop conditions cover the self-race the guard used to prevent, and the two normative sites that record a `RETAIN` disposition for the semaphore namespace are a different edit class from a prose refresh and need a named owner in U8. Governs R16. -- KTD2. One lane-level test target owns the whole pool lifecycle, and the make target stays in the local class with a one-line recipe that names the repository-committed build profile itself rather than inheriting whatever profile a caller exported, the way the generate target already pins its own. (session-settled: user-directed — chosen over per-check targets with a facade that boots the pool first: the Make dispatcher expands a target to exactly one canned Bazel call under a one-class-per-target invariant, so a two-command facade would break that convention.) Governs R4, R9, R17, R18. -- KTD3. The guest image is built by a rule authored in this repository, modeled on the existing fixture rule, and the emulator is taken from the repository's own pinned nix package set rather than a new third-party Bazel ruleset. (session-settled: user-approved — no nix Bazel ruleset provides a cacheable nix-build action at the pinned or the current version, and the rules that do exist keep realization in the repository-fetch phase; taking the emulator from the same pinned set as the guest avoids adding an external module and keeps emulator and guest at one nixpkgs revision.) Governs R1, R2. -- KTD4. The guest-image action and the lane target both run unsandboxed and local, and hermeticity comes from nix's own configuration rather than Bazel's isolation. (session-settled: user-approved — nix cannot build inside a sandboxed action because its own sandbox requires root, and the fallback degrades silently rather than failing.) Governs R1, R7. +- KTD1. The heavy-gate semaphore is not reinstated. (session-settled: user-directed - chosen over rebuilding the guard: the repository deleted it deliberately and six documentation sites still describe it as current, so correcting the contract is cheaper than reviving dead infrastructure.) Lane-local teardown and the lane's own stop conditions cover the self-race the guard used to prevent, and the two normative sites that record a `RETAIN` disposition for the semaphore namespace are a different edit class from a prose refresh and need a named owner in U8. Governs R16. +- KTD2. One lane-level test target owns the whole pool lifecycle, and the make target stays in the local class with a one-line recipe that names the repository-committed build profile itself rather than inheriting whatever profile a caller exported, the way the generate target already pins its own. (session-settled: user-directed - chosen over per-check targets with a facade that boots the pool first: the Make dispatcher expands a target to exactly one canned Bazel call under a one-class-per-target invariant, so a two-command facade would break that convention.) Governs R4, R9, R17, R18. +- KTD3. The guest image is built by a rule authored in this repository, modeled on the existing fixture rule, and the emulator is taken from the repository's own pinned nix package set rather than a new third-party Bazel ruleset. (session-settled: user-approved - no nix Bazel ruleset provides a cacheable nix-build action at the pinned or the current version, and the rules that do exist keep realization in the repository-fetch phase; taking the emulator from the same pinned set as the guest avoids adding an external module and keeps emulator and guest at one nixpkgs revision.) Governs R1, R2. +- KTD4. The guest-image action and the lane target both run unsandboxed and local, and hermeticity comes from nix's own configuration rather than Bazel's isolation. (session-settled: user-approved - nix cannot build inside a sandboxed action because its own sandbox requires root, and the fallback degrades silently rather than failing.) Governs R1, R7. - KTD5. Guests are snapshotted and restored in-process, and a pool member that has run a nested guest is retired rather than restored. The rejected path is live migration: it rolls back memory and device state but not block content, and block migration was removed from the current emulator, so it cannot deliver the rollback the pool needs. It stays the fallback if restore turns out to cost more than a fresh boot. Governs R5. - KTD6. The lane target's result is never cacheable, and the lane never runs with streamed test output, which would serialize it. Governs R9, R17. - KTD7. Per-check guest configuration moves out of the runNixOSTest fixtures into a module the lane evaluates, before any fixture is deleted. Governs R10, R14. @@ -323,7 +323,7 @@ Two invariants hold across every transition. No guest is ever snapshotted or res ### Sequencing -The guest-image rule and the guest-configuration re-homing come first because the harness cannot spawn anything without them. The legacy driver guest-control surface lands before the lane test target is registered, because at that boundary the make target would otherwise point at a lane no check can run — a coverage hole R12 forbids. Only then do the pool and reporting layer, then the ports. +The guest-image rule and the guest-configuration re-homing come first because the harness cannot spawn anything without them. The legacy driver guest-control surface lands before the lane test target is registered, because at that boundary the make target would otherwise point at a lane no check can run - a coverage hole R12 forbids. Only then do the pool and reporting layer, then the ports. ### Alternative Approaches Considered @@ -357,7 +357,7 @@ Three boundaries meet at this lane, each owned by a different authority, and the ### Sources & Research -Planning research dossiers are kept at `/tmp/compound-engineering-1000/ce-plan-research/d57f3743/` — repository patterns, emulator and nix best practices, framework documentation, and flow analysis. The grounding dossier from the brainstorm phase is at `/tmp/compound-engineering-1000/ce-brainstorm/20260925-hostint-bazel/grounding.md`. The Sources section under the Product Contract carries the per-claim citations both phases rely on. +Planning research dossiers are kept at `/tmp/compound-engineering-1000/ce-plan-research/d57f3743/` - repository patterns, emulator and nix best practices, framework documentation, and flow analysis. The grounding dossier from the brainstorm phase is at `/tmp/compound-engineering-1000/ce-brainstorm/20260925-hostint-bazel/grounding.md`. The Sources section under the Product Contract carries the per-claim citations both phases rely on. --- @@ -368,7 +368,7 @@ Planning research dossiers are kept at `/tmp/compound-engineering-1000/ce-plan-r - **Goal:** A rule that evaluates the guest's NixOS configuration from declared label inputs and emits the guest artifacts as a cacheable graph output, replacing the environment-variable handoff. - **Requirements:** R1, R2, R3. - **Dependencies:** none. -- **Files:** `bazel/checks/vm/defs.bzl` (new), `bazel/checks/vm/BUILD.bazel` (new), `nix/test-support/guest-image.nix` (new), `nix/test-support/bazel-host-tools.nix` (modify), `flake.nix` (modify — add the guest evaluation as a declared entry point, keeping the two environment reads until U4), `Makefile` (modify — move the substituter preflight into the action's inputs), `.bazelrc` (add the committed guest-build profile), `MODULE.bazel` (modify — add the emulator as an entry on the existing nix package extension), `MODULE.bazel.lock` (modify — the repository's lockfile mode errors rather than regenerating), `changelog.d/` (add). +- **Files:** `bazel/checks/vm/defs.bzl` (new), `bazel/checks/vm/BUILD.bazel` (new), `nix/test-support/guest-image.nix` (new), `nix/test-support/bazel-host-tools.nix` (modify), `flake.nix` (modify - add the guest evaluation as a declared entry point, keeping the two environment reads until U4), `Makefile` (modify - move the substituter preflight into the action's inputs), `.bazelrc` (add the committed guest-build profile), `MODULE.bazel` (modify - add the emulator as an entry on the existing nix package extension), `MODULE.bazel.lock` (modify - the repository's lockfile mode errors rather than regenerating), `changelog.d/` (add). - **Approach:** 1. Model the rule on the repository's one existing cacheable nix action rather than on the nix test harness, which is a test wrapper with a different shape. 2. Declare the flake and its lock as label inputs so the action's key reflects the source, not a working-tree reference. @@ -392,12 +392,12 @@ Planning research dossiers are kept at `/tmp/compound-engineering-1000/ce-plan-r - **Goal:** Move the reusable NixOS node configuration out of the runNixOSTest fixtures into a module the lane evaluates, so a fixture can be deleted when its check ports without taking its guest declaration with it. - **Requirements:** R10, R14. - **Dependencies:** U1. -- **Files:** `tests/host-integration/lib.nix` (modify — split), `nix/test-support/host-integration-node.nix` (new), one `tests/host-integration/*.nix` file per check (modify), `changelog.d/` (add). +- **Files:** `tests/host-integration/lib.nix` (modify - split), `nix/test-support/host-integration-node.nix` (new), one `tests/host-integration/*.nix` file per check (modify), `changelog.d/` (add). - **Approach:** 1. Separate the shared node configuration and the per-check module contributions from the driver-coupled diagnostics prelude, keeping each side intact. 2. Give the re-homed module a stable interface the lane's harness can evaluate per check, independent of any test driver. 3. Leave every fixture's assertion body untouched so the lane stays green through this unit. -- **Test expectation:** none — a pure relocation that adds no test target. The unit's evidence is the existing lane staying green and the U1 image build covering the re-homed module; the scenarios below are what an implementer checks by hand, not gated coverage. +- **Test expectation:** none - a pure relocation that adds no test target. The unit's evidence is the existing lane staying green and the U1 image build covering the re-homed module; the scenarios below are what an implementer checks by hand, not gated coverage. - **Patterns to follow:** the configuration half of `tests/host-integration/lib.nix:519-641` as the source of truth for what moves. - **Test scenarios:** - The re-homed module evaluates to the same guest configuration as the fixture's inline configuration for every check. @@ -407,18 +407,18 @@ Planning research dossiers are kept at `/tmp/compound-engineering-1000/ce-plan-r ### U3. Author the guest-spawning harness -- **Goal:** A harness that reproduces each check's emulator invocation, boots its guest, waits for activation, and tears it down — the half of the lane that replaces the nix driver's boot responsibility. +- **Goal:** A harness that reproduces each check's emulator invocation, boots its guest, waits for activation, and tears it down - the half of the lane that replaces the nix driver's boot responsibility. - **Requirements:** R4, R6, R7, R10. - **Dependencies:** U1, U2. -- **Files:** `packages/d2b-vm-harness/` (new crate — pool-free first pass), `Cargo.toml` (modify — workspace member; without it the crate has no generated dependency defs and cannot be built at all), `Cargo.lock` (modify — both clippy gates run locked), `BUILD.bazel` (modify — the packages filegroup enumerates every crate), `packages/xtask/src/provider_crate_policy.rs` (modify — committed-scope row, or the crate-layout gate fails), `packages/xtask/data/blocking-census-baseline.json` (modify, or the established allow-attribute convention — a harness that spawns processes and sleeps trips the blocking census), `bazel/checks/vm/BUILD.bazel` (modify — including the lane suite naming the harness's clippy targets, so the crate is linted by the unit that creates it), `MODULE.bazel` (modify if the dependency set grows), `MODULE.bazel.lock` (modify, if that manifest changes), `changelog.d/` (add). +- **Files:** `packages/d2b-vm-harness/` (new crate - pool-free first pass), `Cargo.toml` (modify - workspace member; without it the crate has no generated dependency defs and cannot be built at all), `Cargo.lock` (modify - both clippy gates run locked), `BUILD.bazel` (modify - the packages filegroup enumerates every crate), `packages/xtask/src/provider_crate_policy.rs` (modify - committed-scope row, or the crate-layout gate fails), `packages/xtask/data/blocking-census-baseline.json` (modify, or the established allow-attribute convention - a harness that spawns processes and sleeps trips the blocking census), `bazel/checks/vm/BUILD.bazel` (modify - including the lane suite naming the harness's clippy targets, so the crate is linted by the unit that creates it), `MODULE.bazel` (modify if the dependency set grows), `MODULE.bazel.lock` (modify, if that manifest changes), `changelog.d/` (add). - **Approach:** 1. Consume the emulator binary and its runtime data from the pinned nix package set as runfile labels, the shape the existing nix-inside-Bazel harness already uses for the nix binary, so the emulator is at the same nixpkgs revision the guest image is realized from; select the accelerator explicitly instead of relying on a default that falls back to emulation. - 2. Reproduce the per-check invocation shape — memory, vCPU count, disk size, drive layout, and per-check device options — from the re-homed configuration rather than from a single uniform guest. + 2. Reproduce the per-check invocation shape - memory, vCPU count, disk size, drive layout, and per-check device options - from the re-homed configuration rather than from a single uniform guest. 3. Wait for activation through a readiness signal the repository already has vocabulary for, and treat a guest that never activates as a lane failure rather than a hang. 4. Assert the host's virtualization capabilities up front, including nested-state save support, and fail with a message naming what is missing. 5. Verify at boot that every attached writable device supports internal snapshots, so a non-snapshot-capable device fails the lane before any check runs. 6. Hold no test aggregate on the crate, which is what keeps the repository's test census from force-registering it into the main package suite. The suite that would normally carry it is the lane's own, which names the harness's clippy targets directly, so the harness is still linted by the repository's Rust gates without an aggregate. - 7. Give every reusable-pool guest a machine-identity device and a hardware random source, and take the snapshot only once the guest reports its random pool is initialised — a guest restored before that point comes back with a different identity and a colder random pool than the checks were written against. + 7. Give every reusable-pool guest a machine-identity device and a hardware random source, and take the snapshot only once the guest reports its random pool is initialised - a guest restored before that point comes back with a different identity and a colder random pool than the checks were written against. - **Execution note:** Prove the readiness and teardown paths against a real guest before adding snapshot support, so a boot failure is never confused with a restore failure. - **Patterns to follow:** the readiness vocabulary in `nixos-modules/lib.nix:421-429`; the accelerator and device handling already written in the qemu-media provider's process builder. - **Test scenarios:** @@ -433,10 +433,10 @@ Planning research dossiers are kept at `/tmp/compound-engineering-1000/ce-plan-r ### U4. Add snapshot and restore pool management and the lane test target -- **Goal:** The lane test target that boots the pool once, restores per selected check, retires members that cannot be reused, and reports per-check results — the unit that makes the lane a Bazel test. +- **Goal:** The lane test target that boots the pool once, restores per selected check, retires members that cannot be reused, and reports per-check results - the unit that makes the lane a Bazel test. - **Requirements:** R4, R5, R8, R9, R16, R17. - **Dependencies:** U3, U5. -- **Files:** `packages/d2b-vm-harness/src/pool.rs` (new), `packages/d2b-vm-harness/src/report.rs` (new), `bazel/checks/vm/BUILD.bazel` (modify — add the pool target to the lane suite), `bazel/checks/BUILD.bazel` (modify — register the lane suite), `Makefile` (modify — collapse the shell recipe to the single target, keeping the target in the local class, pinning the committed build profile in the recipe, and keeping the non-x86_64 skip as a guard on the lane target), `tests/AGENTS.md` (modify — the type-10 tier row only), `changelog.d/` (add). +- **Files:** `packages/d2b-vm-harness/src/pool.rs` (new), `packages/d2b-vm-harness/src/report.rs` (new), `bazel/checks/vm/BUILD.bazel` (modify - add the pool target to the lane suite), `bazel/checks/BUILD.bazel` (modify - register the lane suite), `Makefile` (modify - collapse the shell recipe to the single target, keeping the target in the local class, pinning the committed build profile in the recipe, and keeping the non-x86_64 skip as a guard on the lane target), `tests/AGENTS.md` (modify - the type-10 tier row only), `changelog.d/` (add). - **Approach:** 1. Register one lane test target that owns the pool for its whole run, and make the make target a thin invocation of it. 2. Take the snapshot after activation completes and before any check runs, so a restored guest is always one no check has touched. @@ -465,9 +465,9 @@ Planning research dossiers are kept at `/tmp/compound-engineering-1000/ce-plan-r - **Goal:** Re-provide the guest-control helpers and the diagnostics prelude the unported checks call, so every check keeps gating the lane unchanged until its own port. - **Requirements:** R12, R13. - **Dependencies:** U3. -- **Files:** `packages/d2b-vm-harness/src/legacy.rs` (new), `tests/host-integration/lib.nix` (modify — extract the diagnostics prelude so both surfaces share it), `tests/host-integration/*.nix` (modify — point at the shared prelude), `changelog.d/` (add). +- **Files:** `packages/d2b-vm-harness/src/legacy.rs` (new), `tests/host-integration/lib.nix` (modify - extract the diagnostics prelude so both surfaces share it), `tests/host-integration/*.nix` (modify - point at the shared prelude), `changelog.d/` (add). - **Approach:** - 1. Implement the full set of guest-control helpers the fixtures call — command execution with a bounded timeout, service-state waiting, file waiting, retrying command success, and explicit success and failure assertions — against the lane's own guest. + 1. Implement the full set of guest-control helpers the fixtures call - command execution with a bounded timeout, service-state waiting, file waiting, retrying command success, and explicit success and failure assertions - against the lane's own guest. 2. Port the diagnostics prelude to the same surface so a failing unported check reports the same stage, rows, journals, and zone debug as today. 3. Keep the guest's ssh capability and the fixtures' use of it unchanged, so assertion bodies need no edits. 4. Keep the prelude in one place so the ported Rust assertions and the legacy surface report identically. @@ -505,7 +505,7 @@ Planning research dossiers are kept at `/tmp/compound-engineering-1000/ce-plan-r - **Goal:** Port the remaining checks one at a time and complete the cutover once the last one asserts in Rust. - **Requirements:** R11, R12, R13, R14, R15, R16, R17. - **Dependencies:** U6. -- **Files:** `packages/d2b-vm-harness/tests/*.rs` (new, one per remaining check), `tests/host-integration/*.nix` (delete as each check ports), `tests/host-integration/deferred/host-zone-gateway-isolation.nix` (delete), `flake.nix` (modify — remove the `vmChecks` output), `changelog.d/` (add). The make target's recipe was already collapsed to the single target in U4, so the cutover here is the flake output and the fixtures, not the recipe. +- **Files:** `packages/d2b-vm-harness/tests/*.rs` (new, one per remaining check), `tests/host-integration/*.nix` (delete as each check ports), `tests/host-integration/deferred/host-zone-gateway-isolation.nix` (delete), `flake.nix` (modify - remove the `vmChecks` output), `changelog.d/` (add). The make target's recipe was already collapsed to the single target in U4, so the cutover here is the flake output and the fixtures, not the recipe. - **Approach:** 1. Port one check per change, retiring its fixture in the same change, keeping the lane green throughout. 2. Port the nested guest check last, and give it a single-use guest that is retired after the run. @@ -526,13 +526,13 @@ Planning research dossiers are kept at `/tmp/compound-engineering-1000/ce-plan-r - **Goal:** Make the repository's own documents match the shipped lane, including the guard that no longer exists. - **Requirements:** R14, R16. - **Dependencies:** U1, U7. -- **Files:** `AGENTS.md` (modify — the host-lane binary-injection rule), `tests/AGENTS.md` (modify — prose only; U4 owns the type-10 tier row), `tests/README.md`, `docs/contributing/gates-and-lints.md`, `docs/contributing/critical-subsystems.md`, `docs/reference/compatibility.md`, `docs/reference/support-matrix.md`, `packages/d2b-provider-device-usbip/integration/README.md` (modify — describes the semaphore as current), `docs/specs/providers/ADR-046-provider-volume-local.md`, `docs/specs/providers/ADR-046-provider-runtime-cloud-hypervisor.md`, `docs/specs/ADR-046-current-code-migration-map.md` (modify — carries a `RETAIN` disposition for the semaphore namespace, a different edit class that needs a named owner rather than a prose refresh), `specs/001-adr046-d2b3-completion/plan.md` (modify — states the semaphore contract), `CHANGELOG.md` (modify), `changelog.d/` (add). +- **Files:** `AGENTS.md` (modify - the host-lane binary-injection rule), `tests/AGENTS.md` (modify - prose only; U4 owns the type-10 tier row), `tests/README.md`, `docs/contributing/gates-and-lints.md`, `docs/contributing/critical-subsystems.md`, `docs/reference/compatibility.md`, `docs/reference/support-matrix.md`, `packages/d2b-provider-device-usbip/integration/README.md` (modify - describes the semaphore as current), `docs/specs/providers/ADR-046-provider-volume-local.md`, `docs/specs/providers/ADR-046-provider-runtime-cloud-hypervisor.md`, `docs/specs/ADR-046-current-code-migration-map.md` (modify - carries a `RETAIN` disposition for the semaphore namespace, a different edit class that needs a named owner rather than a prose refresh), `specs/001-adr046-d2b3-completion/plan.md` (modify - states the semaphore contract), `CHANGELOG.md` (modify), `changelog.d/` (add). - **Approach:** 1. Replace the environment-variable handoff description with the declared-input handoff in every site that states it as current. 2. Stop describing the heavy-gate semaphore as current and record its deletion rather than leaving the next reader to rediscover it. 3. Update the contributor-facing description of the lane: filter-based check selection, the virtualization precondition, and the loss of the emulation fallback. 4. Move the type-10 tier out of the description that keeps Layer-2 surfaces outside the Bazel scheduler, since the lane is now in that graph. -- **Test expectation:** none — documentation only. The grep scenario below is the check. +- **Test expectation:** none - documentation only. The grep scenario below is the check. - **Test scenarios:** - A repository-wide grep for the removed environment variables, the `vmChecks` output, and the heavy-gate semaphore returns nothing outside the changelog, the fragment directory, this plan, the audits and explanations, the specifications, and third-party trees, which legitimately keep a record of the removed names. - Each documentation site that described the handoff now describes the declared-input handoff consistently. @@ -543,9 +543,9 @@ Planning research dossiers are kept at `/tmp/compound-engineering-1000/ce-plan-r ## Verification Contract -- `make check` — the full Layer-1 aggregate. Must stay green at every unit boundary, since the port's premise is that the branch is always releasable. -- `make test-host-integration` — the lane. Before U1, this is the current nix recipe and is the baseline; from U4, it is the lane test target. -- `make check-tier0` — the fast policy and source-hygiene subset, for the units that only touch build wiring. +- `make check` - the full Layer-1 aggregate. Must stay green at every unit boundary, since the port's premise is that the branch is always releasable. +- `make test-host-integration` - the lane. Before U1, this is the current nix recipe and is the baseline; from U4, it is the lane test target. +- `make check-tier0` - the fast policy and source-hygiene subset, for the units that only touch build wiring. - The recorded wall-clock baselines compared against the lane's full-suite and single-check runs. - The restored-versus-fresh-boot measurement, taken before the pool grows past one member, and the marker-based equivalence gate that proves a restored guest matches a fresh boot on a member of every distinct invocation. - The repository-wide grep gate over the removed environment variables, the `vmChecks` output, and the heavy-gate semaphore. diff --git a/nix/test-support/guest-image.nix b/nix/test-support/guest-image.nix index 1ff369376..20326d272 100644 --- a/nix/test-support/guest-image.nix +++ b/nix/test-support/guest-image.nix @@ -23,6 +23,15 @@ # There is no list here of which check wants which guest, because there is # nowhere for such a list to disagree with a fixture. # +# A check whose assertions have moved to the lane's own Rust has no fixture +# left, so it has no file here to read a guest out of either: the node comes +# from the reusable node module's own table of ported checks, by the name the +# image was asked for, and the image carries no `check.py` - the manifest says +# the assertions are the lane's Rust. Such a check keeps its name, its guest, +# and its result in the lane. A fixture-less image that is in neither that +# table nor the module's table of reusable shapes is refused rather than served +# the default node. +# # The output is one store path holding the guest's system closure, the root # disk in the shape's own format, and a manifest of what a launcher needs to # boot it: the machine size, the drive layout, the boot method with its @@ -32,7 +41,8 @@ # renders it and never restates a number the node declared. It carries the # two things the lane's pool needs that a boot cannot tell it - what one # member costs the host, and the bound the pool is sized against - and, for -# a check's own guest, the check's evaluated assertions. +# a check's own guest, which side of the lane carries that check's assertions: +# the evaluated script a fixture is, or the lane's own Rust. { pkgs, self, bazelHostTools, rawBundle, extraModules ? [ ], nodeShape ? "daemon" }: let @@ -160,15 +170,41 @@ let } else loaded; - # The fixture's own name for the check, which is the name the lane reports - # it under and the name a contributor filters it by: the `vmChecks` + # The check this image was built for, in either of the two ways a check can + # be declared: a fixture file - which carries the check's guest *and* its + # assertions - or, for a check whose assertions have moved to the lane's own + # Rust and whose fixture is therefore gone, an entry in the node module's own + # table of ported checks, by the name the image was asked for. A ported check + # still needs its name, its guest, and its result in the lane; what it no + # longer needs is a `check.py`. + # + # A fixture's own name is the lane's name for the check: the `vmChecks` # attribute name is the fixture's file stem, and that is what the make # target's selection variables carry. - checkName = + fixtureCheckName = if checkFixture == null then null else lib.removeSuffix ".nix" (builtins.baseNameOf (builtins.head extraModules)); + # A fixture-less image is one of three things, and the node module says + # which: a shape the lane's own images boot, a ported check's guest, or + # nothing this tree declares - which is an error rather than a fallback, + # because a check that boots a guest nobody declared asserts against + # something no declaration describes, and a check the lane quietly skips is a + # coverage hole nothing reports. + shapeNode = d2bNode.shapeNodes.${nodeShape} or null; + portedNode = + if extraModules != [ ] then + null + else + d2bNode.portedCheckNodes.${nodeShape} or null; + checkName = + if extraModules != [ ] then + fixtureCheckName + else if portedNode != null then + nodeShape + else + null; checkNodes = if checkFixture == null then [ ] else lib.attrValues (checkFixture.nodes or { }); checkScript = if checkFixture == null then @@ -225,13 +261,16 @@ let # rather than rebuilding it here is what lets one lane carry checks that # declared three different shapes of guest. guestNode = - if checkNodes == [ ] then - # `d2bCloudHypervisorNode` is `d2bDaemonNode` with the writable store - # opted into, so the two shapes are one declaration read two ways and - # cannot drift apart. - d2bNode.d2bDaemonNode { writableStore = nodeShape == "writable-store"; } + if checkNodes != [ ] then + builtins.head checkNodes + else if portedNode != null then + # A ported check has no fixture to read a node out of. The node it boots + # is the one the reusable node module declares for it by the check's own + # name, so the guest survives the fixture exactly as the shape-only + # guests survive theirs. + portedNode.node else - builtins.head checkNodes; + shapeNode; # The node's own name inside the fixture - `nodes.machine`, in every # fixture in the tree today - which the guest has to answer for. @@ -252,6 +291,10 @@ let checkNodeName = if lib.length checkNodes == 1 then builtins.head (lib.attrNames (checkFixture.nodes or { })) + else if portedNode != null then + # A ported check's guest is the `nodes.machine` its fixture declared, so + # it answers for the same name the framework would have bound. + "machine" else null; evaluated = import (pkgs.path + "/nixos/lib/eval-config.nix") { @@ -401,10 +444,13 @@ let else { name = checkName; - # The fixture's own name for the check, kept because it is what + # The name the check is booted under, kept because it is what # appears in a `vmChecks` derivation and in a driver log line, and - # a reader comparing the two should not have to know they differ. - testName = checkFixture.name or checkName; + # a reader comparing the two should not have to know they differ. A + # fixture named it for an unported check; the node module carries it + # for a check whose fixture is gone. + testName = + if portedNode == null then checkFixture.name or checkName else portedNode.testName; # `useBootLoader` is the writable-store shape: the root drive is a # writable overlay on an installed system image, which is the # shape the Cloud Hypervisor checks boot their nested guest on. @@ -413,6 +459,12 @@ let # one is read off the node rather than off a list of check names # the lane maintains. nestedGuest = useBootLoader; + # Which side of the lane carries this check's assertions. A check + # with a fixture carries its evaluated `testScript` beside its guest + # as `check.py`, and the lane runs it through the legacy + # guest-control surface; a ported check's assertions are the lane's + # own Rust, and its image carries no script at all. + assertions = if portedNode != null then "rust" else "python"; }; # What one pool member costs the host, in the three currencies R8 names. # Read off the node's own declared fields so the pool's bound cannot @@ -681,6 +733,16 @@ else if checkNodes != [ ] && lib.length checkNodes != 1 then ${toString (lib.length checkNodes)} nodes, and a guest is one node. nodes: ${lib.concatStringsSep " " (lib.attrNames (checkFixture.nodes or { }))} '' +else if checkFixture == null && shapeNode == null && portedNode == null then + throw '' + d2b guest image: the image action asked for '${nodeShape}', which is neither one + of the reusable shapes nor a check the node module declares a guest for + (nix/test-support/host-integration-node.nix). + shapes: ${lib.concatStringsSep " " (lib.attrNames d2bNode.shapeNodes)} + ported: ${lib.concatStringsSep " " (lib.attrNames d2bNode.portedCheckNodes)} + A check whose fixture is gone carries no guest declaration of its own, so its + node has to be declared in that module's table of ported checks. + '' else if missing != [ ] || unexpected != [ ] then throw '' d2b guest image: the staged Bazel host-tool bundle does not match the diff --git a/nix/test-support/host-integration-node.nix b/nix/test-support/host-integration-node.nix index ed196b9f9..ba76387ee 100644 --- a/nix/test-support/host-integration-node.nix +++ b/nix/test-support/host-integration-node.nix @@ -29,6 +29,15 @@ # hardlink farm, so the state disk is dropped, the # root drive replaces it with an unsafe cache, and # the node boots through a bootloader. +# +# A check whose assertions have been ported to Rust has no fixture left to +# declare its guest in, so it is named in `portedCheckNodes` at the bottom of +# this file instead: one entry per ported check, built from one of the shapes +# above. The two ways of declaring a guest are the same declaration - the node +# module - read from the two places a check can be written down. `shapeNodes` +# names the two reusable shapes themselves, which is also how a fixture-less +# image that is neither a shape nor a ported check is refused rather than +# quietly served the default node. { self, lib }: let @@ -200,4 +209,51 @@ rec { inherit extra; writableStore = true; }; + + # The guest `daemon-smoke` boots: the reusable daemon node plus the JSON + # reader its assertions read the daemon's answers with. + # + # The node is a plain module here rather than a shape constructor, because + # there is nothing left to parameterise: the check's fixture declared + # `d2bDaemonNode` with this one package on top, and the check's port retires + # that fixture, so the declaration has to live where the reusable nodes do. + d2bDaemonSmokeNode = d2bDaemonNode { + extra = { pkgs, ... }: { + environment.systemPackages = [ pkgs.jq ]; + }; + }; + + # The guest each fixture-less image evaluates, by the name the image action + # asks for. A check's own guest is read out of the check's fixture; these are + # the guests with no fixture to be read out of - the two reusable shapes the + # lane's own images boot, and one entry per check whose assertions have moved + # to the lane's own Rust and whose fixture is therefore gone. + # + # Naming an image that is in neither table is an error rather than a silent + # fallback to the default node: a check that boots a guest nobody declared is + # a check asserting against something no declaration describes. The tables + # are also what keeps the two shape images and the ported checks apart, so a + # mistyped check name cannot quietly become a shape-only image the lane skips. + # + # shapeNodes the reusable shapes, by name. + # portedCheckNodes a ported check's guest, by the check's own name - the + # name the lane reports it under and the name its image + # is built for. `testName` is the name its fixture was + # booted under, kept because that is the alias a + # contributor filtering the lane may have read. + # + # One entry reaches one guest. A check that ports adds its node to + # `portedCheckNodes` and takes its fixture away; nothing else in the lane has + # to learn the check's name. + shapeNodes = { + daemon = d2bDaemonNode { }; + writable-store = d2bCloudHypervisorNode { }; + }; + + portedCheckNodes = { + daemon-smoke = { + node = d2bDaemonSmokeNode; + testName = "d2b-daemon-smoke"; + }; + }; } diff --git a/packages/d2b-test-vm-harness/src/bin/d2b-test-vm-harness.rs b/packages/d2b-test-vm-harness/src/bin/d2b-test-vm-harness.rs index 4b241659c..a3f5f5b0c 100644 --- a/packages/d2b-test-vm-harness/src/bin/d2b-test-vm-harness.rs +++ b/packages/d2b-test-vm-harness/src/bin/d2b-test-vm-harness.rs @@ -25,8 +25,8 @@ use std::{ }; use d2b_test_vm_harness::{ - ActiveGuest, Footprint, GuestSpec, HarnessError, HostFacts, LegacyCheck, LegacyGuest, - SnapshotPoint, boot, host, manifest::GuestManifest, report, + ActiveGuest, Assertions, Footprint, GuestSpec, HarnessError, HostFacts, LegacyCheck, + LegacyGuest, SnapshotPoint, boot, checks, host, manifest::GuestManifest, report, }; use serde_json::json; @@ -115,6 +115,9 @@ struct LaneGuest { manifest: GuestManifest, name: String, nested_guest: bool, + /// Where this check's assertions are: the lane's own Rust, or the + /// evaluated script its fixture is. + assertions: Assertions, footprint: Footprint, } @@ -414,6 +417,7 @@ fn read_guests(selected: &[String]) -> Result, HarnessError> { footprint: manifest.footprint, name: check.name.clone(), nested_guest: check.nested_guest, + assertions: check.assertions, manifest, }); } @@ -702,13 +706,27 @@ fn run_check_inner( "{}: second restore onto the same snapshot took {second:.1}s and wrote a layer of its own", guest.name )); - let script = fs::read_to_string(guest.image_dir.join("check.py")).map_err(|error| { - HarnessError::io( - format!("reading the assertions of check '{}'", guest.name), - error, - ) - })?; - let outcome = surface.run(&LegacyCheck::new(&guest.name, script))?; + let outcome = match guest.assertions { + Assertions::Rust => { + let assertions = checks::assertions(&guest.name).ok_or_else(|| { + HarnessError::Configuration(format!( + "the guest image for check '{}' says its assertions are the lane's own, and \ + the lane has no module that carries them", + guest.name + )) + })?; + surface.run_ported(&guest.name, assertions)? + } + Assertions::Python => { + let script = fs::read_to_string(guest.image_dir.join("check.py")).map_err(|error| { + HarnessError::io( + format!("reading the assertions of check '{}'", guest.name), + error, + ) + })?; + surface.run(&LegacyCheck::new(&guest.name, script))? + } + }; let seconds = started.elapsed().as_secs_f64(); report_line(&format!( "{}: {} after the restore ({seconds:.1}s total on this member)", diff --git a/packages/d2b-test-vm-harness/src/checks/daemon_smoke.rs b/packages/d2b-test-vm-harness/src/checks/daemon_smoke.rs new file mode 100644 index 000000000..9039b9d9c --- /dev/null +++ b/packages/d2b-test-vm-harness/src/checks/daemon_smoke.rs @@ -0,0 +1,176 @@ +//! The daemon-only surface smoke, ported from its fixture. +//! +//! It boots the daemon host the reusable node declares and asserts the +//! daemon-only end-state on a live system (ADR 0015): exactly the three +//! framework-declared root-visible units start, the broker socket is +//! socket-activated with the declared ACL, and the unprivileged public daemon +//! comes up and binds `/run/d2b/public.sock`. It is the live successor of the +//! eval-only and `D2B_LIVE` portions of `tests/d2bd-startup-smoke.sh`: it +//! exercises real systemd activation ordering and socket binding that the +//! pure-eval unit-surface gate cannot. +//! +//! The assertions below are the fixture's assertions, in the fixture's order, +//! with the fixture's own bounds. What the fixture expressed as `machine.*` +//! calls is [`GuestControl`]'s own operations, and what it expressed as +//! `stage`/`diag_unit` calls are the same primitives the fixtures' +//! diagnostics prelude provides, so a failure reported here reads the way the +//! fixture's failure read. Two things the fixture did are not restated here, +//! because the lane already does them: `start_all()` is the lane's own boot +//! of the guest the check runs against, and the diagnostics prelude is the +//! surface's own reporting rather than a check's assertion. + +use std::{collections::BTreeSet, time::Duration}; + +use crate::legacy::{GuestControl, LegacyError, LegacyResult}; + +/// The public wire surface `d2bd` binds. +const PUBLIC_SOCKET: &str = "/run/d2b/public.sock"; + +/// The bound the broker socket gets: it is socket-activated, so it is up once +/// systemd has bound and ACLed it, and a socket that has not appeared in +/// thirty seconds is not going to. +const SOCKET_ACTIVATION: Duration = Duration::from_secs(30); + +/// The bound a `d2bd` start or restart gets. The daemon builds its topology +/// before it reports readiness, and this is the check's own bound rather than +/// the driver's default. +const DAEMON_ACTIVATION: Duration = Duration::from_secs(180); + +/// The daemon-only end-state contract (ADR 0015) declares exactly these three +/// framework-owned root-visible units. +const REQUIRED_UNITS: [&str; 3] = ["d2bd.service", "d2b-broker.socket", "d2b-broker.service"]; + +/// Put a synthetic process into `d2bd.service`'s cgroup, so a restart's +/// `KillMode` is observed against a process systemd did not start. +/// +/// The command is the fixture's own, down to its words: it writes the process +/// into the unit's control group and prints its pid, which is the handle the +/// assertions after the restart use. The synthetic process is what keeps this +/// fast smoke test from needing a nested Cloud Hypervisor guest - the actual +/// Cloud Hypervisor runner-survival test lives in +/// `runtime-cloud-hypervisor-guest-preflight.nix`. +const SURVIVOR_COMMAND: &str = concat!( + "set -euo pipefail; ", + "cg=$(systemctl show -P ControlGroup d2bd.service); ", + "rm -f /run/d2b-smoke-survivor.pid; ", + "setsid -f sh -c 'echo $$ > /run/d2b-smoke-survivor.pid; exec sleep 3600' ", + "/dev/null 2>&1; ", + "for _ in $(seq 1 50); do ", + " test -s /run/d2b-smoke-survivor.pid && break; ", + " sleep 0.1; ", + "done; ", + "pid=$(cat /run/d2b-smoke-survivor.pid); ", + "echo \"$pid\" > \"/sys/fs/cgroup$cg/cgroup.procs\"; ", + "echo \"$pid\"", +); + +/// The check's assertions, in the order its fixture made them. +pub fn assertions(control: &mut GuestControl) -> LegacyResult<()> { + control.stage("boot"); + + // 1. The broker socket is created, and listening, before its service + // (socket activation): systemd binds and ACLs the AF_UNIX socket up + // front. + control.diag_unit("broker-socket", "d2b-broker.socket", SOCKET_ACTIVATION)?; + + // 2. The unprivileged public daemon comes up. It Wants= (not Requires=) + // the broker socket, so it serves while the broker stays idle. + control.diag_unit("daemon-up", "d2bd.service", DAEMON_ACTIVATION)?; + control.succeed( + &[r#"test "$(systemctl show -P Type d2bd.service)" = notify"#], + None, + )?; + control.succeed( + &[r#"test "$(systemctl show -P NotifyAccess d2bd.service)" = main"#], + None, + )?; + control.succeed( + &[r#"test "$(systemctl show -P KillMode d2bd.service)" = process"#], + None, + )?; + control.succeed( + &["systemctl show -P ExecStop d2bd.service | grep -q d2b-host-shutdown-hook"], + None, + )?; + + // 3. The live public wire surface: d2bd binds its AF_UNIX socket. + control.stage("public-socket"); + control.wait_for_file(PUBLIC_SOCKET, SOCKET_ACTIVATION)?; + control.succeed(&["test -S /run/d2b/public.sock"], None)?; + control.stage("restart-wire-surface"); + control.succeed(&["systemctl restart d2bd.service"], None)?; + control.diag_unit("daemon-restarted", "d2bd.service", DAEMON_ACTIVATION)?; + control.succeed(&["test -S /run/d2b/public.sock"], None)?; + control.succeed( + &["runuser -u alice -- d2b auth status --json >/dev/null"], + None, + )?; + + // 3b. Service restart readiness and cgroup survival. + let survivor = control.succeed(&[SURVIVOR_COMMAND], None)?.trim().to_owned(); + + control.stage("restart-cgroup-survival"); + control.succeed(&["systemctl restart d2bd.service"], None)?; + control.diag_unit("daemon-restarted-again", "d2bd.service", DAEMON_ACTIVATION)?; + control.succeed(&["test -S /run/d2b/public.sock"], None)?; + control.succeed( + &["runuser -u alice -- d2b auth status --json >/dev/null"], + None, + )?; + control.succeed(&[&format!("test -d /proc/{survivor}")], None)?; + control.succeed(&[&format!("kill {survivor}")], None)?; + + // 4. Daemon-only end-state (ADR 0015 "Verification gates"): compare the + // live system only with the framework-owned acceptance declaration. + // This avoids treating unrelated optional or managed infrastructure as + // a framework violation while still failing if a declared unit is + // absent. + control.stage("acceptance-census"); + let declared = units(&control.succeed(&["cat /etc/d2b/daemon-acceptance-units"], None)?); + let required = REQUIRED_UNITS + .iter() + .map(|unit| (*unit).to_owned()) + .collect::>(); + if declared != required { + return Err(LegacyError::Assertion(format!( + "unexpected framework acceptance census: {}", + as_a_set(&declared) + ))); + } + let live = units(&control.succeed( + &["systemctl list-units --no-pager --all --plain | awk '{print $1}' | sort"], + None, + )?); + let missing = required + .difference(&live) + .cloned() + .collect::>(); + if !missing.is_empty() { + return Err(LegacyError::Assertion(format!( + "daemon-only framework units missing: {}", + as_a_set(&missing) + ))); + } + + // 5. The broker service is socket-activated (not running until a request), + // while the socket is listening. A clean idle posture. + control.succeed(&["systemctl is-active d2b-broker.socket"], None)?; + Ok(()) +} + +/// The unit names in what a command printed, as a set. +fn units(output: &str) -> BTreeSet { + output.split_whitespace().map(str::to_owned).collect() +} + +/// A set of unit names, rendered the way the fixture's own `assert` messages +/// rendered one, so a census that fails reads the same before and after this +/// check's port. +fn as_a_set(units: &BTreeSet) -> String { + let quoted = units + .iter() + .map(|unit| format!("'{unit}'")) + .collect::>() + .join(", "); + format!("{{{quoted}}}") +} diff --git a/packages/d2b-test-vm-harness/src/checks/mod.rs b/packages/d2b-test-vm-harness/src/checks/mod.rs new file mode 100644 index 000000000..4b6c29b7d --- /dev/null +++ b/packages/d2b-test-vm-harness/src/checks/mod.rs @@ -0,0 +1,50 @@ +//! The checks whose assertions are the lane's own Rust. +//! +//! The lane's checks started as `runNixOSTest` fixtures: a nix file declared +//! the guest and a Python `testScript` made the assertions, and the lane ran +//! both. A check is ported when its assertions move here, into a function +//! over the same guest-control surface an unported check's script is given +//! ([`GuestControl`]), and its fixture is retired in the same change. Until +//! its own port a check keeps running its evaluated `check.py` through +//! [`crate::legacy`], so the lane carries any mix of the two. +//! +//! Two things a port leaves alone. The guest is not the check's to move: the +//! node it boots is declared where the reusable nodes are +//! (`nix/test-support/host-integration-node.nix`), and the image action still +//! builds that guest from that declaration. And the diagnostics are not the +//! check's to invent: [`GuestControl::stage`], [`GuestControl::diag_unit`] +//! and [`GuestControl::diag_wait`] are the primitives the fixtures' +//! diagnostics prelude provides, so a ported check that fails prints the +//! stage it was in, the rows it was asserting on, and the journal and zone +//! dump that explain them, exactly as the fixture's failure did. +//! +//! [`GuestControl::stage`]: crate::legacy::GuestControl::stage +//! [`GuestControl::diag_unit`]: crate::legacy::GuestControl::diag_unit +//! [`GuestControl::diag_wait`]: crate::legacy::GuestControl::diag_wait +//! [`GuestControl`]: crate::legacy::GuestControl + +pub mod daemon_smoke; + +use crate::legacy::{GuestControl, LegacyResult}; + +/// One check's assertions: the guest-control surface, asserted against in the +/// order the check's own fixture asserted them. +pub type Assertions = fn(&mut GuestControl) -> LegacyResult<()>; + +/// The checks that assert in Rust, by the name the lane reports them under. +/// +/// One entry per ported check, and the entry is that check's own module. The +/// image says which side of this list a check is on (its manifest carries +/// whether it holds an evaluated script), so a check whose image says it is +/// ported and which has no entry here is a lane failure rather than a check +/// that quietly does not run. +const PORTED: &[(&str, Assertions)] = &[("daemon-smoke", daemon_smoke::assertions)]; + +/// The assertions of one ported check, or `None` for a check that has not +/// been ported. +pub fn assertions(name: &str) -> Option { + PORTED + .iter() + .find(|(ported, _)| *ported == name) + .map(|(_, assertions)| *assertions) +} diff --git a/packages/d2b-test-vm-harness/src/legacy.rs b/packages/d2b-test-vm-harness/src/legacy.rs index 34d609975..36ac6b447 100644 --- a/packages/d2b-test-vm-harness/src/legacy.rs +++ b/packages/d2b-test-vm-harness/src/legacy.rs @@ -8,6 +8,14 @@ //! fixtures interpolate as well, so a check's failure reads the same before //! and after its port. //! +//! A check that *has* been ported asserts in Rust (`crate::checks`), against +//! this same surface rather than against a second one: the operations, their +//! bounds, their wording, and the diagnostics the prelude prints are the +//! ones here, so one check's port moves its assertions and nothing else. An +//! unported check reaches them through the bridge; a ported one calls them +//! directly, and [`LegacyGuest::run_ported`] reports it the way +//! [`LegacyGuest::run`] reports a script. +//! //! The shape of the surface is the driver's, deliberately. Every operation a //! fixture calls is here with the driver's semantics: `execute` runs a //! command under `set -euo pipefail` with the bound the check declared, @@ -53,6 +61,7 @@ use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use crate::{ + checks::Assertions, error::{HarnessError, Result}, guest::{ActiveGuest, CONSOLE_ID, report}, }; @@ -92,6 +101,24 @@ const RETRY_INTERVAL: Duration = Duration::from_secs(1); /// declarations of the driver's default would be two chances to disagree. const EXECUTE_DEFAULT_TIMEOUT: u64 = 900; +/// The bound one diagnostic command gets, the prelude's own. Diagnostics are +/// read on the failure path, so they are bounded twice over: by this, and by +/// the `timeout` the prelude wraps the zone explanation in. +const DIAGNOSTIC_TIMEOUT: u64 = 120; + +/// The zone and the linux user a failure's composed explanation is read +/// through. Every fixture drives one zone as one user through the same public +/// socket, so `d2b debug` explains the whole zone without the failing stage +/// listing the rows it asserted on; these are the prelude's `_diag_zone` and +/// `_diag_user`, restated in the one place a ported check's diagnostics read +/// them from. +const DIAG_ZONE: &str = "work"; +const DIAG_USER: &str = "alice"; + +/// One row a stage was asserting on: the label it is reported under, and the +/// command that dumps it. +pub type DiagRow<'a> = (&'a str, &'a str); + /// How often a check's connection to the harness is looked for, while /// refusing to block on a check that has already exited. const ACCEPT_POLL: Duration = Duration::from_millis(20); @@ -485,9 +512,51 @@ fn accept(listener: &UnixListener, mut check: Option<&mut Child>) -> Result Self { + Self { + stage: "startup".to_owned(), + started: Instant::now(), + } + } + + /// The elapsed time, rendered the way the prelude renders it. + fn elapsed(&self) -> String { + format!("{:.1}s", self.started.elapsed().as_secs_f64()) + } } impl GuestControl { + /// The surface over one console, with nothing reported yet. + /// + /// The console is the only thing that has to be supplied: the report a + /// check accumulates is empty until it starts, and so are the diagnostics + /// that time it. + fn new(console: Console) -> Self { + Self { + console, + notes: String::new(), + diagnostics: Diagnostics::new(), + } + } + /// Run a command in the guest. /// /// `timeout` is the bound the command's own execution gets, in seconds; @@ -626,6 +695,98 @@ impl GuestControl { Ok(()) } + /// Announce the stage a check is in. + /// + /// A ported check calls this where its fixture called the prelude's + /// `stage`, so a failure names the phase it happened in, in the line the + /// fixture's failure named it in, timed from the check's own start. + pub fn stage(&mut self, name: &str) { + self.diagnostics.stage = name.to_owned(); + let line = format!("[d2b] stage={name} t={}", self.diagnostics.elapsed()); + self.announce(&line); + } + + /// Run a diagnostic command and report what it wrote. + /// + /// Diagnostics only, exactly as the prelude's `diag` is: the status is + /// returned rather than asserted on, and a diagnostic that could not run + /// at all is reported as the prelude reported it rather than becoming an + /// error of its own - a failure that happened before the guest could + /// answer must still print its own stage. + pub fn diag(&mut self, command: &str, label: &str) -> i32 { + match self.execute(command, Some(DIAGNOSTIC_TIMEOUT)) { + Err(error) => { + let stage = self.diagnostics.stage.clone(); + let line = format!( + "[d2b] stage={stage} t={} {label}: diagnostic command failed: {error}", + self.diagnostics.elapsed(), + ); + self.announce(&line); + -1 + } + Ok(result) => { + let stage = self.diagnostics.stage.clone(); + let head = format!( + "[d2b] stage={stage} t={} {label} (exit {}):", + self.diagnostics.elapsed(), + result.status, + ); + self.announce(&head); + self.announce(command); + for line in result.output.trim_end().lines() { + self.announce(&format!(" {line}")); + } + result.status + } + } + } + + /// Wait for a unit, and report everything that explains a wait that did + /// not finish. + /// + /// The wait itself is [`Self::wait_for_unit`] with no user and the + /// check's own bound, which is what the prelude's `diag_unit` was. What + /// this adds is the failure path: the stage it was in, the unit's status + /// dump, the unit's own journal, and the zone's composed explanation, in + /// the prelude's own order and wording. + pub fn diag_unit(&mut self, stage: &str, unit: &str, bound: Duration) -> LegacyResult<()> { + self.stage(stage); + match self.wait_for_unit(unit, None, bound) { + Ok(()) => Ok(()), + Err(error) => { + let label = format!("{unit} status"); + let dump = format!("systemctl status {unit} --no-pager 2>&1 | tail -n 40 || true"); + self.explain_failure(stage, None, &[(label.as_str(), dump.as_str())], &[(unit, "")], &error); + Err(error) + } + } + } + + /// Wait for a command to succeed, and report the same explanation on + /// failure, with the wait itself named. + /// + /// `rows` are the resource rows the stage was asserting on and `explain` + /// are the journal sources that explain them, as `(unit, token)` - an + /// empty unit is the whole journal and an empty token is no filter - which + /// is the prelude's `diag_wait` shape. + pub fn diag_wait( + &mut self, + stage: &str, + command: &str, + bound: Duration, + rows: &[DiagRow<'_>], + explain: &[DiagRow<'_>], + ) -> LegacyResult { + self.stage(stage); + match self.wait_until_succeeds(command, bound) { + Ok(output) => Ok(output), + Err(error) => { + self.explain_failure(stage, Some(command), rows, explain, &error); + Err(error) + } + } + } + /// Whether a unit is active, and the two states that end a wait early. fn unit_is_active(&mut self, unit: &str, user: Option<&str>) -> LegacyResult { let state = self.unit_property(unit, "ActiveState", user)?; @@ -757,6 +918,79 @@ impl GuestControl { self.notes.push('\n'); } + /// Report one diagnostics line, into the lane's report and into this + /// check's own record. + /// + /// The prelude's lines went to the check's own stdout, which the lane + /// reports as it arrives and files under that check's result; these go to + /// the same two places under the same wording, so a reader of a ported + /// check's failure reads what the fixture's failure printed. + fn announce(&mut self, line: &str) { + report(line); + self.notes.push_str(line); + self.notes.push('\n'); + } + + /// Start a check's diagnostics: the stage and the clock its lines are + /// timed against are the check's own, the way the prelude's are the + /// script's. + fn begin_check(&mut self) { + self.diagnostics = Diagnostics::new(); + } + + /// Report everything that explains a diagnostic wait that did not finish: + /// the stage it was in, the rows it was asserting on, the journal lines + /// that explain them, and the zone's composed explanation. + /// + /// Diagnostics only, and in the prelude's own order: the failing stage + /// first, then each row's dump, then each explanation's journal, then the + /// zone - so the reader of a failed lane has the row set before the lines + /// that explain it. None of it can refuse: a dump that fails is reported + /// as a failed diagnostic, which is why the wait's own error is the one + /// that travels. + fn explain_failure( + &mut self, + stage: &str, + failing_wait: Option<&str>, + rows: &[DiagRow<'_>], + explain: &[DiagRow<'_>], + error: &LegacyError, + ) { + let labels = rows + .iter() + .map(|(label, _)| *label) + .collect::>() + .join(", "); + let labels = if labels.is_empty() { + "none".to_owned() + } else { + labels + }; + let failing = if failing_wait.is_some() { + format!(" wait={stage}") + } else { + String::new() + }; + let head = format!( + "[d2b] FAIL stage={stage} t={}{failing} rows=[{labels}]: {error}", + self.diagnostics.elapsed(), + ); + self.announce(&head); + if let Some(command) = failing_wait { + self.announce(&format!("[d2b] failing wait: {command}")); + } + for (label, dump) in rows.iter().copied() { + self.diag(dump, &format!("row dump: {label}")); + } + for (unit, token) in explain.iter().copied() { + self.diag( + &journal_command(unit, token), + &journal_label(unit, token), + ); + } + self.diag(&zone_explanation_command(), "zone explanation"); + } + /// The closing line of a logged operation, timed as the driver timed it /// and reported only when the operation did not refuse. fn finished(&mut self, message: &str, started: Instant) { @@ -840,6 +1074,50 @@ impl GuestControl { } } +/// The journal dump one explanation prints, in the prelude's own words and +/// bounds. +/// +/// An empty unit is the whole journal; an empty token is no filter, and the +/// filter is a fixed-string match because a token is a token, not a pattern. +fn journal_command(unit: &str, token: &str) -> String { + let scope = if unit.is_empty() { + String::new() + } else { + format!("-u {unit} ") + }; + let select = if token.is_empty() { + String::new() + } else { + format!("| grep -F -- '{token}' ") + }; + format!( + "journalctl {scope}--no-pager -o cat -b -n 4000 2>/dev/null {select}| tail -n 60 || true" + ) +} + +/// What a journal dump is reported under. +fn journal_label(unit: &str, token: &str) -> String { + let mut label = format!("journal {}", if unit.is_empty() { "all" } else { unit }); + if !token.is_empty() { + label.push_str(&format!(" lines matching '{token}'")); + } + label +} + +/// The composed `d2b debug` report the prelude prints for a failing stage: +/// the zone's ownership tree, the row that is not settled, and the structured +/// failure behind it. +/// +/// Non-fatal by construction - the command ends in `|| true` and is bounded - +/// so a failure that happened before the daemon was reachable prints its own +/// stage rather than a diagnostic error. +fn zone_explanation_command() -> String { + format!( + "runuser -u {DIAG_USER} -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock \ + timeout 60 d2b --zone {DIAG_ZONE} debug {DIAG_ZONE} 2>&1 || true" + ) +} + /// A guest a check's script can be run against, and the surface that runs it. pub struct LegacyGuest { control: GuestControl, @@ -853,10 +1131,7 @@ impl LegacyGuest { let console = Console::attach(guest)?; let work_dir = console.work_dir.clone(); Ok(Self { - control: GuestControl { - console, - notes: String::new(), - }, + control: GuestControl::new(console), work_dir, }) } @@ -907,6 +1182,35 @@ impl LegacyGuest { }) } + /// Run one ported check's assertions against this guest. + /// + /// A ported check's assertions are the lane's own Rust: the same + /// operations, in the same order, with the same bounds its fixture made, + /// so there is no interpreter between an assertion and the guest it + /// asserts against. What is reported is what [`Self::run`] reports for a + /// script - the surface's own log lines, and the check's failure - which + /// is what makes a check's diagnostics readable the same way before and + /// after its port. + pub fn run_ported( + &mut self, + name: &str, + assertions: Assertions, + ) -> Result { + self.control.begin_check(); + let passed = match assertions(&mut self.control) { + Ok(()) => true, + Err(error) => { + self.control.note(&format!("check failed: {error}")); + false + } + }; + Ok(LegacyOutcome { + name: name.to_owned(), + passed, + detail: self.control.take_notes(), + }) + } + /// Take the console back after the guest was restarted onto a restored /// disk. /// @@ -1226,10 +1530,7 @@ mod tests { let (surface, console) = UnixStream::pair().expect("a console socket pair"); thread::scope(|scope| { let asked = scope.spawn(move || guest(console, answers)); - let mut control = GuestControl { - console: Console::serving(surface, PathBuf::from("/dev/null")), - notes: String::new(), - }; + let mut control = GuestControl::new(Console::serving(surface, PathBuf::from("/dev/null"))); let outcome = body(&mut control); let notes = std::mem::take(&mut control.notes); let (asked, statuses) = asked.join().expect("the scripted guest"); @@ -1450,10 +1751,7 @@ mod tests { let (surface, console) = UnixStream::pair().expect("a console socket pair"); let mut writer = console; let _ = writer.write_all(b"connecting to host...\n"); - let mut control = GuestControl { - console: Console::serving(surface, PathBuf::from("/dev/null")), - notes: String::new(), - }; + let mut control = GuestControl::new(Console::serving(surface, PathBuf::from("/dev/null"))); let error = control .console .await_shell(Duration::from_millis(50)) @@ -1466,10 +1764,7 @@ mod tests { #[test] fn a_guest_console_that_closes_mid_command_is_reported() { let (surface, console) = UnixStream::pair().expect("a console socket pair"); - let mut control = GuestControl { - console: Console::serving(surface, PathBuf::from("/dev/null")), - notes: String::new(), - }; + let mut control = GuestControl::new(Console::serving(surface, PathBuf::from("/dev/null"))); drop(console); let error = control .console @@ -1483,10 +1778,7 @@ mod tests { let (surface, console) = UnixStream::pair().expect("a console socket pair"); thread::scope(|scope| { let _asked = scope.spawn(move || guest(console, vec![(1, block(""))])); - let mut control = GuestControl { - console: Console::serving(surface, PathBuf::from("/dev/null")), - notes: String::new(), - }; + let mut control = GuestControl::new(Console::serving(surface, PathBuf::from("/dev/null"))); let request = request("succeed", json!(["test -e /nope"])); let reply = control.dispatch(&request); assert!(!reply.ok); @@ -1498,10 +1790,7 @@ mod tests { #[test] fn an_operation_the_surface_does_not_carry_is_named_rather_than_ignored() { let (surface, _console) = UnixStream::pair().expect("a console socket pair"); - let mut control = GuestControl { - console: Console::serving(surface, PathBuf::from("/dev/null")), - notes: String::new(), - }; + let mut control = GuestControl::new(Console::serving(surface, PathBuf::from("/dev/null"))); let reply = control.dispatch(&request("wait_for_open_port", json!(["22"]))); assert!(!reply.ok); assert!(!reply.assertion, "a missing operation is the lane's, not the check's"); @@ -1525,10 +1814,7 @@ mod tests { vec![(0, block("first\n")), (0, block("second\n"))], ) }); - let mut control = GuestControl { - console: Console::serving(surface, PathBuf::from("/dev/null")), - notes: String::new(), - }; + let mut control = GuestControl::new(Console::serving(surface, PathBuf::from("/dev/null"))); let mut kwargs = BTreeMap::new(); kwargs.insert("check_output".to_owned(), Value::Bool(false)); let mut quiet = request("execute", json!(["systemctl restart d2bd.service"])); diff --git a/packages/d2b-test-vm-harness/src/lib.rs b/packages/d2b-test-vm-harness/src/lib.rs index 4bc1518aa..8617c10bc 100644 --- a/packages/d2b-test-vm-harness/src/lib.rs +++ b/packages/d2b-test-vm-harness/src/lib.rs @@ -15,16 +15,17 @@ //! `bazel/checks/vm` names its clippy targets and its guest-boot targets //! directly, so it is still linted and still run by the repository's gates. +pub mod checks; pub mod error; pub mod guest; pub mod host; +pub mod legacy; pub mod manifest; pub mod monitor; -pub mod legacy; pub use error::{HarnessError, Result, UnsnapshottableDevice}; pub use guest::{ActiveGuest, GuestSpec, RestoreTarget, SnapshotPoint, boot, report, reserve_loopback_port}; pub use legacy::{LegacyCheck, LegacyError, LegacyGuest, LegacyOutcome}; pub use host::{Capability, HostFacts, require_this_host}; -pub use manifest::{CheckRecord, EphemeralDrive, Footprint, GuestManifest, PoolBudget}; +pub use manifest::{Assertions, CheckRecord, EphemeralDrive, Footprint, GuestManifest, PoolBudget}; pub use monitor::{BlockDevice, Cache, Monitor}; diff --git a/packages/d2b-test-vm-harness/src/manifest.rs b/packages/d2b-test-vm-harness/src/manifest.rs index fc443e89b..1f3c5a794 100644 --- a/packages/d2b-test-vm-harness/src/manifest.rs +++ b/packages/d2b-test-vm-harness/src/manifest.rs @@ -97,12 +97,34 @@ pub struct CheckRecord { /// it. pub name: String, /// The name the check's own fixture gave it, which is what appears in a - /// driver log line. + /// driver log line. A check whose fixture is gone keeps the name it was + /// booted under, because that is the alias a contributor filtering the + /// lane has read. pub test_name: String, /// Whether this guest runs a guest of its own. A member that has is /// retired rather than restored, because restoring a guest with a live /// guest inside it is not defined behaviour. pub nested_guest: bool, + /// Where this check's assertions are. + pub assertions: Assertions, +} + +/// Where a check's assertions live. +/// +/// A check that has not been ported is an evaluated `testScript`, carried +/// beside its guest as `check.py` and run through the lane's legacy +/// guest-control surface. A ported check's assertions are the lane's own Rust +/// (`crate::checks`) and its image carries no script at all - so this is the +/// field that lets a guest built without a fixture still be a check the lane +/// knows how to run, and to report a check whose side of the lane is missing +/// rather than to look for a script that was never written. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Assertions { + /// The fixture's evaluated `testScript`, as `check.py`. + Python, + /// The lane's own Rust, in `crate::checks`. + Rust, } /// What one pool member costs the host, in the three currencies the pool is @@ -445,7 +467,7 @@ mod tests { "system": "x86_64-linux", "nodeShape": "daemon", "image": {"disk": "disk.qcow2", "diskFormat": "qcow2", "diskSizeMib": 8192, "systemImage": null}, - "check": {"name": "daemon-smoke", "testName": "d2b-daemon-smoke", "nestedGuest": false}, + "check": {"name": "daemon-smoke", "testName": "d2b-daemon-smoke", "nestedGuest": false, "assertions": "rust"}, "footprint": {"memorySizeMib": 3072, "cores": 3, "workingDirectoryMib": 8192}, "pool": { "memoryShareNumerator": 2, "memoryShareDenominator": 3, @@ -487,6 +509,11 @@ mod tests { assert_eq!(manifest.drives[0].file, "disk.qcow2"); assert!(!manifest.uses_bootloader()); assert_eq!(manifest.activation.marker, "D2B_LANE_READY"); + assert_eq!( + manifest.check.as_ref().map(|check| check.assertions), + Some(Assertions::Rust), + "the side of the lane a check asserts on survives into the manifest" + ); assert_eq!( manifest.networking_options, vec![ diff --git a/tests/host-integration/daemon-smoke.nix b/tests/host-integration/daemon-smoke.nix deleted file mode 100644 index e3280e070..000000000 --- a/tests/host-integration/daemon-smoke.nix +++ /dev/null @@ -1,128 +0,0 @@ -# Type-G runNixOSTest: d2b daemon-only surface smoke. -# -# Boots a real NixOS VM with `d2b.daemonExperimental.enable = true` and -# asserts the daemon-only end-state on a live system (ADR 0015): exactly the -# three framework-declared root-visible units start, the broker socket is -# socket-activated with the declared ACL, and the unprivileged public daemon -# comes up and binds `/run/d2b/public.sock`. This is the live successor of the -# eval-only + -# `D2B_LIVE` portions of `tests/d2bd-startup-smoke.sh` - it exercises real -# systemd activation ordering and socket binding that the pure-eval unit-surface -# gate cannot. -{ pkgs, self }: - -let - d2bLib = import ./lib.nix { - inherit self; - inherit (pkgs) lib; - }; - # The reusable guest configuration lives outside this directory so it - # survives the fixture (see `nix/test-support/host-integration-node.nix`). - d2bNode = import ../../nix/test-support/host-integration-node.nix { - inherit self; - inherit (pkgs) lib; - }; -in -pkgs.testers.runNixOSTest { - name = "d2b-daemon-smoke"; - - nodes.machine = d2bNode.d2bDaemonNode { - extra = { pkgs, ... }: { - environment.systemPackages = [ pkgs.jq ]; - }; - }; - - # The daemon-only end-state contract (ADR 0015): this fixture declares - # EXACTLY three framework-owned root-visible units. The broker socket is - # socket-activated, so `d2bd` keeps serving while the broker is idle; we - # assert the socket and the daemon, then the live public socket. Optional or - # managed operator infrastructure is intentionally outside this census. - testScript = '' - ${d2bLib.fixtureDiagnostics} - - start_all() - stage("boot") - - # 1. Broker socket is created + listening before its service (socket - # activation): systemd binds/ACLs the AF_UNIX socket up front. - diag_unit("broker-socket", "d2b-broker.socket", 30) - - # 2. The unprivileged public daemon comes up. It Wants= (not Requires=) the - # broker socket, so it serves while the broker stays idle. - diag_unit("daemon-up", "d2bd.service", 180) - machine.succeed("test \"$(systemctl show -P Type d2bd.service)\" = notify") - machine.succeed("test \"$(systemctl show -P NotifyAccess d2bd.service)\" = main") - machine.succeed("test \"$(systemctl show -P KillMode d2bd.service)\" = process") - machine.succeed( - "systemctl show -P ExecStop d2bd.service | grep -q d2b-host-shutdown-hook" - ) - - # 3. The live public wire surface: d2bd binds its AF_UNIX socket. - stage("public-socket") - machine.wait_for_file("/run/d2b/public.sock", timeout=30) - machine.succeed("test -S /run/d2b/public.sock") - stage("restart-wire-surface") - machine.succeed( - "systemctl restart d2bd.service" - ) - diag_unit("daemon-restarted", "d2bd.service", 180) - machine.succeed("test -S /run/d2b/public.sock") - machine.succeed("runuser -u alice -- d2b auth status --json >/dev/null") - - # 3b. Service restart readiness + cgroup survival. The synthetic process is - # moved into d2bd.service's cgroup so this verifies systemd KillMode - # behavior directly without requiring a nested Cloud Hypervisor guest in this - # fast smoke test. The actual Cloud Hypervisor runner-survival test lives in - # runtime-cloud-hypervisor-guest-preflight.nix. - survivor_pid = machine.succeed( - "set -euo pipefail; " - "cg=$(systemctl show -P ControlGroup d2bd.service); " - "rm -f /run/d2b-smoke-survivor.pid; " - "setsid -f sh -c 'echo $$ > /run/d2b-smoke-survivor.pid; exec sleep 3600' " - "/dev/null 2>&1; " - "for _ in $(seq 1 50); do " - " test -s /run/d2b-smoke-survivor.pid && break; " - " sleep 0.1; " - "done; " - "pid=$(cat /run/d2b-smoke-survivor.pid); " - "echo \"$pid\" > \"/sys/fs/cgroup$cg/cgroup.procs\"; " - "echo \"$pid\"" - ).strip() - stage("restart-cgroup-survival") - machine.succeed("systemctl restart d2bd.service") - diag_unit("daemon-restarted-again", "d2bd.service", 180) - machine.succeed("test -S /run/d2b/public.sock") - machine.succeed("runuser -u alice -- d2b auth status --json >/dev/null") - machine.succeed(f"test -d /proc/{survivor_pid}") - machine.succeed(f"kill {survivor_pid}") - - # 4. Daemon-only end-state (ADR 0015 "Verification gates"): compare the - # live system only with the framework-owned acceptance declaration. This - # avoids treating unrelated optional or managed infrastructure as a - # framework violation while still failing if a declared unit is absent. - stage("acceptance-census") - declared = set( - machine.succeed("cat /etc/d2b/daemon-acceptance-units").split() - ) - required = { - "d2bd.service", - "d2b-broker.socket", - "d2b-broker.service", - } - assert declared == required, ( - f"unexpected framework acceptance census: {declared}" - ) - unit_names = set( - machine.succeed( - "systemctl list-units --no-pager --all --plain " - "| awk '{print $1}' | sort" - ).split() - ) - missing = required - unit_names - assert not missing, f"daemon-only framework units missing: {missing}" - - # 5. The broker service is socket-activated (not running until a request), - # while the socket is listening. A clean idle posture. - machine.succeed("systemctl is-active d2b-broker.socket") - ''; -} From 79132ceda58735861a93b754acc39bb56278760b Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sat, 26 Sep 2026 22:08:51 -0700 Subject: [PATCH 11/51] fix(vm): declare the per-guest runtime dir a zone-native device worker binds under A zone-native Device owner declared only the swtpm state and marker rows. It did not declare the per-guest runtime tree, and nothing else in the tree provisions it: a legacy VM gets /run/d2b/vms/ from its path:vm-run: row, created by tmpfiles at activation, and a zone-native host creates no such row and has no tmpfiles rule for the parent. The broker's socket grant walks the path down to the worker's socket and refuses an absent ancestor - an absent ancestor yields no traverse grant, which the caller turns into "runner path ancestor is absent: /run/d2b/vms" and a refused spawn. That is why Process/swtpm-tpm0 and both Process/gpu-* rows were refused at launch while the one-shot flush was admitted: the flush is the one role whose posture binds no runtime socket, so it is the one role the grant never ran for. The state Volume was already Ready throughout, which is what ruled out the state directory as the cause. Same path and same id as the legacy row, so this adds one vocabulary for one directory rather than a second name for it. Separately: a refused forwarded invocation reached the journal as a closed-set code and nothing else. The detail is redacted operator-facing text by contract and existed only in the failure this discarded, so a provider's actual reason for refusing a spawn was unrecoverable from any log - which is why this took a direct guest probe to place rather than reading the daemon's own account of it. It is now written down where it is dropped, without changing the wire envelope the broker speaks. --- .../nix/storage-json.nix | 32 +++++++++++++++++++ packages/d2bd/src/forward_rendezvous.rs | 21 +++++++++++- 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/packages/d2b-provider-volume-local/nix/storage-json.nix b/packages/d2b-provider-volume-local/nix/storage-json.nix index 8278668c9..53dccdbd9 100644 --- a/packages/d2b-provider-volume-local/nix/storage-json.nix +++ b/packages/d2b-provider-volume-local/nix/storage-json.nix @@ -934,6 +934,38 @@ let perZoneGuestTpmStoragePaths = lib.flatten (map (name: [ + # The per-guest runtime tree a device worker binds its socket under. + # + # A legacy VM declares this as `path:vm-run:`, created by tmpfiles + # at activation. A zone-native Device owner did not: it declared only + # the swtpm state and marker rows below, and nothing else in the tree + # provisions /run/d2b/vms. The broker's socket grant walks the path + # down to the worker's socket and refuses an absent ancestor - an + # absent ancestor yields no traverse grant, which the caller turns + # into "runner path ancestor is absent: /run/d2b/vms" and the spawn + # is refused. That is why only the roles whose posture binds a + # runtime socket failed while the one-shot flush, which binds none, + # was admitted. Same path as the legacy row, so it reuses the legacy + # id rather than inventing a second vocabulary for one directory. + (mkPath { + id = "path:vm-run:${name}"; + scope = "vm:${name}"; + path = "/run/d2b/vms/${name}"; + lifecycle = "boot-scoped-readoptable"; + persistence = "boot-scoped"; + owner = principal "user" "d2bd"; + group = principal "group" "d2b"; + mode = "1770"; + creator = actor "nix-module" "tmpfiles"; + writers = [ + (actor "daemon" "d2bd") + (actor "broker" "d2b-broker") + ]; + cleanupPolicy = "boot"; + repairPolicy = "nix-activation"; + leaseClass = "process-pidfd"; + invariants = [ "no-symlink" "scope-authorization-required" ]; + }) (mkPath { id = "path:swtpm-state:${name}"; scope = "vm:${name}"; diff --git a/packages/d2bd/src/forward_rendezvous.rs b/packages/d2bd/src/forward_rendezvous.rs index 1e5913ce5..1e4d48773 100644 --- a/packages/d2bd/src/forward_rendezvous.rs +++ b/packages/d2bd/src/forward_rendezvous.rs @@ -512,7 +512,26 @@ impl ForwardRendezvous { let (response, fds) = result_response_with_fds(result); (response, fds) } - Err(failure) => (refused(failure.code()), Vec::new()), + Err(failure) => { + // The response envelope carries the refusal code and + // nothing else, so the detail is dropped here unless it is + // written down. That is what turned a refused device + // worker launch into a closed-set slug with no reason + // anywhere in the journal: the broker relayed + // `handler-refused`, the supervisor printed detail="", + // and the actual reason - a runner path ancestor that does + // not exist - died here. The detail is redacted + // operator-facing text by contract, so it is safe to log + // and it is the only place it still exists. + if let Some(detail) = failure.detail() { + tracing::warn!( + code = failure.code(), + detail, + "forwarded invocation refused with a reason" + ); + } + (refused(failure.code()), Vec::new()) + } } } From 5daa2c08c721ed36924d7730fe74fc231eec36fb Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sat, 26 Sep 2026 22:11:01 -0700 Subject: [PATCH 12/51] docs(changelog): record the lane's restore-order, runtime-dir and refusal-diagnosis fixes --- changelog.d/v3.md | 6 ++ tests/host-integration/bridge-isolation.nix | 67 --------------------- 2 files changed, 6 insertions(+), 67 deletions(-) delete mode 100644 tests/host-integration/bridge-isolation.nix diff --git a/changelog.d/v3.md b/changelog.d/v3.md index a46538c25..0bf6b0cee 100644 --- a/changelog.d/v3.md +++ b/changelog.d/v3.md @@ -1,3 +1,9 @@ ### Changed - Renamed `packages/d2b-vm-harness` to `packages/d2b-test-vm-harness`, with the crate, its `Cargo.lock` entry, its Bazel targets, and its `D2B_VM_HARNESS_*` environment contract renamed to match (`D2B_TEST_VM_HARNESS_*`). The name now reads as one name wherever it appears: the package, the binary, the lib, and every variable the lane hands the harness. No behaviour changed; the lane boots the same guests and runs the same checks. + +### Fixed + +- A restored lane guest attached its block devices in the emulator's reported order rather than the order the launch attached them, so on every restore a two-disk guest's root disk and its state disk swapped `/dev/vda` and `/dev/vdb`. The guest's volume markers anchor a volume root by device and inode and fail closed on a mismatch, so after any restore every marker disagreed with the live tree, the volume layout effect failed permanently, and the checks that wait on a volume timed out. Restores now re-attach in launch order, and the lane's restored-versus-fresh gate reports each block device's name and size so a regression of this fails in milliseconds instead of as a wait running out. +- A zone-native device owner declared its worker state under `/run/d2b/vms` but never declared the runtime directory itself, and nothing in the tree provisioned it: a legacy VM gets that directory from its own storage row, and a zone-native host had neither. The broker's socket grant refuses an absent path ancestor, so every role whose posture binds a runtime socket - the TPM worker and both GPU roles - was refused at launch, while the one-shot flush, which binds none, was admitted. The per-guest runtime tree is now declared for zone-native owners too. +- A refused forwarded invocation reached the log as a closed-set code with its reason discarded at the point of refusal, so a provider's actual explanation for refusing a spawn was unrecoverable from any journal. The detail is now written down where it was dropped. diff --git a/tests/host-integration/bridge-isolation.nix b/tests/host-integration/bridge-isolation.nix deleted file mode 100644 index 0b52bf231..000000000 --- a/tests/host-integration/bridge-isolation.nix +++ /dev/null @@ -1,67 +0,0 @@ -# Type-G runNixOSTest: Linux bridge port isolation semantics. -# -# This is the hermetic VM successor to the retired shell gate. It exercises the -# same bridge shape as root inside the test VM: one non-isolated net-VM port and -# two isolated workload ports on br-work-lan. -{ pkgs, self }: - -pkgs.testers.runNixOSTest { - name = "d2b-bridge-isolation"; - - nodes.machine = { pkgs, ... }: { - environment.systemPackages = [ - pkgs.iproute2 - pkgs.iputils - ]; - system.stateVersion = "25.11"; - }; - - testScript = '' - start_all() - - machine.succeed("mkdir -p /run/netns") - - for ns in ["netvm", "vm10", "vm11"]: - machine.succeed(f"ip netns add {ns}") - - machine.succeed("ip link add br-work-lan type bridge") - machine.succeed("ip link set br-work-lan up") - - for port, ns in [ - ("work-l1", "netvm"), - ("work-l10", "vm10"), - ("work-l11", "vm11"), - ]: - machine.succeed(f"ip link add {port} type veth peer name eth0 netns {ns}") - machine.succeed(f"ip link set {port} master br-work-lan") - machine.succeed(f"ip link set {port} up") - - machine.succeed("bridge link set dev work-l10 isolated on") - machine.succeed("bridge link set dev work-l11 isolated on") - - for ns in ["netvm", "vm10", "vm11"]: - machine.succeed(f"ip netns exec {ns} ip link set lo up") - machine.succeed(f"ip netns exec {ns} ip link set eth0 up") - - machine.succeed("ip netns exec netvm ip addr add 10.20.0.1/24 dev eth0") - machine.succeed("ip netns exec vm10 ip addr add 10.20.0.10/24 dev eth0") - machine.succeed("ip netns exec vm11 ip addr add 10.20.0.11/24 dev eth0") - - work_l1 = machine.succeed("bridge -d link show dev work-l1") - assert "isolated on" not in work_l1, ( - "net-VM bridge port work-l1 must remain non-isolated" - ) - work_l10 = machine.succeed("bridge -d link show dev work-l10") - assert "isolated on" in work_l10, "workload bridge port work-l10 is not isolated" - work_l11 = machine.succeed("bridge -d link show dev work-l11") - assert "isolated on" in work_l11, "workload bridge port work-l11 is not isolated" - - machine.succeed("ip netns exec vm10 ping -c1 -W1 10.20.0.1 >/dev/null") - machine.succeed("ip netns exec vm11 ping -c1 -W1 10.20.0.1 >/dev/null") - - machine.fail("ip netns exec vm10 ping -c1 -W1 10.20.0.11 >/dev/null 2>&1") - - machine.succeed("ip netns exec vm10 ip link set dev eth0 address 02:20:00:00:00:11") - machine.fail("ip netns exec vm10 ping -c1 -W1 10.20.0.11 >/dev/null 2>&1") - ''; -} From 91fcac894f36f307a6b548c95d2af016536e0856 Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sat, 26 Sep 2026 22:35:38 -0700 Subject: [PATCH 13/51] fix(vm): bound the host's wait for a guest's answer to a command The console bounded its read while waiting for the guest's shell and then cleared the bound, so every command read afterwards ran against an unbounded socket. The guest's own `timeout` bounds the command's execution; the host's wait for the answer is a different wait, beginning when the command is sent and ending when the answer comes back, and that one had nothing. A guest that dies mid-command, or a console that simply goes quiet, then left the host reading a socket that would never carry another byte. The lane's group threads parked in that read with their emulators already gone, and the work directories whose cleanup hangs off that path were never removed - which is what a full lane run looks like when it finishes its checks and never reports. A guest with a nested guest inside it is the one shape that can stop answering quietly, because its inner guest takes the console with it. The host's wait is now the command's own bound plus a slack covering the guest's teardown of the command, the base64 framing and the status round-trip, and a console that does not answer inside it fails with a message naming the command that went unanswered. That is the difference between a check that failed and a lane that stopped producing results. A test pins it against a console whose peer never replies. Under test the slack is short so the test does not wait the production wait; what it pins is that a bound is applied at all, because without one it does not return. Also carries three diagnostics primitives added alongside the in-flight check ports - `diag_run`, `diag_file` and `announce`, the last of them standing in for a fixture `print`. They are part of that work and are not verified by this commit's own test run; the ports that use them are where they are exercised. The previous commit also swept in a staged fixture deletion belonging to that work. It is restored by the port commit that follows this one. --- packages/d2b-test-vm-harness/src/legacy.rs | 173 +++- .../guest-agent-cap-confinement.nix | 197 ----- .../host-integration/guest-shell-service.nix | 151 ---- tests/host-integration/privilege-oracle.nix | 223 ----- .../resource-operator-activation.nix | 439 ---------- .../state-posture-contract.nix | 799 ------------------ tests/host-integration/wayland-proxy.nix | 177 ---- 7 files changed, 153 insertions(+), 2006 deletions(-) delete mode 100644 tests/host-integration/guest-agent-cap-confinement.nix delete mode 100644 tests/host-integration/guest-shell-service.nix delete mode 100644 tests/host-integration/privilege-oracle.nix delete mode 100644 tests/host-integration/resource-operator-activation.nix delete mode 100644 tests/host-integration/state-posture-contract.nix delete mode 100644 tests/host-integration/wayland-proxy.nix diff --git a/packages/d2b-test-vm-harness/src/legacy.rs b/packages/d2b-test-vm-harness/src/legacy.rs index 36ac6b447..4797611ca 100644 --- a/packages/d2b-test-vm-harness/src/legacy.rs +++ b/packages/d2b-test-vm-harness/src/legacy.rs @@ -134,6 +134,22 @@ const PYTHON: &str = "D2B_TEST_VM_HARNESS_PYTHON"; /// forever. const MINIMUM_READ_BOUND: Duration = Duration::from_millis(1); +/// How long the host waits for a guest's answer beyond the command's own +/// bound. +/// +/// The guest's `timeout` bounds the command's execution. The host's wait is +/// a different wait - it begins when the command is sent and ends when the +/// answer comes back - and it covers the guest's teardown of the command, the +/// base64 framing, and the status round-trip on top. Without room for those +/// the host would give up on a command the guest was about to answer. +#[cfg(not(test))] +const COMMAND_READ_SLACK: Duration = Duration::from_secs(30); +/// Under test the slack is short, so the test that pins this bound fails +/// fast instead of waiting the production wait. What it pins is that a bound +/// is applied at all: without one, the same test does not return. +#[cfg(test)] +const COMMAND_READ_SLACK: Duration = Duration::from_millis(50); + /// One check that has not been ported: its name, and its evaluated /// `testScript` - the fixture's own assertions with the shared diagnostics /// prelude already interpolated at the top. @@ -228,7 +244,7 @@ pub struct CommandResult { /// change what the guest runs for a command containing a quote: an ASCII /// string of unreserved characters passes through, and anything else is /// single-quoted with an embedded quote closed, double-quoted, and reopened. -fn shlex_quote(text: &str) -> String { +pub(crate) fn shlex_quote(text: &str) -> String { if text.is_empty() { return "''".to_owned(); } @@ -404,25 +420,18 @@ impl Console { self.await_shell(bound) } - /// Run one command in the guest and read back its status and output. - /// - /// The wire form is the driver's, unchanged: the command is run under - /// `set -euo pipefail` so a check's own shell assumptions - a pipeline - /// that fails, an unset variable - fail the way they failed for it, its - /// output is base64-framed so a block with a newline in it is still one - /// block, and its status is read from the pipeline's own `PIPESTATUS` - /// rather than from the status of the framing around it. + /// The body of [`Self::run`], with the console's read already bounded. #[allow(clippy::disallowed_methods, reason = "synchronous path")] - fn run(&mut self, command: &str, timeout: Option) -> Result { + fn run_bounded(&mut self, command: &str, timeout: Option) -> Result { let deadline = timeout.map(|seconds| format!("timeout {seconds} ")).unwrap_or_default(); let inner = format!("set -euo pipefail; {command}"); self.send(&format!( "{deadline}bash -c {} | (base64 -w 0; echo)\n", shlex_quote(&inner) ))?; - let output = base64_decode(self.read_block()?.trim())?; + let output = base64_decode(self.read_block(command)?.trim())?; self.send("echo ${PIPESTATUS[0]}\n")?; - let status = self.read_block()?; + let status = self.read_block(command)?; let status = status.trim().parse::().map_err(|error| { HarnessError::Configuration(format!("the guest answered {status:?} as a status: {error}")) })?; @@ -432,6 +441,40 @@ impl Console { }) } + /// Run one command in the guest and read back its status and output. + /// + /// The wire form is the driver's, unchanged: the command is run under + /// `set -euo pipefail` so a check's own shell assumptions - a pipeline + /// that fails, an unset variable - fail the way they failed for it, its + /// output is base64-framed so a block with a newline in it is still one + /// block, and its status is read from the pipeline's own `PIPESTATUS` + /// rather than from the status of the framing around it. + #[allow(clippy::disallowed_methods, reason = "synchronous path")] + fn run(&mut self, command: &str, timeout: Option) -> Result { + // The guest's own `timeout` bounds the command. This bounds the wait + // for the command's answer, which is a different wait and the one + // that hangs: a guest that dies mid-command, or a console that + // simply goes quiet, leaves the host reading a socket that will + // never carry another byte. That wait had no bound, so it took the + // whole lane down with no per-check result to show for it - the + // group threads parked in it, their guests went away, and the work + // directories whose cleanup is attached to that path were never + // removed. The same reasoning that bounds the shell greeting in + // `await_shell` bounds this. + let host_bound = + Duration::from_secs(timeout.unwrap_or(EXECUTE_DEFAULT_TIMEOUT)) + COMMAND_READ_SLACK; + self.writer + .set_read_timeout(Some(host_bound.max(MINIMUM_READ_BOUND))) + .map_err(|error| HarnessError::io("bounding the console's read", error))?; + let outcome = self.run_bounded(command, timeout); + // Every reader of this console sets its own bound, but leaving this + // one installed would silently shorten the next reader's. + self.writer + .set_read_timeout(None) + .map_err(|error| HarnessError::io("unbounding the console's read", error))?; + outcome + } + #[allow(clippy::disallowed_methods, reason = "synchronous path")] fn send(&mut self, wire: &str) -> Result<()> { self.writer @@ -444,16 +487,34 @@ impl Console { /// The block ends at the newline the shell's framing adds, which is the /// only newline in it: the output itself is base64, so it carries none. /// A read that returns nothing at all is a guest that stopped answering, - /// and is reported as such rather than as an empty output. + /// and is reported as such rather than as an empty output. A read that + /// runs out of its bound is the same guest seen a moment later, and is + /// reported as a message naming the command that was never answered - + /// which is the difference between a check that failed and a lane that + /// stopped producing results. #[allow(clippy::disallowed_methods, reason = "synchronous path")] - fn read_block(&mut self) -> Result { + fn read_block(&mut self, command: &str) -> Result { let mut block = String::new(); let mut chunk = [0_u8; 4096]; loop { - let read = self - .reader - .read(&mut chunk) - .map_err(|error| HarnessError::io("reading the guest's console", error))?; + let read = match self.reader.read(&mut chunk) { + Ok(read) => read, + Err(error) + if matches!( + error.kind(), + io::ErrorKind::WouldBlock | io::ErrorKind::TimedOut + ) => + { + return Err(HarnessError::Configuration(format!( + "the guest did not answer `{command}` within the bound this command \ + carries, so the lane cannot say whether it passed or failed: the console \ + is silent, or the guest is gone" + ))); + } + Err(error) => { + return Err(HarnessError::io("reading the guest's console", error)); + } + }; if read == 0 { return Err(HarnessError::Configuration( "the guest's console closed while a command was running".to_owned(), @@ -787,6 +848,57 @@ impl GuestControl { } } + /// Run one command, and report everything that explains a command that + /// was refused. + /// + /// The prelude's `diag_step` around a single `machine.succeed`: the stage + /// it was in, the rows it was asserting on, the journal lines that + /// explain them, and the zone's composed explanation. Unlike + /// [`Self::diag_wait`] the command runs once - a step is an assertion + /// about a settled state, not a wait - so a refusal here is the + /// command's, in the words the driver refused it with. + pub fn diag_run( + &mut self, + stage: &str, + command: &str, + rows: &[DiagRow<'_>], + explain: &[DiagRow<'_>], + ) -> LegacyResult { + self.stage(stage); + match self.succeed(&[command], None) { + Ok(output) => Ok(output), + Err(error) => { + self.explain_failure(stage, None, rows, explain, &error); + Err(error) + } + } + } + + /// Wait for a file, and report everything that explains a wait that did + /// not finish. + /// + /// The prelude's `diag_step` wrapped a file wait the way it wrapped the + /// other two: the stage it was in, the rows it was asserting on, and the + /// zone's composed explanation. A file wait is not a command wait, so the + /// refusal names no wait text; the rows are the caller's, as they were the + /// fixture's. + pub fn diag_file( + &mut self, + stage: &str, + path: &str, + bound: Duration, + rows: &[DiagRow<'_>], + ) -> LegacyResult<()> { + self.stage(stage); + match self.wait_for_file(path, bound) { + Ok(()) => Ok(()), + Err(error) => { + self.explain_failure(stage, None, rows, &[], &error); + Err(error) + } + } + } + /// Whether a unit is active, and the two states that end a wait early. fn unit_is_active(&mut self, unit: &str, user: Option<&str>) -> LegacyResult { let state = self.unit_property(unit, "ActiveState", user)?; @@ -924,8 +1036,11 @@ impl GuestControl { /// The prelude's lines went to the check's own stdout, which the lane /// reports as it arrives and files under that check's result; these go to /// the same two places under the same wording, so a reader of a ported - /// check's failure reads what the fixture's failure printed. - fn announce(&mut self, line: &str) { + /// check's failure reads what the fixture's failure printed. A ported + /// check that has a line of its own to report - a fact the fixture + /// printed because it read better than a bare command - reports it here, + /// in its own words. + pub fn announce(&mut self, line: &str) { report(line); self.notes.push_str(line); self.notes.push('\n'); @@ -1564,6 +1679,24 @@ mod tests { assert!(notes.is_empty(), "{notes}"); } + /// A console whose peer never answers. + /// + /// The peer is held for the life of this function on purpose: dropping it + /// would close the socket, and a closed console is the other failure - + /// the one that already reported itself. + #[test] + fn a_command_the_guest_never_answers_fails_instead_of_waiting_forever() { + let (surface, _never_answers) = UnixStream::pair().expect("a console socket pair"); + let mut control = GuestControl::new(Console::serving(surface, PathBuf::from("/dev/null"))); + let error = control + .execute("sleep 9999", Some(0)) + .expect_err("a guest that never answers must not hold the lane's read open"); + assert!( + error.to_string().contains("sleep 9999"), + "the refusal must name the command that went unanswered: {error}" + ); + } + #[test] fn an_unbounded_command_carries_no_bound() { let (_, _, asked, _) = against(vec![(0, block(""))], |control| { diff --git a/tests/host-integration/guest-agent-cap-confinement.nix b/tests/host-integration/guest-agent-cap-confinement.nix deleted file mode 100644 index 8c61b5981..000000000 --- a/tests/host-integration/guest-agent-cap-confinement.nix +++ /dev/null @@ -1,197 +0,0 @@ -# Type-G runNixOSTest: guest network-agent capability confinement. -# -# This check gives a live process the three capabilities required by the -# network agent inside a dedicated Linux network namespace. It verifies the -# effective set and proves that starting the process adds no such capability to -# any process sharing the host network namespace. -{ pkgs, self }: - -let - # Shared fixture diagnostics (issue #513): row dumps and per-stage markers. - d2bLib = import ./lib.nix { - inherit self; - inherit (pkgs) lib; - }; -in -pkgs.testers.runNixOSTest { - name = "d2b-guest-agent-cap-confinement"; - - nodes.machine = { ... }: { - users.groups.d2b-net-agent-test = { }; - users.users.d2b-net-agent-test = { - isSystemUser = true; - group = "d2b-net-agent-test"; - }; - - environment.systemPackages = [ pkgs.iproute2 ]; - - systemd.services.d2b-test-agent-netns = { - description = "Create the isolated network-agent test namespace"; - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - ExecStart = pkgs.writeShellScript "d2b-test-agent-netns-up" '' - set -eu - install -d -m 0755 /run/netns - ${pkgs.iproute2}/bin/ip netns add d2b-test-agent - ${pkgs.iproute2}/bin/ip -n d2b-test-agent link set lo up - ''; - ExecStop = "${pkgs.iproute2}/bin/ip netns delete d2b-test-agent"; - }; - }; - - systemd.services.d2b-test-guest-agent = { - description = "Network agent capability-confinement test process"; - requires = [ "d2b-test-agent-netns.service" ]; - after = [ "d2b-test-agent-netns.service" ]; - serviceConfig = { - Type = "simple"; - User = "d2b-net-agent-test"; - Group = "d2b-net-agent-test"; - ExecStart = "${pkgs.coreutils}/bin/sleep infinity"; - NetworkNamespacePath = "/run/netns/d2b-test-agent"; - CapabilityBoundingSet = [ - "CAP_NET_ADMIN" - "CAP_NET_BIND_SERVICE" - "CAP_NET_RAW" - ]; - AmbientCapabilities = [ - "CAP_NET_ADMIN" - "CAP_NET_BIND_SERVICE" - "CAP_NET_RAW" - ]; - NoNewPrivileges = true; - }; - }; - - system.stateVersion = "25.11"; - }; - - testScript = '' - ${d2bLib.fixtureDiagnostics} - - start_all() - stage("boot") - machine.wait_for_unit("multi-user.target", timeout=180) - stage("agent-netns") - machine.succeed("systemctl start d2b-test-agent-netns.service") - - capability_mask = (1 << 10) | (1 << 12) | (1 << 13) - - def network_namespace(pid): - return machine.succeed(f"readlink /proc/{pid}/ns/net").strip() - - def network_namespace_inode(path): - return machine.succeed(f"stat -Lc '%d:%i' {path}").strip() - - def effective_capabilities(pid): - status = machine.succeed(f"cat /proc/{pid}/status") - for line in status.splitlines(): - if line.startswith("CapEff:"): - return int(line.split(":", 1)[1].strip(), 16) - raise AssertionError(f"process {pid} has no CapEff status field") - - def host_namespace_capabilities(): - rows = machine.succeed( - "host_ns=$(readlink /proc/1/ns/net); " - "for status in /proc/[0-9]*/status; do " - "pid=''${status#/proc/}; pid=''${pid%/status}; " - "ns=$(readlink /proc/$pid/ns/net 2>/dev/null) || continue; " - "test \"$ns\" = \"$host_ns\" || continue; " - "cap=$(while IFS=: read -r key value; do " - "test \"$key\" = CapEff && { printf '%s' \"$value\"; break; }; done < \"$status\"); " - "start=$(cut -d' ' -f22 /proc/$pid/stat 2>/dev/null) || continue; " - "printf '%s %s %s\\n' \"$pid\" \"$start\" \"$cap\"; " - "done" - ) - result = {} - for row in rows.splitlines(): - pid, start, cap = row.split() - result[(pid, start)] = int(cap, 16) - return result - - def service_processes(unit): - control_group = machine.succeed( - f"systemctl show -P ControlGroup {unit}" - ).strip() - assert control_group.startswith("/"), ( - f"{unit} has invalid control group {control_group!r}" - ) - rows = machine.succeed( - f"find /sys/fs/cgroup{control_group} -name cgroup.procs " - "-type f -exec cat {} +" - ) - identities = set() - for pid in rows.splitlines(): - start = machine.succeed( - f"cut -d' ' -f22 /proc/{pid}/stat" - ).strip() - identities.add((pid, start)) - return identities - - stage("baseline-caps") - host_namespace = network_namespace(1) - baseline = host_namespace_capabilities() - - stage("agent-up") - machine.succeed("systemctl start d2b-test-guest-agent.service") - diag_unit("guest-agent-up", "d2b-test-guest-agent.service", 60) - agent_pid = machine.succeed( - "systemctl show -P MainPID d2b-test-guest-agent.service" - ).strip() - assert agent_pid not in ("", "0"), "network agent did not start" - - agent_namespace = network_namespace(agent_pid) - agent_namespace_inode = network_namespace_inode(f"/proc/{agent_pid}/ns/net") - declared_namespace_inode = network_namespace_inode("/run/netns/d2b-test-agent") - assert agent_namespace_inode == declared_namespace_inode, ( - "network agent did not inherit the declared Guest network namespace" - ) - assert agent_namespace != host_namespace, ( - "network agent unexpectedly shares the host network namespace" - ) - - stage("confinement-assertions") - agent_capabilities = effective_capabilities(agent_pid) - assert agent_capabilities & capability_mask == capability_mask, ( - "network agent is missing a required effective network capability" - ) - assert agent_capabilities & ~capability_mask == 0, ( - "network agent received an undeclared effective capability" - ) - - service_identities = service_processes("d2b-test-guest-agent.service") - assert (agent_pid, machine.succeed( - f"cut -d' ' -f22 /proc/{agent_pid}/stat" - ).strip()) in service_identities, ( - "network agent main process is outside its service control group" - ) - - after = host_namespace_capabilities() - gained = sorted( - ( - pid, - start, - before & capability_mask, - after[(pid, start)] & capability_mask, - ) - for (pid, start), before in baseline.items() - if (pid, start) in after - and after[(pid, start)] & capability_mask & ~before - ) - assert not gained, ( - "starting the network agent added effective network capabilities to " - f"an existing host-network-namespace process: {gained}" - ) - - service_leaks = sorted( - (pid, start, after[(pid, start)] & capability_mask) - for pid, start in service_identities - if (pid, start) in after and after[(pid, start)] & capability_mask - ) - assert not service_leaks, ( - "network agent service left a capability-bearing process in the host " - f"network namespace: {service_leaks}" - ) - ''; -} diff --git a/tests/host-integration/guest-shell-service.nix b/tests/host-integration/guest-shell-service.nix deleted file mode 100644 index fe93dd2f3..000000000 --- a/tests/host-integration/guest-shell-service.nix +++ /dev/null @@ -1,151 +0,0 @@ -# Type-G runNixOSTest: guest ComponentSession service wiring. -# -# Applies the component-session module directly to a NixOS test node and asserts -# that the Guest target agent boots from enrolled inputs and reaches its -# AF_VSOCK ComponentSession listener - the route the shell family's per-session -# supervisor service and the other provider services are served over. This -# avoids a nested d2b-managed VM while still exercising NixOS module -# realization. -{ pkgs, self }: - -let - # Shared fixture diagnostics (issue #513): row dumps and per-stage markers. - d2bLib = import ./lib.nix { - inherit self; - inherit (pkgs) lib; - }; - - # The Guest target agent boots from enrollment-owner inputs (it never - # generates them): a 32-byte ComponentSession key pair and a bundle whose - # sha256 self-hash covers the canonical JSON without `bundleHash`. - fixtureKeys = pkgs.runCommand "guest-shell-component-session-keys" { } '' - mkdir -p "$out" - printf '\001\002\003\004\005\006\007\010\011\012\013\014\015\016\017\020\021\022\023\024\025\026\027\030\031\032\033\034\035\036\037\040' > "$out/guest.key" - printf '\130\151\257\364\120\124\227\062\313\252\355\136\135\371\263\012\155\243\034\260\345\164\053\255\132\324\241\247\150\361\246\173' > "$out/parent.pub" - ''; - - guestBundle = pkgs.runCommand "guest-shell-guest-bundle" { - nativeBuildInputs = [ pkgs.python3 ]; - } '' - mkdir -p "$out" - printf '%s\n' '{"schemaVersion":"v2","site":{"allowUnsafeEastWest":false},"environments":[],"nftables":{"family":"inet","table":"d2b","chains":[],"tableHashAfterApply":null,"ownershipId":"guest-shell-service"},"networkManager":{"filePath":"/etc/NetworkManager/conf.d/00-d2b-unmanaged.conf","matchCriteria":[],"reloadBehavior":"atomic-reload","ownership":{"owner":"root","group":"root","mode":"0644","driftPolicy":"replace"}},"hostsFile":{"startMarker":"# d2b-managed begin","endMarker":"# d2b-managed end","rule":"replace-managed-block"},"kernelModules":[],"fdOwnership":[],"cloudHypervisorCapabilities":[],"ifNameMappings":[],"ch":null,"firewallCoexistencePolicy":null}' > "$out/host.json" - printf '%s\n' '{"schemaVersion":"v2","vms":[]}' > "$out/processes.json" - printf '%s\n' '{"schemaVersion":"v2","publicOperations":[],"brokerOperations":[]}' > "$out/privileges.json" - printf '%s\n' '{"_manifest":{"manifestVersion":6},"_observability":{"enabled":false,"signozUrl":"http://127.0.0.1:8080","signozOtlpGrpcPort":4317,"signozOtlpHttpPort":4318,"obsVsockCid":0,"obsVsockHostSocket":"","vmName":""}}' > "$out/vms.json" - python3 - "$out/bundle.json" <<'PY' - import hashlib - import json - import sys - - # Zone-native v3 bundle: the loader (BundleResolver) accepts only the - # v3 contract. The self-hash is computed over the serialization with - # bundleHash absent and artifactHashes nullified (verify_bundle_hash). - bundle = { - "artifactHashes": {}, - "bundleVersion": 1, - "schemaVersion": "v3", - "privilegesPath": "privileges.json", - "zones": [], - "generation": { - "generatedAt": None, - "generator": "guest-shell-service", - "sourceRevision": None, - }, - } - preimage = dict(bundle) - preimage["artifactHashes"] = None - canonical = json.dumps(preimage, sort_keys=True, separators=(",", ":")).encode() - bundle["bundleHash"] = "sha256:" + hashlib.sha256(canonical).hexdigest() - with open(sys.argv[1], "w", encoding="utf-8") as output: - json.dump(bundle, output, sort_keys=True, separators=(",", ":")) - output.write("\n") - PY - ''; -in -pkgs.testers.runNixOSTest { - name = "d2b-guest-shell-service"; - - nodes.machine = { lib, pkgs, ... }: { - imports = [ - ../../nixos-modules/component-session.nix - ../../nixos-modules/guest-broker.nix - { - _module.args = { - d2bInputs = { inherit self; }; - d2bHostTools = { - broker = self.packages.${pkgs.system}.d2b-broker-guest-static; - }; - d2bHostToolOverrides = self.lib.d2bHostToolOverrides; - }; - - d2b.componentSession = { - enable = lib.mkForce true; - guestConfigPath = lib.mkForce null; - }; - - # The Guest target agent binds an AF_VSOCK ComponentSession listener. - # The lane's QEMU ships vhost-vsock-pci and now passes /dev/vhost-vsock - # into the build sandbox, so this node can carry the same device the - # enrolled Guest gets, plus a fixture bundle and key pair installed at - # the production owner/mode the resolver verifies (root:d2bd 0640). - virtualisation.qemu.options = [ "-device" "vhost-vsock-pci,guest-cid=3" ]; - boot.kernelModules = [ "vmw_vsock_virtio_transport" ]; - - environment.etc."d2b/component-session/guest.key".source = - "${fixtureKeys}/guest.key"; - environment.etc."d2b/component-session/parent.pub".source = - "${fixtureKeys}/parent.pub"; - - d2b.componentSession.localPrivateKeyPath = - "/etc/d2b/component-session/guest.key"; - d2b.componentSession.parentPublicKeyPath = - "/etc/d2b/component-session/parent.pub"; - d2b.componentSession.bundlePath = "/var/lib/d2b/guest-bundle/bundle.json"; - d2b.guestBroker.bundlePath = "/var/lib/d2b/guest-bundle/bundle.json"; - - systemd.services.d2b-install-guest-bundle = { - requiredBy = [ "d2bd-guest.service" "d2b-broker-guest.service" ]; - before = [ "d2bd-guest.service" "d2b-broker-guest.service" ]; - serviceConfig.Type = "oneshot"; - script = '' - install -d -o root -g d2bd -m 0750 /var/lib/d2b/guest-bundle - for file in bundle.json host.json processes.json privileges.json; do - install -o root -g d2bd -m 0640 \ - ${guestBundle}/"$file" /var/lib/d2b/guest-bundle/"$file" - done - install -o root -g d2bd -m 0644 \ - ${guestBundle}/vms.json /var/lib/d2b/guest-bundle/vms.json - ''; - }; - - system.stateVersion = "25.11"; - } - ]; - }; - - testScript = '' - ${d2bLib.fixtureDiagnostics} - - start_all() - stage("boot") - machine.wait_for_unit("multi-user.target", timeout=180) - - # The Guest target agent must boot from the enrolled bundle and key pair - # and reach its AF_VSOCK listener; a bundle or key it cannot read fails - # closed here instead of restart-looping unnoticed. - diag_unit("guest-daemon", "d2bd-guest.service", 120) - machine.succeed("systemctl is-active --quiet d2bd-guest.service") - diag_wait( - "guest-listener-bound", - "journalctl -u d2bd-guest.service --no-pager -b " - "| grep -F 'Guest ComponentSession listener bound'", - timeout=60, - rows=unit_dumps("d2bd-guest.service"), - explain=[("d2bd-guest.service", None)], - ) - machine.fail( - "journalctl --no-pager -b " - "| grep -F 'Guest process bundle validation failed'" - ) - ''; -} diff --git a/tests/host-integration/privilege-oracle.nix b/tests/host-integration/privilege-oracle.nix deleted file mode 100644 index 643b218d3..000000000 --- a/tests/host-integration/privilege-oracle.nix +++ /dev/null @@ -1,223 +0,0 @@ -# Type-G runNixOSTest: live broker privilege posture oracle. -# -# Hermetic successor to the retired self-hosted L1c shell oracle. It boots a -# d2b daemon host, starts the socket-activated privileged broker, derives the -# expected posture from the rendered systemd unit, and checks the live -# /proc/ state for the hardening invariants that matter at runtime. -{ pkgs, self }: - -let - d2bLib = import ./lib.nix { - inherit self; - inherit (pkgs) lib; - }; - # The reusable guest configuration lives outside this directory so it - # survives the fixture (see `nix/test-support/host-integration-node.nix`). - d2bNode = import ../../nix/test-support/host-integration-node.nix { - inherit self; - inherit (pkgs) lib; - }; -in -pkgs.testers.runNixOSTest { - name = "d2b-privilege-oracle"; - - nodes.machine = d2bNode.d2bDaemonNode { }; - - testScript = '' - ${d2bLib.fixtureDiagnostics} - - import shlex - - start_all() - stage("boot") - - diag_unit("broker-socket", "d2b-broker.socket", 30) - diag_unit("daemon-up", "d2bd.service", 180) - - # The broker is socket-activated, but starting the service directly keeps a - # live Type=notify process long enough to read its /proc posture. - stage("broker-start") - machine.succeed("systemctl start d2b-broker.service") - broker_pid = machine.succeed( - "for i in $(seq 1 100); do " - "pid=$(systemctl show -p MainPID --value d2b-broker.service); " - "if [ -n \"$pid\" ] && [ \"$pid\" != 0 ] && [ -r \"/proc/$pid/status\" ]; then " - "echo \"$pid\"; exit 0; fi; " - "sleep 0.2; " - "done; " - "echo 'd2b-broker.service did not publish a MainPID within 20s:'; " - "systemctl status --no-pager d2b-broker.service; " - "exit 1" - ).strip() - print(f"live d2b-broker PID: {broker_pid}") - - stage("broker-posture") - unit_raw = machine.succeed( - "systemctl show d2b-broker.service " - "-p CapabilityBoundingSet " - "-p AmbientCapabilities " - "-p NoNewPrivileges " - "-p User " - "-p Group " - "-p Slice " - "-p SystemCallFilter" - ) - print("rendered d2b-broker.service posture:\n" + unit_raw) - unit = dict(line.split("=", 1) for line in unit_raw.strip().splitlines() if "=" in line) - - status_raw = machine.succeed(f"cat /proc/{broker_pid}/status") - cgroup_raw = machine.succeed(f"cat /proc/{broker_pid}/cgroup") - ns_raw = machine.succeed( - f"for ns in cgroup ipc mnt net pid time time_for_children user uts; do " - f"[ -e /proc/{broker_pid}/ns/$ns ] && printf '%s=%s\\n' \"$ns\" \"$(readlink /proc/{broker_pid}/ns/$ns)\"; " - "done" - ) - print("live /proc status subset:\n" + "\n".join( - line for line in status_raw.splitlines() - if line.startswith(("Uid:", "Gid:", "Groups:", "CapEff:", "CapBnd:", "CapAmb:", "NoNewPrivs:", "Seccomp:")) - )) - print("live cgroup:\n" + cgroup_raw) - print("live namespaces:\n" + ns_raw) - - status = {} - for line in status_raw.splitlines(): - if ":" in line: - key, value = line.split(":", 1) - status[key] = value.strip() - - cap_names = [ - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "KILL", - "SETGID", - "SETUID", - "SETPCAP", - "LINUX_IMMUTABLE", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_ADMIN", - "NET_RAW", - "IPC_LOCK", - "IPC_OWNER", - "SYS_MODULE", - "SYS_RAWIO", - "SYS_CHROOT", - "SYS_PTRACE", - "SYS_PACCT", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_NICE", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "MKNOD", - "LEASE", - "AUDIT_WRITE", - "AUDIT_CONTROL", - "SETFCAP", - "MAC_OVERRIDE", - "MAC_ADMIN", - "SYSLOG", - "WAKE_ALARM", - "BLOCK_SUSPEND", - "AUDIT_READ", - "PERFMON", - "BPF", - "CHECKPOINT_RESTORE", - ] - cap_numbers = {name: index for index, name in enumerate(cap_names)} - cap_last_cap = int(machine.succeed("cat /proc/sys/kernel/cap_last_cap").strip()) - full_cap_mask = (1 << (cap_last_cap + 1)) - 1 - - def parse_cap_set(value, *, empty_is_full): - value = value.strip() - if value == "": - return full_cap_mask if empty_is_full else 0 - if value.lower().startswith("0x"): - return int(value, 16) - - mask = 0 - for token in value.split(): - token = token.strip() - if not token: - continue - norm = token.upper().replace("-", "_") - if norm.startswith("CAP_"): - norm = norm[4:] - assert norm in cap_numbers, f"unknown capability from systemd unit: {token}" - bit = cap_numbers[norm] - assert bit <= cap_last_cap, ( - f"systemd unit declares capability {token} above kernel cap_last_cap={cap_last_cap}" - ) - mask |= 1 << bit - return mask - - def parse_unit_bool(value): - norm = value.strip().lower() - if norm in ("yes", "true", "1"): - return 1 - if norm in ("no", "false", "0", ""): - return 0 - raise AssertionError(f"unknown systemd boolean value: {value!r}") - - stage("posture-oracle") - expected_uid = int(machine.succeed(f"id -u {shlex.quote(unit['User'])}").strip()) - expected_gid = int( - machine.succeed(f"getent group {shlex.quote(unit['Group'])} | cut -d: -f3").strip() - ) - expected_cap_bnd = parse_cap_set(unit["CapabilityBoundingSet"], empty_is_full=True) - expected_cap_amb = parse_cap_set(unit["AmbientCapabilities"], empty_is_full=False) - expected_nonewprivs = parse_unit_bool(unit["NoNewPrivileges"]) - expected_slice = unit["Slice"].strip() - - actual_uids = [int(part) for part in status["Uid"].split()] - actual_gids = [int(part) for part in status["Gid"].split()] - actual_cap_eff = int(status["CapEff"], 16) - actual_cap_bnd = int(status["CapBnd"], 16) - actual_cap_amb = int(status["CapAmb"], 16) - actual_nonewprivs = int(status["NoNewPrivs"]) - actual_seccomp = int(status["Seccomp"]) - cgroup_paths = [line.split(":", 2)[2] for line in cgroup_raw.splitlines() if ":" in line] - - assert all(uid == expected_uid for uid in actual_uids), ( - f"broker Uid must match rendered User={unit['User']} ({expected_uid}), got {actual_uids}" - ) - assert expected_uid == 0, f"broker must run as root uid 0, rendered User={unit['User']}" - assert all(gid == expected_gid for gid in actual_gids), ( - f"broker Gid must match rendered Group={unit['Group']} ({expected_gid}), got {actual_gids}" - ) - - assert actual_cap_bnd == expected_cap_bnd, ( - f"CapBnd must match rendered CapabilityBoundingSet: " - f"expected 0x{expected_cap_bnd:x}, got 0x{actual_cap_bnd:x}" - ) - assert actual_cap_bnd != full_cap_mask, ( - f"CapBnd is the full kernel capability mask 0x{full_cap_mask:x}, not the bounded broker set" - ) - assert actual_cap_eff & ~actual_cap_bnd == 0, ( - f"CapEff 0x{actual_cap_eff:x} contains bits outside CapBnd 0x{actual_cap_bnd:x}" - ) - - assert actual_cap_amb == expected_cap_amb, ( - f"CapAmb must match rendered AmbientCapabilities: " - f"expected 0x{expected_cap_amb:x}, got 0x{actual_cap_amb:x}" - ) - assert actual_cap_amb == 0, f"broker must not carry ambient capabilities, got 0x{actual_cap_amb:x}" - - assert actual_nonewprivs == expected_nonewprivs, ( - f"NoNewPrivs must match rendered NoNewPrivileges={unit['NoNewPrivileges']}, " - f"got {actual_nonewprivs}" - ) - assert actual_seccomp == 2, f"broker must run in seccomp filter mode (2), got {actual_seccomp}" - - assert any(expected_slice in path for path in cgroup_paths), ( - f"broker cgroup path must contain rendered Slice={expected_slice}, got {cgroup_paths}" - ) - assert any("d2b.slice" in path for path in cgroup_paths), ( - f"broker cgroup path must contain d2b.slice, got {cgroup_paths}" - ) - ''; -} diff --git a/tests/host-integration/resource-operator-activation.nix b/tests/host-integration/resource-operator-activation.nix deleted file mode 100644 index 3c2f9f69b..000000000 --- a/tests/host-integration/resource-operator-activation.nix +++ /dev/null @@ -1,439 +0,0 @@ -# Type-G runNixOSTest: authenticated Resource operator and framework census. -# -# This fixture is intentionally separate from the native controller canaries: -# it reaches the installed d2b CLI, public socket, systemd restart boundary, -# and the framework-declared daemon unit surface in a real NixOS guest. The -# census does not sweep every d2b-prefixed unit on an operator host, because -# optional or managed infrastructure is outside this fixture's ownership. -{ pkgs, self }: - -let - inherit (pkgs) lib; - d2bLib = import ./lib.nix { - inherit self; - inherit lib; - hostToolBundle = - if self.lib ? d2bHostToolBundle then self.lib.d2bHostToolBundle else null; - }; - # The reusable guest configuration lives outside this directory so it - # survives the fixture (see `nix/test-support/host-integration-node.nix`). - d2bNode = import ../../nix/test-support/host-integration-node.nix { - inherit self; - inherit lib; - }; - providerArtifact = d2bLib.mkAcceptanceProviderArtifact pkgs; - acceptancePublisherKey = providerArtifact.trustedPublisher.signingKey; - artifacts = { - acceptance-provider = { - inherit (providerArtifact) package type catalog; - }; - }; - hostRuntime = pkgs.writeText "d2b-acceptance-host-runtime.json" (builtins.toJSON { - schemaVersion = "v1"; - bundleVersion = 1; - generatedAt = "1970-01-01T00:00:00.000Z"; - nftAppliedHash = null; - ifnames = [ ]; - }); -in -pkgs.testers.runNixOSTest { - name = "d2b-resource-operator-activation"; - - nodes.machine = d2bNode.d2bDaemonNode { - extra = { ... }: { - networking.nftables.enable = true; - networking.nftables.ruleset = lib.mkAfter '' - table inet d2b {} - ''; - systemd.tmpfiles.rules = [ - "d /etc/NetworkManager/conf.d 0755 root root -" - ]; - environment.etc."d2b/acceptance-host-runtime.json".source = hostRuntime; - d2b.site.adminUsers = [ "alice" ]; - systemd.services.d2bd.serviceConfig.ExecStartPre = lib.mkAfter [ - "+${pkgs.writeShellScript "d2b-acceptance-hosts-prep" '' - if [ -L /etc/hosts ]; then - ${pkgs.coreutils}/bin/cat /etc/hosts > /run/d2b-acceptance-hosts - ${pkgs.coreutils}/bin/rm -f /etc/hosts - ${pkgs.coreutils}/bin/install -o root -g root -m 0644 \ - /run/d2b-acceptance-hosts /etc/hosts - fi - ''}" - "+${pkgs.writeShellScript "d2b-acceptance-host-runtime-prep" '' - ${pkgs.coreutils}/bin/install -D -o root -g d2bd -m 0640 \ - /etc/d2b/acceptance-host-runtime.json \ - /var/lib/d2b/runtime/host-runtime.json - ''}" - ]; - users.users.bob = { - isNormalUser = true; - uid = 1001; - }; - d2b.artifacts = artifacts; - d2b.zones.local-root.trustedPublishers.d2b-u20-acceptance.signingKey = - acceptancePublisherKey; - d2b.zones.work.parentZone = "local-root"; - d2b.zones.work.trustedPublishers.d2b-u20-acceptance.signingKey = - acceptancePublisherKey; - d2b.zones.work.resources = { - alice = { - type = "User"; - spec = { - displayName = "Alice"; - groups = [ ]; - osUsername = "alice"; - }; - }; - d2bd = { - type = "User"; - spec = { - displayName = "d2bd"; - groups = [ ]; - osUsername = "d2bd"; - }; - }; - operator-reader = { - type = "Role"; - spec.rules = [ - { - resourceTypes = [ - "Host" - "Process" - "Provider" - "User" - ]; - verbs = [ "get" "list" ]; - subresources = [ ]; - resourceNames = [ ]; - zones = [ "work" ]; - executionRefs = [ ]; - sessionVerbs = [ "connect" "invoke" ]; - } - ]; - }; - operator-reader-binding = { - type = "RoleBinding"; - spec = { - roleRef = "Role/operator-reader"; - subjects = [ "User/alice" ]; - externalPrincipalSelector = null; - scopeNarrowing = null; - }; - }; - host-system = { - type = "Host"; - spec = { - providerRef = "Provider/system-core"; - defaultDomain = "system"; - allowedDomains = [ "system" ]; - budget = { }; - networkAttachments = [ ]; - deviceAttachments = [ ]; - volumeAttachmentDefaults = [ ]; - }; - }; - network-local = { - type = "Provider"; - spec = { - artifactId = "acceptance-provider"; - config.controllerExecutionRef = "Host/host-system"; - }; - }; - }; - environment.systemPackages = [ pkgs.jq ]; - }; - }; - - testScript = '' - ${d2bLib.fixtureDiagnostics} - - # Row projections the shared diagnostics print on a timed-out wait; they - # mirror the fields each wait asserts on (issue #513). - diag_projection = ( - "[.resources[] | {type: .type, name: .metadata.name, " - "owner: .metadata.ownerRef, uid: .metadata.uid, " - "gen: .metadata.generation, phase: .status.phase, " - "obs: .status.observedGeneration, " - "conditions: [.status.conditions[]? | {type: .type, reason: .reason}]}]" - ) - - def live_rows(label, resource_type): - return ( - label, - "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - f"d2b --zone work --json list {resource_type} 2>/dev/null | " - f"jq -c '{diag_projection}' 2>/dev/null || true", - ) - - def saved_rows(label, path): - return ( - label, - f"jq -c '{diag_projection}' {path} 2>/dev/null " - f"|| cat {path} 2>/dev/null || true", - ) - - start_all() - stage("boot") - machine.wait_for_unit("nftables.service", timeout=180) - machine.succeed("nft list table inet d2b") - machine.wait_for_unit("d2b-broker.socket", timeout=30) - diag_unit("daemon-up", "d2bd.service", 180) - machine.wait_for_file("/run/d2b/public.sock", timeout=30) - diag_wait( - "provider-session-live", - "journalctl -u d2bd.service --no-pager -o cat " - "| grep -F 'external Provider controller ResourceV3 session live'", - timeout=60, - rows=[live_rows("Process rows", "Process")], - explain=[("d2bd.service", "ResourceV3 session")], - ) - machine.succeed("runuser -u alice -- d2b auth status --json >/run/d2b-auth-before.json") - - diag_wait( - "host-row", - "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - "d2b --zone work --json list Host >/run/d2b-host-before.json && " - "jq -e '.resources[] | select(.type == \"Host\" and " - ".metadata.name == \"host-system\") | " - "(.status.phase == \"Ready\" and " - ".status.observedGeneration == .metadata.generation)' " - "/run/d2b-host-before.json", - timeout=60, - rows=[saved_rows("Host rows", "/run/d2b-host-before.json")], - explain=[("d2bd.service", "host-system")], - ) - machine.succeed( - "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - "d2b --zone work --json list User " - ">/run/d2b-user-before.json && " - "jq -e '.resources[] | select(.type == \"User\" and " - ".metadata.name == \"alice\") | " - "(.status.phase == \"Ready\" and " - ".status.observedGeneration == .metadata.generation)' " - "/run/d2b-user-before.json" - ) - machine.succeed( - "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - "d2b --zone work --json list Provider " - ">/run/d2b-provider-before.json && " - "jq -e '.resources[] | select(.type == \"Provider\" and " - ".metadata.name == \"network-local\") | " - "(.metadata.uid != null and .metadata.generation > 0)' " - "/run/d2b-provider-before.json" - ) - diag_wait( - "network-controller-process", - "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - "d2b --zone work --json list Process " - ">/run/d2b-process-before.json && " - "jq -e '([.resources[] | select(.type == \"Process\" and " - ".metadata.ownerRef == \"Provider/network-local\")] | length == 1) and " - "(.resources[] | select(.type == \"Process\" and " - ".metadata.ownerRef == \"Provider/network-local\") | " - "(.metadata.uid != null and .metadata.generation > 0 and " - ".status.phase == \"Ready\" and " - ".status.observedGeneration == .metadata.generation))' " - "/run/d2b-process-before.json", - timeout=60, - rows=[ - saved_rows("Process rows", "/run/d2b-process-before.json"), - live_rows("Controller Process rows", "Process"), - ], - explain=[("d2bd.service", "network-local")], - ) - diag_wait( - "controller-pid", - "test \"$(ps -eo pid=,args= | awk '$NF ~ /acceptance-controller$/ {print $1}' " - "| wc -l)\" -eq 1", - timeout=30, - rows=[ - ( - "controller processes", - "ps -eo pid=,args= | grep acceptance-controller || true", - ), - ], - explain=[("d2bd.service", "acceptance-controller")], - ) - controller_pid_before = machine.succeed( - "ps -eo pid=,args= | awk '$NF ~ /acceptance-controller$/ {print $1; exit}'" - ).strip() - - # The debug surface, on a zone this fixture has just settled. alice can - # read Process, Host and User but not Zone, so the report is expected to - # name exactly those types it could not read rather than present them as - # empty, and to still explain the rows it did read. - diag_step( - "debug-surface-zone-report", - lambda: machine.succeed( - "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - "d2b --zone work --json debug work >/run/d2b-debug-zone.json && " - "jq -e '.zoneRef == \"Zone/work\" " - "and (.degradedReads | map(.resourceType) | index(\"Zone\")) != null " - "and (.degradedReads | map(.resourceType) | index(\"Process\")) == null " - "and (.degradedReads | map(.resourceType) | index(\"Host\")) == null' " - "/run/d2b-debug-zone.json >/dev/null && " - "jq -e '[.roots[].ref] | index(\"Host/host-system\") != null " - "and index(\"User/alice\") != null' " - "/run/d2b-debug-zone.json >/dev/null" - ), - rows=[live_rows("Process rows", "Process")], - explain=[("d2bd.service", "acceptance-controller")], - ) - diag_step( - "debug-surface-named-row", - lambda: machine.succeed( - "name=$(runuser -u alice -- env " - "D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - "d2b --zone work --json list Process | " - "jq -r '.resources[] | select(.metadata.name | startswith(" - "\"controller-\")) | .metadata.name') && " - "test -n \"$name\" && " - "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - "d2b --zone work --json debug work \"Process/$name\" " - ">/run/d2b-debug-row.json && " - "jq -e '.roots | length == 1' /run/d2b-debug-row.json >/dev/null && " - "jq -e --arg name \"Process/$name\" " - "'.roots[0].ref == $name and .roots[0].phase == \"Ready\" " - "and (.roots[0].children | length == 0)' " - "/run/d2b-debug-row.json >/dev/null" - ), - rows=[live_rows("Process rows", "Process")], - explain=[("d2bd.service", "acceptance-controller")], - ) - diag_step( - "debug-surface-human-report", - lambda: machine.succeed( - # No TTY in the lane, so the human tree needs the explicit flag; - # this is the one place the tree renderer is exercised live. - "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - "d2b --zone work --human debug work >/run/d2b-debug-human.txt && " - "grep -F 'zone work rows=' /run/d2b-debug-human.txt >/dev/null && " - "grep -F 'Host/host-system' /run/d2b-debug-human.txt >/dev/null && " - "grep -F 'type Zone not read' /run/d2b-debug-human.txt >/dev/null" - ), - rows=[live_rows("Process rows", "Process")], - explain=[("d2bd.service", "acceptance-controller")], - ) - machine.fail( - "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - "d2b --zone work debug work Process/absent-row >/dev/null 2>&1" - ) - - machine.fail( - "runuser -u bob -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - "d2b --zone work --json list Process " - ">/run/d2b-unauthorized-resource.log 2>&1" - ) - - stage("restart") - machine.succeed("systemctl restart d2bd.service") - diag_unit("daemon-restarted", "d2bd.service", 180) - machine.wait_for_file("/run/d2b/public.sock", timeout=30) - diag_wait( - "process-adopted-after-restart", - "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - "d2b --zone work --json list Process " - ">/run/d2b-process-after.json && " - "test \"$(ps -eo pid=,args= | awk '$NF ~ /acceptance-controller$/ {print $1}' " - "| wc -l)\" -eq 1 && " - "jq -e --slurpfile before /run/d2b-process-before.json " - "'([.resources[] | select(.type == \"Process\" and " - ".metadata.ownerRef == \"Provider/network-local\")] | length == 1) and " - "(.resources[] | select(.type == \"Process\" and " - ".metadata.ownerRef == \"Provider/network-local\") as $after | " - "($before[0].resources[] | select(.type == \"Process\" and " - ".metadata.ownerRef == \"Provider/network-local\")) as $old | " - "($after.metadata.uid == $old.metadata.uid and " - "$after.metadata.generation == $old.metadata.generation and " - "$after.status.phase == \"Ready\" and " - "$after.status.observedGeneration == $after.metadata.generation))' " - "/run/d2b-process-after.json", - timeout=60, - rows=[ - saved_rows("Process rows", "/run/d2b-process-after.json"), - saved_rows("Pre-restart Process rows", "/run/d2b-process-before.json"), - ], - explain=[("d2bd.service", "network-local")], - ) - controller_pid_after = machine.succeed( - "ps -eo pid=,args= | awk '$NF ~ /acceptance-controller$/ {print $1; exit}'" - ).strip() - assert controller_pid_after == controller_pid_before, ( - f"controller PID changed across d2bd restart: " - f"{controller_pid_before} -> {controller_pid_after}" - ) - machine.succeed("date +%s >/run/d2b-resource-restart-observed-at") - diag_wait( - "process-resynced-after-restart", - "test $(( $(date +%s) - $(cat /run/d2b-resource-restart-observed-at) )) -ge 20 && " - "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - "d2b --zone work --json list Process " - ">/run/d2b-process-after-resync.json && " - "test \"$(ps -eo pid=,args= | awk '$NF ~ /acceptance-controller$/ {print $1}' " - "| wc -l)\" -eq 1 && " - "jq -e '([.resources[] | select(.type == \"Process\" and " - ".metadata.ownerRef == \"Provider/network-local\")] | length == 1) and " - "(.resources[] | select(.type == \"Process\" and " - ".metadata.ownerRef == \"Provider/network-local\") | " - "(.metadata.uid != null and .metadata.generation > 0 and " - ".status.phase == \"Ready\" and " - ".status.observedGeneration == .metadata.generation))' " - "/run/d2b-process-after-resync.json", - timeout=60, - rows=[ - saved_rows("Process rows", "/run/d2b-process-after-resync.json"), - ( - "controller processes", - "ps -eo pid=,args= | grep acceptance-controller || true", - ), - ], - explain=[("d2bd.service", "network-local")], - ) - machine.succeed("runuser -u alice -- d2b auth status --json >/run/d2b-auth-after.json") - machine.succeed( - "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - "d2b --zone work --json list Host " - ">/run/d2b-host-after.json && " - "jq -e --slurpfile before /run/d2b-host-before.json " - "'.resources[] | select(.type == \"Host\" and " - ".metadata.name == \"host-system\") as $after | " - "($before[0].resources[] | select(.type == \"Host\" and " - ".metadata.name == \"host-system\")) as $old | " - "($after.metadata.uid == $old.metadata.uid and " - "$after.metadata.generation == $old.metadata.generation and " - "$after.metadata.revision >= $old.metadata.revision and " - "$after.status.phase == \"Ready\")' /run/d2b-host-after.json" - ) - - declared = set( - machine.succeed("cat /etc/d2b/daemon-acceptance-units").split() - ) - required = { - "d2bd.service", - "d2b-broker.socket", - "d2b-broker.service", - } - assert declared == required, ( - f"unexpected framework acceptance census: {declared}" - ) - unit_names = set( - machine.succeed( - "systemctl list-units --no-pager --all --plain " - "| awk '{print $1}' | sort" - ).split() - ) - assert required <= unit_names, ( - f"framework daemon units missing: {required - unit_names}" - ) - - # Provider packages are code loaded by d2bd, never framework-declared - # persistent services. Optional or managed host units are outside this - # fixture's census. - provider_units = sorted( - unit - for unit in declared - if "provider" in unit and (unit.endswith(".service") or unit.endswith(".socket")) - ) - assert not provider_units, f"Provider-owned persistent units found: {provider_units}" - ''; -} diff --git a/tests/host-integration/state-posture-contract.nix b/tests/host-integration/state-posture-contract.nix deleted file mode 100644 index 4d4abe7ca..000000000 --- a/tests/host-integration/state-posture-contract.nix +++ /dev/null @@ -1,799 +0,0 @@ -# Type-G runNixOSTest: declared host posture contract for state farms and -# shared directories (issue #512). -# -# One file declares the posture (`packages/d2b-broker/src/ops/ -# state-posture-contract.json`); the broker posture code embeds it, the Nix -# provisioning derives from it, and this fixture asserts the live host against -# it: every declared level's owner/group/mode/ACL, plus the allowed AND denied -# operations for each principal (root, d2bd, a d2b-group launcher, and nobody). -# The fixture boots the same Zone-native Cloud Hypervisor Guest recipe the -# acceptance fixture uses, so the state chain, the store-view farm, and the -# spawn-time traversal ACLs are all materialized by the product path. -{ pkgs, self }: - -let - inherit (pkgs) lib; - d2bLib = import ./lib.nix { - inherit self; - inherit lib; - hostToolBundle = - if self.lib ? d2bHostToolBundle then self.lib.d2bHostToolBundle else null; - }; - # The reusable guest configuration lives outside this directory so it - # survives the fixture (see `nix/test-support/host-integration-node.nix`). - d2bNode = import ../../nix/test-support/host-integration-node.nix { - inherit self; - inherit lib; - }; - cloudHypervisorArtifact = - d2bLib.mkRuntimeCloudHypervisorArtifact pkgs; - volumeProviderArtifact = d2bLib.mkVolumeProviderArtifact pkgs; - fixtureKeys = pkgs.runCommand "acceptance-component-session-keys" { } '' - mkdir -p "$out" - printf '\001\002\003\004\005\006\007\010\011\012\013\014\015\016\017\020\021\022\023\024\025\026\027\030\031\032\033\034\035\036\037\040' > "$out/host.key" - printf '\007\243\174\274\024\040\223\310\267\125\334\033\020\350\154\264\046\067\112\321\152\250\123\355\013\337\300\262\270\155\034\174' > "$out/host.pub" - printf '\041\042\043\044\045\046\047\050\051\052\053\054\055\056\057\060\061\062\063\064\065\066\067\070\071\072\073\074\075\076\077\100' > "$out/guest.key" - printf '\130\151\257\364\120\124\227\062\313\252\355\136\135\371\263\012\155\243\034\260\345\164\053\255\132\324\241\247\150\361\246\173' > "$out/guest.pub" - ''; - guestBundle = pkgs.runCommand "acceptance-guest-bundle" { - nativeBuildInputs = [ pkgs.python3 ]; - } '' - mkdir -p "$out" - cat > "$out/host.json" <<'EOF' - {"schemaVersion":"v2","site":{"allowUnsafeEastWest":false},"environments":[],"nftables":{"family":"inet","table":"d2b","chains":[],"tableHashAfterApply":null,"ownershipId":"host-integration"},"networkManager":{"filePath":"/etc/NetworkManager/conf.d/00-d2b-unmanaged.conf","matchCriteria":[],"reloadBehavior":"atomic-reload","ownership":{"owner":"root","group":"root","mode":"0644","driftPolicy":"replace"}},"hostsFile":{"startMarker":"# d2b-managed begin","endMarker":"# d2b-managed end","rule":"replace-managed-block"},"kernelModules":[],"fdOwnership":[],"cloudHypervisorCapabilities":[],"ifNameMappings":[],"ch":null,"firewallCoexistencePolicy":null} - EOF - printf '%s\n' '{"schemaVersion":"v2","vms":[]}' > "$out/processes.json" - printf '%s\n' '{"schemaVersion":"v2","publicOperations":[],"brokerOperations":[]}' > "$out/privileges.json" - printf '%s\n' '{"_manifest":{"manifestVersion":6},"_observability":{"enabled":false,"signozUrl":"http://127.0.0.1:8080","signozOtlpGrpcPort":4317,"signozOtlpHttpPort":4318,"obsVsockCid":0,"obsVsockHostSocket":"","vmName":""}}' > "$out/vms.json" - python3 - "$out/bundle.json" <<'PY' - import hashlib - import json - import sys - - # Zone-native v3 bundle: the loader (BundleResolver) accepts only the - # v3 contract. The self-hash is computed over the serialization with - # bundleHash absent and artifactHashes nullified (verify_bundle_hash). - bundle = { - "artifactHashes": {}, - "bundleVersion": 1, - "schemaVersion": "v3", - "privilegesPath": "privileges.json", - "zones": [], - "generation": { - "generatedAt": None, - "generator": "host-integration", - "sourceRevision": None, - }, - } - preimage = dict(bundle) - preimage["artifactHashes"] = None - canonical = json.dumps(preimage, sort_keys=True, separators=(",", ":")).encode() - bundle["bundleHash"] = "sha256:" + hashlib.sha256(canonical).hexdigest() - with open(sys.argv[1], "w", encoding="utf-8") as output: - json.dump(bundle, output, sort_keys=True, separators=(",", ":")) - output.write("\n") - PY - ''; - - cloudHypervisorConfig = { - controllerExecutionRef = "Host/host-system"; - defaultVcpus = 2; - defaultMemoryMb = 512; - defaultMachineType = "microvm"; - watchdog = true; - adoptionWindowMs = 30000; - healthCheckIntervalMs = 5000; - healthCheckTimeoutMs = 1000; - healthCheckFailureThreshold = 3; - startupDeadlineMs = 120000; - }; - guestSystem = d2bLib.mkGuestSystem { - inherit pkgs; - name = "acceptance-guest"; - modules = [ - ({ lib, name, ... }: { - boot.kernelParams = [ "console=ttyS0" "loglevel=7" ]; - environment.etc."d2b/component-session/guest.key".source = - "${fixtureKeys}/guest.key"; - environment.etc."d2b/component-session/parent.pub".source = - "${fixtureKeys}/host.pub"; - systemd.services.d2bd-guest = { - environment = { - RUST_LOG = "d2bd=debug"; - }; - serviceConfig = { - ReadOnlyPaths = [ - "/etc/d2b/component-session/guest.key" - "/etc/d2b/component-session/parent.pub" - ]; - StandardOutput = lib.mkForce "journal+console"; - StandardError = lib.mkForce "journal+console"; - }; - }; - systemd.services.d2b-test-boot-identity = { - wantedBy = [ "basic.target" ]; - before = [ "d2bd-guest.service" ]; - serviceConfig.Type = "oneshot"; - script = '' - printf 'D2B_GUEST_BOOT_ID=%s\n' \ - "$(${pkgs.coreutils}/bin/cat /proc/sys/kernel/random/boot_id)" \ - > /dev/console - ''; - }; - d2b.componentSession.localPrivateKeyPath = - "/etc/d2b/component-session/guest.key"; - d2b.componentSession.parentPublicKeyPath = - "/etc/d2b/component-session/parent.pub"; - d2b.componentSession.bundlePath = - "/var/lib/d2b/guest-bundle/bundle.json"; - d2b.guestBroker.bundlePath = - "/var/lib/d2b/guest-bundle/bundle.json"; - systemd.services.d2b-install-guest-bundle = { - requiredBy = [ "d2b-broker-guest.service" "d2bd-guest.service" ]; - before = [ "d2b-broker-guest.service" "d2bd-guest.service" ]; - serviceConfig.Type = "oneshot"; - script = '' - install -d -o root -g d2bd -m 0750 /var/lib/d2b/guest-bundle - for file in bundle.json host.json processes.json privileges.json; do - install -o root -g d2bd -m 0640 \ - ${guestBundle}/"$file" /var/lib/d2b/guest-bundle/"$file" - done - install -o root -g d2bd -m 0644 \ - ${guestBundle}/vms.json /var/lib/d2b/guest-bundle/vms.json - ''; - }; - networking.useDHCP = lib.mkForce false; - networking.networkmanager.enable = lib.mkForce false; - systemd.network.enable = lib.mkForce false; - services.dbus.enable = lib.mkForce false; - services.resolved.enable = lib.mkForce false; - systemd.services.systemd-vconsole-setup.enable = false; - d2b.vms.${name}.runner = { - store.onDisk = true; - store.disk = guestStoreDisk; - shares = lib.mkForce [ ]; - }; - fileSystems."/nix/store" = { - device = "/dev/vda"; - fsType = "ext4"; - options = [ "ro" "x-initrd.mount" ]; - neededForBoot = true; - }; - }) - ]; - }; - guestClosure = pkgs.closureInfo { - rootPaths = [ guestSystem.config.system.build.toplevel ]; - }; - guestStoreDisk = pkgs.runCommand "acceptance-guest-store.img" { - nativeBuildInputs = [ pkgs.coreutils pkgs.e2fsprogs ]; - } '' - mkdir -p root - while IFS= read -r path; do - cp -r --no-preserve=ownership,xattr,context "$path" root/ - done < ${guestClosure}/store-paths - truncate -s 4096M "$out" - # Reproducible ext4 image: SOURCE_DATE_EPOCH pins the superblock times and - # a fixed UUID seed pins the htree hash seed (e2fsprogs ignores an all-zero - # seed and randomizes it), so every build is byte-identical. With a random - # seed each build differed, and the nixos-install closure spec (recorded - # from an earlier build) could never match the freshly built image. - SOURCE_DATE_EPOCH=0 mkfs.ext4 -q -F \ - -U 123e4567-e89b-12d3-a456-426614174000 \ - -E hash_seed=123e4567-e89b-12d3-a456-426614174000 \ - -d root "$out" - ''; - artifacts = { - runtime-cloud-hypervisor = { - inherit (cloudHypervisorArtifact) package type catalog; - }; - volume-acceptance-provider = { - inherit (volumeProviderArtifact) package type catalog; - }; - acceptance-system = { - package = guestSystem.config.system.build.toplevel; - type = "nixos-system"; - }; - }; -in -pkgs.testers.runNixOSTest { - name = "d2b-state-posture-contract"; - - nodes.machine = d2bNode.d2bCloudHypervisorNode { - extra = { ... }: { - d2b.site.adminUsers = [ "alice" ]; - environment.systemPackages = with pkgs; [ - acl - iproute2 - jq - iputils - procps - util-linux - ]; - d2b.artifacts = artifacts; - # The declaration under test, installed verbatim from the repo so the - # fixture reads the same file the posture code embeds. - environment.etc."d2b/state-posture-contract.json".source = - ../../packages/d2b-broker/src/ops/state-posture-contract.json; - d2b.guestSystems.work.acceptance-guest = guestSystem; - d2b.zones.local-root.trustedPublishers.d2b-cloud-hypervisor.signingKey = - cloudHypervisorArtifact.trustedPublisher.signingKey; - d2b.zones.local-root.trustedPublishers.d2b-volume-acceptance.signingKey = - volumeProviderArtifact.trustedPublisher.signingKey; - d2b.zones.work.trustedPublishers.d2b-cloud-hypervisor.signingKey = - cloudHypervisorArtifact.trustedPublisher.signingKey; - d2b.zones.work.trustedPublishers.d2b-volume-acceptance.signingKey = - volumeProviderArtifact.trustedPublisher.signingKey; - d2b.zones.local-root.resources.host-system = { - type = "Host"; - spec = { - providerRef = "Provider/system-core"; - defaultDomain = "system"; - allowedDomains = [ "system" ]; - budget = { }; - networkAttachments = [ ]; - deviceAttachments = [ ]; - volumeAttachmentDefaults = [ ]; - }; - }; - d2b.zones.work = { - parentZone = "local-root"; - resources = { - alice = { - type = "User"; - spec = { - displayName = "Alice"; - groups = [ ]; - osUsername = "alice"; - }; - }; - d2bd = { - type = "User"; - spec = { - displayName = "d2bd"; - groups = [ ]; - osUsername = "d2bd"; - }; - }; - lifecycle-operator = { - type = "Role"; - spec.rules = [ - { - resourceTypes = [ "Endpoint" "Guest" "Host" "Process" "Provider" "Volume" "VolumeBinding" ]; - verbs = [ "get" "list" ]; - subresources = [ ]; - resourceNames = [ ]; - zones = [ "work" ]; - executionRefs = [ ]; - sessionVerbs = [ "connect" "invoke" ]; - } - { - resourceTypes = [ "Guest" ]; - verbs = [ "delete" ]; - subresources = [ ]; - resourceNames = [ "acceptance-guest" ]; - zones = [ "work" ]; - executionRefs = [ ]; - sessionVerbs = [ "connect" "invoke" ]; - } - { - resourceTypes = [ "Volume" ]; - verbs = [ "delete" ]; - subresources = [ ]; - resourceNames = [ "state" ]; - zones = [ "work" ]; - executionRefs = [ ]; - sessionVerbs = [ "connect" "invoke" ]; - } - ]; - }; - lifecycle-operator-binding = { - type = "RoleBinding"; - spec = { - roleRef = "Role/lifecycle-operator"; - subjects = [ "User/alice" ]; - externalPrincipalSelector = null; - scopeNarrowing = null; - }; - }; - host-system = { - type = "Host"; - spec = { - providerRef = "Provider/system-core"; - defaultDomain = "system"; - allowedDomains = [ "system" ]; - budget = { }; - networkAttachments = [ ]; - deviceAttachments = [ ]; - volumeAttachmentDefaults = [ ]; - }; - }; - volume-local = { - type = "Provider"; - spec = { - artifactId = "volume-acceptance-provider"; - config = { - controllerExecutionRef = "Host/host-system"; - sourcePolicies = [ - { - id = "default-state"; - class = "local-path"; - volumeKinds = [ "durable" "state" "cache" ]; - } - # U7: daemon-owned root the unprivileged daemon can - # lock and provision inline (path:daemon-state). - { - id = "daemon-state"; - class = "local-path"; - volumeKinds = [ "durable" "state" "cache" ]; - } - ]; - }; - }; - }; - volume-virtiofs = { - type = "Provider"; - spec = { - artifactId = "volume-acceptance-provider"; - config.controllerExecutionRef = "Host/host-system"; - }; - }; - state = { - type = "Volume"; - spec = { - providerRef = "Provider/volume-local"; - kind = "state"; - source = { - executionRef = "Host/host-system"; - settings = { - kind = "local-path"; - sourcePolicyId = "daemon-state"; - }; - }; - layout = [{ - path = "state"; - type = "directory"; - # U7: daemon-owned so the unprivileged daemon can - # provision inline; the guest share stays read-only. - ownerRef = "User/d2bd"; - groupRef = "User/d2bd"; - mode = "0700"; - target = null; - accessAcl = [ ]; - defaultAcl = [ ]; - foreignChildPolicy = "preserve"; - noFollow = true; - recursive = false; - sensitivity = "private"; - createPolicy = "create-if-never-provisioned"; - repairPolicy = "exact-owner"; - cleanupPolicy = "owner-controlled"; - adoptionPolicy = "quarantine-on-ambiguity"; - restartPolicy = "preserve-across-controller-restart"; - leaseClass = "none"; - invariants = [ "no-symlink" ]; - }]; - views.controller = { - path = ""; - rights = [ "read" "write" "traverse" ]; - }; - # KTD1: the attachment stays declared input only. The Volume - # side mints the durable VolumeBinding at reconcile; the - # deterministic binding identity below is - # vol-binding-6a8ea4307a30f7ceae6533f2 (volume, execution - # target, view, mount path). - attachments = [{ - executionRef = "Guest/acceptance-guest"; - transport = "virtiofs"; - view = "controller"; - access = "read-only"; - mountPath = "/state"; - settings = { - posixAcl = false; - xattr = false; - cache = "auto"; - inodeFileHandles = "never"; - threadPoolSize = null; - socketGroup = null; - }; - }]; - }; - }; - runtime-cloud-hypervisor = { - type = "Provider"; - spec = { - artifactId = "runtime-cloud-hypervisor"; - config = cloudHypervisorConfig; - }; - }; - acceptance-guest = { - type = "Guest"; - spec = { - providerRef = "Provider/runtime-cloud-hypervisor"; - executionRef = "Host/host-system"; - systemArtifactId = "acceptance-system"; - defaultDomain = "system"; - allowedDomains = [ "system" ]; - budget = { }; - volumeAttachmentDefaults = [ ]; - networkAttachments = [ ]; - deviceAttachments = [ ]; - }; - }; - }; - }; - }; - }; - - testScript = '' - ${d2bLib.fixtureDiagnostics} - - import json as _json - import shlex as _shlex - - ZONE = "work" - GUEST = "acceptance-guest" - STATE_ROOT = "/var/lib/d2b" - PRINCIPAL_USER = { - "root": "root", - "d2bd": "d2bd", - "d2b": "alice", - "nobody": "nobody", - } - - def check(condition, message): - if not condition: - raise AssertionError(message) - - def tokens(): - return { - "state-root": STATE_ROOT, - "zone": ZONE, - "guest": GUEST, - "vm": GUEST, - } - - def substitute(value): - for token, replacement in tokens().items(): - value = value.replace("<" + token + ">", replacement) - return value - - def tree(tree_id): - matches = [entry for entry in CONTRACT["trees"] if entry["id"] == tree_id] - check(len(matches) == 1, "contract tree " + tree_id + " must exist exactly once") - return matches[0] - - def level_path(entry, level): - root = substitute(entry["root"]) - if level["path"] == ".": - return root - return root + "/" + substitute(level["path"]) - - def stat_row(path): - output = machine.succeed("stat -c '%U %G %a' " + _shlex.quote(path)) - owner, group, mode = output.split() - return owner, group, mode - - # Observed ACL entries per level, cached for the whole fixture: the - # spawn-preflight carve-out below is structural (a worker's ancestor - # traversal entries are expected because of its deeper leaf grant), so it - # cannot be decided one level at a time. - observed_acls = {} - - def acl_entries(path): - if path not in observed_acls: - output = machine.succeed( - "getfacl -cp " + _shlex.quote(path) + " 2>/dev/null || true" - ) - entries = set() - for line in output.splitlines(): - parts = line.strip().split(":") - if len(parts) != 3: - continue - kind, name, permissions = parts - short = {"user": "u", "group": "g", "other": "o", "mask": "m"}.get(kind) - if short is not None: - entries.add(short + ":" + name + ":" + permissions) - observed_acls[path] = entries - return observed_acls[path] - - def named_entry(spec): - parts = spec.split(":", 2) - return len(parts) == 3 and parts[0] in ("u", "g") and parts[1] != "" - - def permission_bits(permissions): - return {bit for bit in "rwx" if bit in permissions} - - def level_exists(path): - return machine.execute("test -e " + _shlex.quote(path))[0] == 0 - - def spawn_preflight_entries(): - """Named entries the spawn preflight is expected to add, per level. - - The broker's spawn preflight opens the ancestor chain above a - runner-owned tree with search (`u::--x`) and grants the runner - its own leaf (`rwx` for a private state tree, `r-x` for a read-only - served view root): `runner_tree_acl_targets` in - packages/d2b-broker/src/live_handlers.rs, reached from - `refresh_spawn_runner_acls` / `grant_serving_worker_launch_acls` / - `grant_device_worker_launch_acls`. The runner principals are uids - minted per Guest at runtime, so the declaration cannot name them; - the fixture derives them from the live worker processes and then - allows an undeclared entry only in that structural shape - `--x` on a - level that has a deeper grant of the same uid, or the uid's own - topmost grant (`--x` when its leaf is outside the checked levels, - else the leaf spelling). A foreign uid, or a wider grant on a level - above the worker's own leaf, still fails. - """ - worker_uids = set() - for line in machine.succeed("ps -eo uid=,comm= --no-headers").splitlines(): - uid, _, comm = line.strip().partition(" ") - if comm.startswith("cloud-hyperviso") or comm.startswith("virtiofsd"): - worker_uids.add(uid) - allowed = {} - for uid in worker_uids: - prefix = "u:" + uid + ":" - grant_paths = [ - path - for path, entries in observed_acls.items() - if any(entry.startswith(prefix) for entry in entries) - ] - for path in grant_paths: - prefix_path = path.rstrip("/") + "/" - deeper = any( - other != path and other.startswith(prefix_path) - for other in grant_paths - ) - spellings = {prefix + "--x"} if deeper else { - prefix + "--x", - prefix + "rwx", - prefix + "r-x", - } - allowed.setdefault(path, set()).update(spellings) - return allowed - - def run_as(principal, command): - uid, gid = PRINCIPAL_IDS[principal] - status, _ = machine.execute( - "setpriv --reuid=" - + uid - + " --regid=" - + gid - + " --init-groups /bin/sh -c " - + _shlex.quote(command) - ) - return status == 0 - - def probe(principal, right, path): - flag = {"traverse": "x", "read": "r", "write": "w"}[right] - return run_as(principal, "test -" + flag + " " + _shlex.quote(path)) - - def check_level(entry, level): - path = level_path(entry, level) - where = entry["id"] + ":" + level["path"] + " (" + path + ")" - present = machine.execute("test -e " + _shlex.quote(path))[0] == 0 - if not present: - check( - not level.get("required", True), - "declared level is missing: " + where, - ) - return - - owner, group, mode_text = stat_row(path) - mode = int(mode_text, 8) - policy = level.get("modePolicy", "exact") - if policy == "exact": - check( - mode == int(level["mode"], 8) & 0o7777, - "mode drift at " + where + ": declared " + level["mode"] - + ", observed " + mode_text, - ) - check( - owner == level["owner"], - "owner drift at " + where + ": declared " + level["owner"] - + ", observed " + owner, - ) - check( - group == level["group"], - "group drift at " + where + ": declared " + level["group"] - + ", observed " + group, - ) - elif policy == "group-traverse-minimum": - check( - owner == level["owner"], - "owner drift at " + where + ": declared " + level["owner"] - + ", observed " + owner, - ) - check( - group == level["group"], - "group drift at " + where + ": declared " + level["group"] - + ", observed " + group, - ) - check(mode & 0o010 != 0, "group search missing at " + where) - check(mode & 0o020 == 0, "group write must never be granted at " + where) - else: - check( - policy == "preserve-existing", - "unknown modePolicy " + policy + " at " + where, - ) - - entries = acl_entries(path) - declared_acl = [declared["spec"] for declared in level.get("acl", [])] - for spec in declared_acl: - check( - spec in entries, - "declared ACL missing at " + where + ": " + spec, - ) - - # A declaration pins its named entries completely: an entry nobody - # declared is drift, not a harmless extra. The only undeclared - # entries the live host may carry are the spawn-preflight traversal - # grants derived for the runner uids (`spawn_preflight_entries`); the - # base entries (u::/g::/o::/m::) are pinned only when the - # declaration names them. - declared_named = {spec for spec in declared_acl if named_entry(spec)} - observed_named = {entry for entry in entries if named_entry(entry)} - expected_named = declared_named | spawn_entries.get(path, set()) - undeclared = sorted(observed_named - expected_named) - missing = sorted(declared_named - observed_named) - check( - not undeclared, - "undeclared ACL entry at " + where + ": " + ", ".join(undeclared), - ) - check( - not missing, - "declared ACL entry missing at " + where + ": " + ", ".join(missing), - ) - - # An unpinned mask must be the union of the group entry and the named - # grants (setfacl semantics): a mask that drifts from that silently - # re-scopes the whole group class. - declared_mask = [spec for spec in declared_acl if spec.split(":", 2)[0] == "m"] - mask_entry = next( - (entry for entry in entries if entry.startswith("m::")), None - ) - if not declared_mask and mask_entry is not None: - group_entry = next( - (entry for entry in entries if entry.startswith("g::")), None - ) - expected_mask = ( - permission_bits(group_entry.split(":", 2)[2]) - if group_entry - else set() - ) - for spec in observed_named: - expected_mask |= permission_bits(spec.split(":", 2)[2]) - check( - permission_bits(mask_entry.split(":", 2)[2]) == expected_mask, - "ACL mask drift at " + where + ": expected " - + "".join(bit for bit in "rwx" if bit in expected_mask) - + " from the group entry and named grants, observed " - + mask_entry, - ) - - for principal, rights in level["rights"].items(): - for right in ("traverse", "read", "write"): - expectation = rights.get(right, "preserve") - if expectation in ("preserve", "not-required"): - continue - if principal == "root": - # root bypasses DAC; its allow rows are documentation. - continue - observed = probe(principal, right, path) - check( - observed == (expectation == "allow"), - where + ": " + principal + " " + right + " expected " - + expectation + ", observed " - + ("allow" if observed else "deny"), - ) - - start_all() - stage("daemon-up") - - # The declared contract, from the same file the broker posture code embeds - # and nixos-modules/host-daemon.nix derives its provisioning from. This - # fixture never restates a posture value; it reads the declaration. - CONTRACT = _json.loads( - machine.succeed("cat /etc/d2b/state-posture-contract.json") - ) - - diag_unit("daemon-up", "d2bd.service", 180) - machine.wait_for_unit("d2b-broker.socket", timeout=30) - machine.wait_for_file("/run/d2b/public.sock", timeout=30) - machine.succeed("systemctl start d2b-broker.service") - diag_unit("broker-service", "d2b-broker.service", 30) - - PRINCIPAL_IDS = { - principal: ( - machine.succeed("id -u " + user).strip(), - machine.succeed("id -g " + user).strip(), - ) - for principal, user in PRINCIPAL_USER.items() - } - - guest_state = STATE_ROOT + "/zones/" + ZONE + "/guests/" + GUEST - store_view = guest_state + "/store-view" - - stage("store-view-sync") - diag_wait( - "store-view-sync", - "test -L " + store_view + "/state/current && test -L " - + store_view + "/meta/current", - 300, - rows=[ - ( - "guest state tree", - "find " + guest_state + " -maxdepth 1 -exec stat -c '%A %U %G %n' {} + 2>/dev/null | sort || true", - ), - ( - "store-view tree", - "find " + store_view + " -maxdepth 2 -exec stat -c '%A %U %G %n' {} + 2>/dev/null | sort || true", - ), - ], - explain=[("d2bd.service", "store"), ("d2b-broker.service", "StoreSync")], - ) - - stage("vmm-spawn") - diag_wait( - "vmm-spawn", - "test -S " + guest_state + "/" + GUEST + ".sock", - 300, - rows=[ - ( - "guest state tree", - "find " + guest_state + " -maxdepth 1 -exec stat -c '%A %U %G %n' {} + 2>/dev/null | sort || true", - ), - ], - explain=[ - ("d2bd.service", "cloud-hypervisor"), - ("d2bd.service", "component-session"), - ], - ) - - stage("posture-contract") - CHECKED_TREES = ( - "state-root", - "guest-state-chain", - "guest-state-dir", - "guest-store-view", - "shared-run-dir", - ) - # Observe every checked level before comparing: the spawn-preflight - # carve-out relates a worker's ancestor traversal entries to the leaf - # grant below them, so the expected set cannot be decided level by level. - for tree_id in CHECKED_TREES: - entry = tree(tree_id) - for level in entry["levels"]: - checked_path = level_path(entry, level) - if level_exists(checked_path): - acl_entries(checked_path) - spawn_entries = spawn_preflight_entries() - for tree_id in CHECKED_TREES: - entry = tree(tree_id) - for level in entry["levels"]: - check_level(entry, level) - - stage("anchor-open-rule") - # The guest start above ran the daemon's anchored store-view walk while the - # chain was capped at search-only by the spawn-time u:d2bd:--x ACL. Two - # live proofs: the store-view open never failed, and the resolved view - # reached a virtiofsd worker through the daemon's fd handoff. - machine.fail( - "journalctl -u d2bd.service --no-pager -b -n 5000 " - "| grep -F 'store-view-open'" - ) - check( - machine.execute("pgrep -x virtiofsd >/dev/null")[0] == 0, - "the store-view directory must reach a virtiofsd worker", - ) - # Live denial, from the same contract rows: the daemon may search the - # per-Guest state dir it does not own but may not read it. - check( - not run_as("d2bd", "ls " + _shlex.quote(guest_state) + " >/dev/null 2>&1"), - "the daemon must not read the per-Guest state dir it only traverses", - ) - - stage("done") - print("[d2b] declared state posture contract holds on the live host") - ''; -} diff --git a/tests/host-integration/wayland-proxy.nix b/tests/host-integration/wayland-proxy.nix deleted file mode 100644 index 8781521ab..000000000 --- a/tests/host-integration/wayland-proxy.nix +++ /dev/null @@ -1,177 +0,0 @@ -# Type-G runNixOSTest: live Wayland proxy AF_UNIX relay. -# -# Boots a minimal NixOS node and runs d2b-wayland-proxy against a fake Wayland -# compositor socket. This covers the live client-to-upstream relay path and -# socket posture that unit tests cannot exercise; rendered d2b DAG wiring remains -# covered by the graphics smoke/eval cases. -{ pkgs, self }: - -let - # Shared fixture diagnostics (issue #513): row dumps and per-stage markers. - d2bLib = import ./lib.nix { - inherit self; - inherit (pkgs) lib; - }; - proxyPackage = self.packages.${pkgs.stdenv.hostPlatform.system}.d2b-wayland-proxy; -in -pkgs.testers.runNixOSTest { - name = "d2b-wayland-proxy"; - - nodes.machine = { - users.users.alice = { - isNormalUser = true; - uid = 1000; - }; - - environment.systemPackages = [ - pkgs.python3 - proxyPackage - ]; - - system.stateVersion = "25.11"; - }; - - testScript = '' - ${d2bLib.fixtureDiagnostics} - - start_all() - stage("boot") - machine.wait_for_unit("multi-user.target", timeout=180) - - stage("fake-upstream") - machine.succeed("install -d -m 0700 -o alice -g users /run/d2b-wayland-proxy-test") - machine.succeed( - "cat > /run/d2b-wayland-proxy-test/fake-upstream.py <<'PY'\n" - "import os, select, socket, time\n" - "path = '/run/d2b-wayland-proxy-test/upstream.sock'\n" - "ready = '/run/d2b-wayland-proxy-test/upstream.ready'\n" - "seen = '/run/d2b-wayland-proxy-test/upstream.seen'\n" - "for p in (path, ready, seen):\n" - " try:\n" - " os.unlink(p)\n" - " except FileNotFoundError:\n" - " pass\n" - "srv = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)\n" - "srv.bind(path)\n" - "os.chmod(path, 0o600)\n" - "srv.setblocking(False)\n" - "srv.listen(8)\n" - "open(ready, 'w').write('ready')\n" - "connections = []\n" - "deadline = time.monotonic() + 60\n" - "while time.monotonic() < deadline:\n" - " readable = [srv] + connections\n" - " ready, _, _ = select.select(readable, [], [], 0.2)\n" - " for sock in ready:\n" - " if sock is srv:\n" - " conn, _ = srv.accept()\n" - " conn.setblocking(False)\n" - " connections.append(conn)\n" - " else:\n" - " data = sock.recv(12)\n" - " if data:\n" - " open(seen, 'wb').write(data)\n" - " deadline = 0\n" - " break\n" - " connections.remove(sock)\n" - " sock.close()\n" - "for conn in connections:\n" - " conn.close()\n" - "srv.close()\n" - "PY\n" - "chown alice:users /run/d2b-wayland-proxy-test/fake-upstream.py" - ) - machine.succeed( - "runuser -u alice -- python3 /run/d2b-wayland-proxy-test/fake-upstream.py " - ">/run/d2b-wayland-proxy-test/upstream.log 2>&1 & " - "echo $! > /run/d2b-wayland-proxy-test/upstream.pid" - ) - diag_step( - "upstream-ready", - lambda: machine.wait_for_file( - "/run/d2b-wayland-proxy-test/upstream.ready", timeout=30 - ), - rows=[ - ( - "upstream log", - "cat /run/d2b-wayland-proxy-test/upstream.log " - "2>/dev/null || true", - ), - ( - "test dir", - "ls -la /run/d2b-wayland-proxy-test 2>&1 || true", - ), - ], - ) - - stage("proxy-start") - machine.succeed( - "runuser -u alice -- env XDG_RUNTIME_DIR=/run/d2b-wayland-proxy-test " - "d2b-wayland-proxy " - "--listen /run/d2b-wayland-proxy-test/proxy.sock " - "--connect /run/d2b-wayland-proxy-test/upstream.sock " - "--target acceptance-guest.local.d2b " - "--provider-kind local-vm " - ">/run/d2b-wayland-proxy-test/proxy.log 2>&1 & " - "echo $! > /run/d2b-wayland-proxy-test/proxy.pid" - ) - machine.succeed( - "for attempt in $(seq 1 300); do " - "test -S /run/d2b-wayland-proxy-test/proxy.sock && exit 0; " - "kill -0 $(cat /run/d2b-wayland-proxy-test/proxy.pid) 2>/dev/null || " - "{ echo 'd2b-wayland-proxy exited before binding its socket:'; " - "cat /run/d2b-wayland-proxy-test/proxy.log; exit 1; }; " - "sleep 0.1; done; " - "echo 'd2b-wayland-proxy did not bind its socket within 30s:'; " - "cat /run/d2b-wayland-proxy-test/proxy.log; exit 1" - ) - machine.succeed("test -S /run/d2b-wayland-proxy-test/proxy.sock") - machine.succeed("test \"$(stat -c %a /run/d2b-wayland-proxy-test)\" = 700") - - # Send a minimal wl_display.get_registry request through the proxy. The fake - # compositor must observe the same 12-byte Wayland request on its upstream - # socket, proving the live proxy accepted a client and relayed protocol - # traffic rather than only binding a socket. - stage("relay-proof") - machine.succeed( - "python3 - <<'PY'\n" - "import socket, struct\n" - "import time\n" - "sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)\n" - "sock.connect('/run/d2b-wayland-proxy-test/proxy.sock')\n" - "sock.sendall(struct.pack('/dev/null || true", - ), - ( - "proxy log", - "cat /run/d2b-wayland-proxy-test/proxy.log " - "2>/dev/null || true", - ), - ], - ) - machine.succeed( - "python3 - <<'PY'\n" - "import pathlib, struct\n" - "data = pathlib.Path('/run/d2b-wayland-proxy-test/upstream.seen').read_bytes()\n" - "assert data == struct.pack(' Date: Sat, 26 Sep 2026 22:37:50 -0700 Subject: [PATCH 14/51] refactor(vm): assert bridge-isolation in Rust The check's assertions move from the fixture's testScript into packages/d2b-test-vm-harness/src/checks/bridge_isolation.rs, in the same order, with the same command text and the same in-guest commands: 32 succeed, 2 fail, and the three `bridge -d link show` reads whose output substrings are the isolation assertion (work-l1 must not be isolated, work-l10 and work-l11 must be). No bound and no stage changed: the fixture declared none, and this port declares none. The fixture declared a plain NixOS node - it never wanted the d2b daemon host - so its two packages and its stateVersion move to nix/test-support/host-integration-node.nix as d2bBridgeIsolationNode, next to the reusable nodes, and its guest is built from that declaration by name. The machine size, the disk and the emulator invocation are the QEMU VM module's defaults in both places, because the fixture declared none of them either. The fixture file itself was removed by 8ad9e01f1, whose broad stage took a deletion this port had staged. This commit is the half that makes the check run again, so the tree is red between that commit and this one. Counts the port must satisfy: 32 succeed, 2 fail, 3 output-substring assertions, before and after. --- bazel/checks/vm/BUILD.bazel | 2 + nix/test-support/host-integration-node.nix | 22 +++ .../src/checks/bridge_isolation.rs | 129 ++++++++++++++++++ .../d2b-test-vm-harness/src/checks/mod.rs | 6 +- 4 files changed, 158 insertions(+), 1 deletion(-) create mode 100644 packages/d2b-test-vm-harness/src/checks/bridge_isolation.rs diff --git a/bazel/checks/vm/BUILD.bazel b/bazel/checks/vm/BUILD.bazel index da360f408..fd92eea12 100644 --- a/bazel/checks/vm/BUILD.bazel +++ b/bazel/checks/vm/BUILD.bazel @@ -131,6 +131,7 @@ _CHECKS = [ "guest-shell-service", "privilege-oracle", "resource-operator-activation", + "state-posture-contract", "runtime-cloud-hypervisor-guest-preflight", "state-posture-contract", "virtiofsd-volume-runtime", @@ -146,6 +147,7 @@ _CHECKS = [ # list; the image is built from the check's name, which is the shape its node # is declared under. _PORTED_CHECKS = [ + "bridge-isolation", "daemon-smoke", ] diff --git a/nix/test-support/host-integration-node.nix b/nix/test-support/host-integration-node.nix index ba76387ee..415201bd9 100644 --- a/nix/test-support/host-integration-node.nix +++ b/nix/test-support/host-integration-node.nix @@ -223,6 +223,23 @@ rec { }; }; + # The guest `bridge-isolation` boots: a plain NixOS node with the two + # userspace tools its assertions drive the bridge and its namespaces with. + # + # The check never wanted the d2b daemon host - it configures a bridge as + # root inside the guest and asserts the kernel's port-isolation semantics + # on it - so this node is not built on `d2bDaemonNode`. It declared these + # two packages and its `stateVersion`; the machine size, the disk, and the + # emulator invocation are the QEMU VM module's defaults, which the lane + # reads back off the image's manifest rather than restating here. + d2bBridgeIsolationNode = { pkgs, ... }: { + environment.systemPackages = [ + pkgs.iproute2 + pkgs.iputils + ]; + system.stateVersion = "25.11"; + }; + # The guest each fixture-less image evaluates, by the name the image action # asks for. A check's own guest is read out of the check's fixture; these are # the guests with no fixture to be read out of - the two reusable shapes the @@ -251,9 +268,14 @@ rec { }; portedCheckNodes = { + bridge-isolation = { + node = d2bBridgeIsolationNode; + testName = "d2b-bridge-isolation"; + }; daemon-smoke = { node = d2bDaemonSmokeNode; testName = "d2b-daemon-smoke"; }; + }; } diff --git a/packages/d2b-test-vm-harness/src/checks/bridge_isolation.rs b/packages/d2b-test-vm-harness/src/checks/bridge_isolation.rs new file mode 100644 index 000000000..cbd6e3aef --- /dev/null +++ b/packages/d2b-test-vm-harness/src/checks/bridge_isolation.rs @@ -0,0 +1,129 @@ +//! The bridge port isolation check, ported from its fixture. +//! +//! It exercises the bridge shape the host configures at runtime, as root, +//! inside the guest: one non-isolated net-VM port and two isolated workload +//! ports on `br-work-lan`, each port's peer living in its own network +//! namespace. The assertions are the fixture's, in the fixture's order and +//! with the fixture's own command text and bounds: the port that carries the +//! net VM stays reachable from both workloads, and the two workload ports +//! stay isolated from each other - including after one of them changes its +//! MAC address, which is the case a MAC-address-based filter would not catch. +//! +//! The fixture declared a plain NixOS node - it never wanted the d2b daemon +//! host - so the guest it boots is declared in +//! `nix/test-support/host-integration-node.nix` beside the reusable nodes, +//! with the same two packages it declared, and the guest's activation +//! contract is `multi-user.target`, which is the unit a node that declares no +//! acceptance units falls back to. +//! +//! `start_all()` is not restated here: it is the lane's own boot of the +//! guest the check runs against. + +use crate::legacy::{GuestControl, LegacyError, LegacyResult}; + +/// The network namespaces the fixture creates, in its own order. +const NAMESPACES: [&str; 3] = ["netvm", "vm10", "vm11"]; + +/// Each bridge port and the namespace its peer device lives in, in the +/// fixture's own order. +const PORTS: [(&str, &str); 3] = [ + ("work-l1", "netvm"), + ("work-l10", "vm10"), + ("work-l11", "vm11"), +]; + +/// The address each namespace's `eth0` gets, in the fixture's own order. +const ADDRESSES: [(&str, &str); 3] = [ + ("netvm", "10.20.0.1/24"), + ("vm10", "10.20.0.10/24"), + ("vm11", "10.20.0.11/24"), +]; + +/// The MAC address `vm10` changes its `eth0` to. The second isolation +/// assertion is made after this, so isolation is shown not to rest on the +/// address the port was isolated with. +const REPLACEMENT_MAC: &str = "02:20:00:00:00:11"; + +/// The check's assertions, in the order its fixture made them. +pub fn assertions(control: &mut GuestControl) -> LegacyResult<()> { + // The netns mount point, the namespaces, the bridge, and the three veth + // pairs the fixture built before it asserted anything about them. + control.succeed(&["mkdir -p /run/netns"], None)?; + for namespace in NAMESPACES { + control.succeed(&[&format!("ip netns add {namespace}")], None)?; + } + control.succeed(&["ip link add br-work-lan type bridge"], None)?; + control.succeed(&["ip link set br-work-lan up"], None)?; + for (port, namespace) in PORTS { + control.succeed( + &[&format!( + "ip link add {port} type veth peer name eth0 netns {namespace}" + )], + None, + )?; + control.succeed(&[&format!("ip link set {port} master br-work-lan")], None)?; + control.succeed(&[&format!("ip link set {port} up")], None)?; + } + + // Port isolation on the two workload ports, and not on the net-VM port. + control.succeed(&["bridge link set dev work-l10 isolated on"], None)?; + control.succeed(&["bridge link set dev work-l11 isolated on"], None)?; + + for namespace in NAMESPACES { + control.succeed(&[&format!("ip netns exec {namespace} ip link set lo up")], None)?; + control.succeed( + &[&format!("ip netns exec {namespace} ip link set eth0 up")], + None, + )?; + } + for (namespace, address) in ADDRESSES { + control.succeed( + &[&format!("ip netns exec {namespace} ip addr add {address} dev eth0")], + None, + )?; + } + + // The declared isolation state, read back off the live bridge: the + // net-VM port must remain non-isolated and both workload ports isolated. + let work_l1 = control.succeed(&["bridge -d link show dev work-l1"], None)?; + if work_l1.contains("isolated on") { + return Err(LegacyError::Assertion( + "net-VM bridge port work-l1 must remain non-isolated".to_owned(), + )); + } + for (port, message) in [ + ("work-l10", "workload bridge port work-l10 is not isolated"), + ("work-l11", "workload bridge port work-l11 is not isolated"), + ] { + let port_state = control.succeed(&[&format!("bridge -d link show dev {port}")], None)?; + if !port_state.contains("isolated on") { + return Err(LegacyError::Assertion(message.to_owned())); + } + } + + // Both workloads reach the net VM. + control.succeed(&["ip netns exec vm10 ping -c1 -W1 10.20.0.1 >/dev/null"], None)?; + control.succeed(&["ip netns exec vm11 ping -c1 -W1 10.20.0.1 >/dev/null"], None)?; + + // The two workloads do not reach each other. + control.fail( + &["ip netns exec vm10 ping -c1 -W1 10.20.0.11 >/dev/null 2>&1"], + None, + )?; + + // ... and still do not after vm10's eth0 changes address, so the + // isolation is a property of the port rather than of the address it was + // isolated with. + control.succeed( + &[&format!( + "ip netns exec vm10 ip link set dev eth0 address {REPLACEMENT_MAC}" + )], + None, + )?; + control.fail( + &["ip netns exec vm10 ping -c1 -W1 10.20.0.11 >/dev/null 2>&1"], + None, + )?; + + Ok(()) +} diff --git a/packages/d2b-test-vm-harness/src/checks/mod.rs b/packages/d2b-test-vm-harness/src/checks/mod.rs index 4b6c29b7d..f23ce0a43 100644 --- a/packages/d2b-test-vm-harness/src/checks/mod.rs +++ b/packages/d2b-test-vm-harness/src/checks/mod.rs @@ -23,6 +23,7 @@ //! [`GuestControl::diag_wait`]: crate::legacy::GuestControl::diag_wait //! [`GuestControl`]: crate::legacy::GuestControl +pub mod bridge_isolation; pub mod daemon_smoke; use crate::legacy::{GuestControl, LegacyResult}; @@ -38,7 +39,10 @@ pub type Assertions = fn(&mut GuestControl) -> LegacyResult<()>; /// whether it holds an evaluated script), so a check whose image says it is /// ported and which has no entry here is a lane failure rather than a check /// that quietly does not run. -const PORTED: &[(&str, Assertions)] = &[("daemon-smoke", daemon_smoke::assertions)]; +const PORTED: &[(&str, Assertions)] = &[ + ("bridge-isolation", bridge_isolation::assertions), + ("daemon-smoke", daemon_smoke::assertions), +]; /// The assertions of one ported check, or `None` for a check that has not /// been ported. From ca8b4216008a00d491e9c33a4fd912918a70776f Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sat, 26 Sep 2026 22:37:50 -0700 Subject: [PATCH 15/51] refactor(vm): assert guest-agent-cap-confinement in Rust The check's assertions, and the five helper reads its testScript defined, move into packages/d2b-test-vm-harness/src/checks/guest_agent_cap_confinement.rs in the fixture's own order: 5 stage, 1 wait_for_unit (multi-user.target, 180s), 1 diag_unit (d2b-test-guest-agent.service, 60s), the two succeed calls that start the namespace and the agent, and twelve assertions over the live state - the agent's namespace inode equals the declared one, it does not share the host namespace, its effective set carries all three declared capabilities and nothing else, its main process is inside its own control group, starting it added no network capability to a process that was already in the host namespace, and its service left no capability-bearing process there. The helper reads the fixture's Python performed - readlink of a namespace, stat of a netns, CapEff out of /proc//status, the host namespace's per-process capability table, and a unit's process identities - are functions of the module with the fixture's own command text: nine of them run once, and the control-group walk adds one pid read per process. The fixture declared a plain NixOS node, so its unprivileged user, its two units and its iproute2 move to nix/test-support/host-integration-node.nix as d2bGuestAgentCapConfinementNode. Its fixture file was removed by 8ad9e01f1's broad stage; this commit adds the Rust that replaces it. Counts the port must satisfy: 5 stage, 1 wait_for_unit, 1 diag_unit, 2 succeed, 12 assertions, plus the helper reads, before and after. --- bazel/checks/vm/BUILD.bazel | 1 + nix/test-support/host-integration-node.nix | 66 ++++ .../src/checks/guest_agent_cap_confinement.rs | 304 ++++++++++++++++++ .../d2b-test-vm-harness/src/checks/mod.rs | 5 + 4 files changed, 376 insertions(+) create mode 100644 packages/d2b-test-vm-harness/src/checks/guest_agent_cap_confinement.rs diff --git a/bazel/checks/vm/BUILD.bazel b/bazel/checks/vm/BUILD.bazel index fd92eea12..3065c03ad 100644 --- a/bazel/checks/vm/BUILD.bazel +++ b/bazel/checks/vm/BUILD.bazel @@ -149,6 +149,7 @@ _CHECKS = [ _PORTED_CHECKS = [ "bridge-isolation", "daemon-smoke", + "guest-agent-cap-confinement", ] [ diff --git a/nix/test-support/host-integration-node.nix b/nix/test-support/host-integration-node.nix index 415201bd9..ac8f7bb1b 100644 --- a/nix/test-support/host-integration-node.nix +++ b/nix/test-support/host-integration-node.nix @@ -240,6 +240,68 @@ rec { system.stateVersion = "25.11"; }; + # The guest `guest-agent-cap-confinement` boots: a plain NixOS node with an + # unprivileged network-agent user, an isolated network namespace, and the + # agent process confined to it. + # + # As with `bridge-isolation`, the check never wanted the d2b daemon host, + # so this node is not built on `d2bDaemonNode`. The two units are the + # fixture's own: the namespace unit creates `/run/netns/d2b-test-agent`, + # and the agent unit runs as the unprivileged user inside that namespace + # with the three capabilities the check asserts on, no ambient privilege + # beyond them, and `NoNewPrivileges`. Moving them here is what lets the + # fixture go without the guest the check asserts against going with it. + d2bGuestAgentCapConfinementNode = { pkgs, ... }: { + users.groups.d2b-net-agent-test = { }; + users.users.d2b-net-agent-test = { + isSystemUser = true; + group = "d2b-net-agent-test"; + }; + + environment.systemPackages = [ pkgs.iproute2 ]; + + systemd.services.d2b-test-agent-netns = { + description = "Create the isolated network-agent test namespace"; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + ExecStart = pkgs.writeShellScript "d2b-test-agent-netns-up" '' + set -eu + install -d -m 0755 /run/netns + ${pkgs.iproute2}/bin/ip netns add d2b-test-agent + ${pkgs.iproute2}/bin/ip -n d2b-test-agent link set lo up + ''; + ExecStop = "${pkgs.iproute2}/bin/ip netns delete d2b-test-agent"; + }; + }; + + systemd.services.d2b-test-guest-agent = { + description = "Network agent capability-confinement test process"; + requires = [ "d2b-test-agent-netns.service" ]; + after = [ "d2b-test-agent-netns.service" ]; + serviceConfig = { + Type = "simple"; + User = "d2b-net-agent-test"; + Group = "d2b-net-agent-test"; + ExecStart = "${pkgs.coreutils}/bin/sleep infinity"; + NetworkNamespacePath = "/run/netns/d2b-test-agent"; + CapabilityBoundingSet = [ + "CAP_NET_ADMIN" + "CAP_NET_BIND_SERVICE" + "CAP_NET_RAW" + ]; + AmbientCapabilities = [ + "CAP_NET_ADMIN" + "CAP_NET_BIND_SERVICE" + "CAP_NET_RAW" + ]; + NoNewPrivileges = true; + }; + }; + + system.stateVersion = "25.11"; + }; + # The guest each fixture-less image evaluates, by the name the image action # asks for. A check's own guest is read out of the check's fixture; these are # the guests with no fixture to be read out of - the two reusable shapes the @@ -276,6 +338,10 @@ rec { node = d2bDaemonSmokeNode; testName = "d2b-daemon-smoke"; }; + guest-agent-cap-confinement = { + node = d2bGuestAgentCapConfinementNode; + testName = "d2b-guest-agent-cap-confinement"; + }; }; } diff --git a/packages/d2b-test-vm-harness/src/checks/guest_agent_cap_confinement.rs b/packages/d2b-test-vm-harness/src/checks/guest_agent_cap_confinement.rs new file mode 100644 index 000000000..922d79c47 --- /dev/null +++ b/packages/d2b-test-vm-harness/src/checks/guest_agent_cap_confinement.rs @@ -0,0 +1,304 @@ +//! The guest network-agent capability confinement check, ported from its +//! fixture. +//! +//! It gives a live process the three capabilities the network agent is +//! declared to need, inside a dedicated Linux network namespace, and asserts +//! the effective set it ends up with - then proves that starting it added no +//! such capability to any process sharing the host network namespace. The +//! assertions are the fixture's, in the fixture's order and with the +//! fixture's own command text and bounds, including the three helper reads +//! the fixture's script defined and this module carries as functions: the +//! namespace of a process, the inode of a namespace, a process's effective +//! capability set, the host namespace's per-process capability table, and a +//! service's process identities. +//! +//! The fixture declared a plain NixOS node - it never wanted the d2b daemon +//! host - so the guest it boots, with the namespace unit, the agent unit and +//! the unprivileged user both run as, is declared in +//! `nix/test-support/host-integration-node.nix` beside the reusable nodes. +//! +//! `start_all()` is not restated here: it is the lane's own boot of the +//! guest the check runs against. + +use std::{collections::{BTreeMap, BTreeSet}, time::Duration}; + +use crate::legacy::{GuestControl, LegacyError, LegacyResult}; + +/// The effective network capabilities the agent unit declares: +/// `CAP_NET_ADMIN`, `CAP_NET_BIND_SERVICE` and `CAP_NET_RAW`. +const CAPABILITY_MASK: u64 = (1 << 10) | (1 << 12) | (1 << 13); + +/// The bound the guest's own `multi-user.target` gets, the fixture's own. +const BOOT: Duration = Duration::from_secs(180); + +/// The bound the agent unit gets once it has been started, the fixture's +/// own; the unit is already started, so this is the wait for systemd to have +/// settled it. +const AGENT_UP: Duration = Duration::from_secs(60); + +/// The fixture's own table of processes in the host network namespace, down +/// to its words: the host namespace's inode, every process sharing it, each +/// one's `CapEff`, and the start time that keeps a recycled pid from being +/// mistaken for the process it replaced. +/// +/// A raw string, because the `\n` in the `printf` is the shell's escape for +/// the newline that separates the rows - the same two characters the +/// fixture's Python sent. +const HOST_NAMESPACE_CAPABILITIES: &str = concat!( + "host_ns=$(readlink /proc/1/ns/net); ", + "for status in /proc/[0-9]*/status; do ", + "pid='${status#/proc/}'; pid='${pid%/status}'; ", + "ns=$(readlink /proc/$pid/ns/net 2>/dev/null) || continue; ", + "test \"$ns\" = \"$host_ns\" || continue; ", + "cap=$(while IFS=: read -r key value; do ", + "test \"$key\" = CapEff && { printf '%s' \"$value\"; break; }; done < \"$status\"); ", + "start=$(cut -d' ' -f22 /proc/$pid/stat 2>/dev/null) || continue; ", + r#"printf '%s %s %s\n' "$pid" "$start" "$cap"; "#, + "done", +); + +/// The check's assertions, in the order its fixture made them. +pub fn assertions(control: &mut GuestControl) -> LegacyResult<()> { + control.stage("boot"); + control.wait_for_unit("multi-user.target", None, BOOT)?; + + // The namespace the agent's own unit is confined to, up before the agent + // that requires it. + control.stage("agent-netns"); + control.succeed(&["systemctl start d2b-test-agent-netns.service"], None)?; + + // The baseline is the host namespace's capability table before the agent + // exists, so the last assertion can be about what changed rather than + // about what is there. + control.stage("baseline-caps"); + let host_namespace = network_namespace(control, "1")?; + let baseline = host_namespace_capabilities(control)?; + + control.stage("agent-up"); + control.succeed(&["systemctl start d2b-test-guest-agent.service"], None)?; + control.diag_unit("guest-agent-up", "d2b-test-guest-agent.service", AGENT_UP)?; + let agent_pid = control + .succeed( + &["systemctl show -P MainPID d2b-test-guest-agent.service"], + None, + )? + .trim() + .to_owned(); + if agent_pid.is_empty() || agent_pid == "0" { + return Err(LegacyError::Assertion( + "network agent did not start".to_owned(), + )); + } + + let agent_namespace = network_namespace(control, &agent_pid)?; + let agent_namespace_inode = network_namespace_inode(control, &format!("/proc/{agent_pid}/ns/net"))?; + let declared_namespace_inode = network_namespace_inode(control, "/run/netns/d2b-test-agent")?; + if agent_namespace_inode != declared_namespace_inode { + return Err(LegacyError::Assertion( + "network agent did not inherit the declared Guest network namespace".to_owned(), + )); + } + if agent_namespace == host_namespace { + return Err(LegacyError::Assertion( + "network agent unexpectedly shares the host network namespace".to_owned(), + )); + } + + // The declared set is exactly what the process holds: every declared + // capability present, and nothing outside the declaration. + control.stage("confinement-assertions"); + let agent_capabilities = effective_capabilities(control, &agent_pid)?; + if agent_capabilities & CAPABILITY_MASK != CAPABILITY_MASK { + return Err(LegacyError::Assertion( + "network agent is missing a required effective network capability".to_owned(), + )); + } + if agent_capabilities & !CAPABILITY_MASK != 0 { + return Err(LegacyError::Assertion( + "network agent received an undeclared effective capability".to_owned(), + )); + } + + // The agent's main process is inside the control group of the unit that + // declared the capabilities, rather than a process systemd started and + // then lost track of. + let service_identities = service_processes(control, "d2b-test-guest-agent.service")?; + let agent_start = control + .succeed(&[&format!("cut -d' ' -f22 /proc/{agent_pid}/stat")], None)? + .trim() + .to_owned(); + if !service_identities.contains(&(agent_pid.clone(), agent_start)) { + return Err(LegacyError::Assertion( + "network agent main process is outside its service control group".to_owned(), + )); + } + + // Starting the agent added nothing to a process that already shared the + // host network namespace ... + let after = host_namespace_capabilities(control)?; + let gained = baseline + .iter() + .filter_map(|((pid, start), before)| { + let current = after.get(&(pid.clone(), start.clone()))?; + let gained = current & CAPABILITY_MASK & !before; + (gained != 0).then(|| { + ( + pid.clone(), + start.clone(), + before & CAPABILITY_MASK, + current & CAPABILITY_MASK, + ) + }) + }) + .collect::>(); + if !gained.is_empty() { + return Err(LegacyError::Assertion(format!( + "starting the network agent added effective network capabilities to \ + an existing host-network-namespace process: {}", + render_gained(&gained) + ))); + } + + // ... and the agent's own service left no capability-bearing process + // sharing that namespace. + let service_leaks = service_identities + .iter() + .filter_map(|(pid, start)| { + let capabilities = after.get(&(pid.clone(), start.clone()))? & CAPABILITY_MASK; + (capabilities != 0).then(|| (pid.clone(), start.clone(), capabilities)) + }) + .collect::>(); + if !service_leaks.is_empty() { + return Err(LegacyError::Assertion(format!( + "network agent service left a capability-bearing process in the host \ + network namespace: {}", + render_service_leaks(&service_leaks) + ))); + } + + Ok(()) +} + +/// The network namespace one process is in, as `readlink` reports it. +fn network_namespace(control: &mut GuestControl, pid: &str) -> LegacyResult { + Ok(control + .succeed(&[&format!("readlink /proc/{pid}/ns/net")], None)? + .trim() + .to_owned()) +} + +/// The device and inode of a namespace, as `stat` reports them. +fn network_namespace_inode(control: &mut GuestControl, path: &str) -> LegacyResult { + Ok(control + .succeed(&[&format!("stat -Lc '%d:%i' {path}")], None)? + .trim() + .to_owned()) +} + +/// The effective capability set of one process, read out of its status. +fn effective_capabilities(control: &mut GuestControl, pid: &str) -> LegacyResult { + let status = control.succeed(&[&format!("cat /proc/{pid}/status")], None)?; + for line in status.lines() { + if let Some(value) = line.strip_prefix("CapEff:") { + return parse_capabilities(value.trim()).ok_or_else(|| { + LegacyError::Assertion(format!( + "process {pid} has an unreadable CapEff status field: {value:?}" + )) + }); + } + } + Err(LegacyError::Assertion(format!( + "process {pid} has no CapEff status field" + ))) +} + +/// The effective capability set of every process in the host network +/// namespace, keyed by `(pid, start time)`. +fn host_namespace_capabilities( + control: &mut GuestControl, +) -> LegacyResult> { + let rows = control.succeed(&[HOST_NAMESPACE_CAPABILITIES], None)?; + let mut capabilities = BTreeMap::new(); + for row in rows.lines() { + let mut fields = row.split_whitespace(); + let (pid, start, cap) = match (fields.next(), fields.next(), fields.next(), fields.next()) { + (Some(pid), Some(start), Some(cap), None) => (pid, start, cap), + _ => { + return Err(LegacyError::Assertion(format!( + "host network namespace capability row is not ' ': {row:?}" + ))); + } + }; + let cap = parse_capabilities(cap).ok_or_else(|| { + LegacyError::Assertion(format!( + "host network namespace process {pid} has an unreadable capability set: {cap:?}" + )) + })?; + capabilities.insert((pid.to_owned(), start.to_owned()), cap); + } + Ok(capabilities) +} + +/// The `(pid, start time)` identities of every process in one unit's control +/// group. +fn service_processes( + control: &mut GuestControl, + unit: &str, +) -> LegacyResult> { + let control_group = control + .succeed(&[&format!("systemctl show -P ControlGroup {unit}")], None)? + .trim() + .to_owned(); + if !control_group.starts_with('/') { + return Err(LegacyError::Assertion(format!( + "{unit} has invalid control group '{control_group}'" + ))); + } + let rows = control.succeed( + &[&format!( + "find /sys/fs/cgroup{control_group} -name cgroup.procs -type f -exec cat {{}} +" + )], + None, + )?; + let mut identities = BTreeSet::new(); + for pid in rows.lines() { + let start = control + .succeed(&[&format!("cut -d' ' -f22 /proc/{pid}/stat")], None)? + .trim() + .to_owned(); + identities.insert((pid.to_owned(), start)); + } + Ok(identities) +} + +/// A capability set as these processes report it: hexadecimal, with the +/// `0x` prefix the kernel does not print but a reader may. +fn parse_capabilities(value: &str) -> Option { + let digits = value + .strip_prefix("0x") + .or_else(|| value.strip_prefix("0X")) + .unwrap_or(value); + u64::from_str_radix(digits, 16).ok() +} + +/// The processes a gain was observed on, rendered the way the fixture's own +/// failure rendered the list it built. +fn render_gained(gained: &[(String, String, u64, u64)]) -> String { + let rows = gained + .iter() + .map(|(pid, start, before, after)| format!("('{pid}', '{start}', {before}, {after})")) + .collect::>() + .join(", "); + format!("[{rows}]") +} + +/// The capability-bearing processes one service left behind, rendered the +/// way the fixture's own failure rendered the list it built. +fn render_service_leaks(leaks: &[(String, String, u64)]) -> String { + let rows = leaks + .iter() + .map(|(pid, start, capabilities)| format!("('{pid}', '{start}', {capabilities})")) + .collect::>() + .join(", "); + format!("[{rows}]") +} diff --git a/packages/d2b-test-vm-harness/src/checks/mod.rs b/packages/d2b-test-vm-harness/src/checks/mod.rs index f23ce0a43..574e5aab3 100644 --- a/packages/d2b-test-vm-harness/src/checks/mod.rs +++ b/packages/d2b-test-vm-harness/src/checks/mod.rs @@ -25,6 +25,7 @@ pub mod bridge_isolation; pub mod daemon_smoke; +pub mod guest_agent_cap_confinement; use crate::legacy::{GuestControl, LegacyResult}; @@ -42,6 +43,10 @@ pub type Assertions = fn(&mut GuestControl) -> LegacyResult<()>; const PORTED: &[(&str, Assertions)] = &[ ("bridge-isolation", bridge_isolation::assertions), ("daemon-smoke", daemon_smoke::assertions), + ( + "guest-agent-cap-confinement", + guest_agent_cap_confinement::assertions, + ), ]; /// The assertions of one ported check, or `None` for a check that has not From 8f2607a91c6dea8b818e53ab3ec3b7e126d970ab Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sat, 26 Sep 2026 22:37:50 -0700 Subject: [PATCH 16/51] refactor(vm): assert guest-shell-service in Rust The check's assertions move into packages/d2b-test-vm-harness/src/checks/guest_shell_service.rs in the fixture's own order: 1 stage, 1 wait_for_unit (multi-user.target, 180s), 1 diag_unit (d2bd-guest.service, 120s), 1 succeed (the unit is active), 1 diag_wait (60s) over the journal line that says the Agent ComponentSession listener is bound - with the fixture's own row set (the unit's status dump) and its own journal source - and 1 fail over the bundle-validation failure line, which must not be there. The listener line alone would not prove a boot, which is why the fixture asserted the absence too, and so does the port. The fixture's node - the component-session and guest-broker modules with the enrolled key pair, the v3 bundle, the bundle-install unit and the AF_VSOCK device - moves to nix/test-support/host-integration-node.nix as d2bGuestShellServiceNode, with the two runCommand derivations it carried. Its fixture file was removed by 8ad9e01f1's broad stage; this commit adds the Rust that replaces it. Counts the port must satisfy: 1 stage, 1 wait_for_unit, 1 diag_unit, 1 diag_wait, 1 succeed, 1 fail, before and after. --- bazel/checks/vm/BUILD.bazel | 1 + nix/test-support/host-integration-node.nix | 120 ++++++++++++++++++ .../src/checks/guest_shell_service.rs | 82 ++++++++++++ .../d2b-test-vm-harness/src/checks/mod.rs | 2 + 4 files changed, 205 insertions(+) create mode 100644 packages/d2b-test-vm-harness/src/checks/guest_shell_service.rs diff --git a/bazel/checks/vm/BUILD.bazel b/bazel/checks/vm/BUILD.bazel index 3065c03ad..18728845b 100644 --- a/bazel/checks/vm/BUILD.bazel +++ b/bazel/checks/vm/BUILD.bazel @@ -150,6 +150,7 @@ _PORTED_CHECKS = [ "bridge-isolation", "daemon-smoke", "guest-agent-cap-confinement", + "guest-shell-service", ] [ diff --git a/nix/test-support/host-integration-node.nix b/nix/test-support/host-integration-node.nix index ac8f7bb1b..7c2bcd428 100644 --- a/nix/test-support/host-integration-node.nix +++ b/nix/test-support/host-integration-node.nix @@ -302,6 +302,122 @@ rec { system.stateVersion = "25.11"; }; + # The guest `guest-shell-service` boots: the component-session and + # guest-broker modules applied directly to a NixOS node, with the enrolled + # inputs the Guest target agent must boot from and the AF_VSOCK device its + # ComponentSession listener binds. + # + # The check's fixture declared this node inline and carried its bundle and + # key pair in its own `let`; both move here with it, so the guest survives + # the fixture. The bundle is a fixture whose self-hash covers the canonical + # JSON without `bundleHash`, and the key pair is the enrollment owner's + # 32-byte inputs - the agent never generates either. + d2bGuestShellServiceNode = + { lib, pkgs, ... }: + let + fixtureKeys = pkgs.runCommand "guest-shell-component-session-keys" { } '' + mkdir -p "$out" + printf '\001\002\003\004\005\006\007\010\011\012\013\014\015\016\017\020\021\022\023\024\025\026\027\030\031\032\033\034\035\036\037\040' > "$out/guest.key" + printf '\130\151\257\364\120\124\227\062\313\252\355\136\135\371\263\012\155\243\034\260\345\164\053\255\132\324\241\247\150\361\246\173' > "$out/parent.pub" + ''; + + guestBundle = pkgs.runCommand "guest-shell-guest-bundle" { + nativeBuildInputs = [ pkgs.python3 ]; + } '' + mkdir -p "$out" + printf '%s\n' '{"schemaVersion":"v2","site":{"allowUnsafeEastWest":false},"environments":[],"nftables":{"family":"inet","table":"d2b","chains":[],"tableHashAfterApply":null,"ownershipId":"guest-shell-service"},"networkManager":{"filePath":"/etc/NetworkManager/conf.d/00-d2b-unmanaged.conf","matchCriteria":[],"reloadBehavior":"atomic-reload","ownership":{"owner":"root","group":"root","mode":"0644","driftPolicy":"replace"}},"hostsFile":{"startMarker":"# d2b-managed begin","endMarker":"# d2b-managed end","rule":"replace-managed-block"},"kernelModules":[],"fdOwnership":[],"cloudHypervisorCapabilities":[],"ifNameMappings":[],"ch":null,"firewallCoexistencePolicy":null}' > "$out/host.json" + printf '%s\n' '{"schemaVersion":"v2","vms":[]}' > "$out/processes.json" + printf '%s\n' '{"schemaVersion":"v2","publicOperations":[],"brokerOperations":[]}' > "$out/privileges.json" + printf '%s\n' '{"_manifest":{"manifestVersion":6},"_observability":{"enabled":false,"signozUrl":"http://127.0.0.1:8080","signozOtlpGrpcPort":4317,"signozOtlpHttpPort":4318,"obsVsockCid":0,"obsVsockHostSocket":"","vmName":""}}' > "$out/vms.json" + python3 - "$out/bundle.json" <<'PY' + import hashlib + import json + import sys + + # Zone-native v3 bundle: the loader (BundleResolver) accepts only the + # v3 contract. The self-hash is computed over the serialization with + # bundleHash absent and artifactHashes nullified (verify_bundle_hash). + bundle = { + "artifactHashes": {}, + "bundleVersion": 1, + "schemaVersion": "v3", + "privilegesPath": "privileges.json", + "zones": [], + "generation": { + "generatedAt": None, + "generator": "guest-shell-service", + "sourceRevision": None, + }, + } + preimage = dict(bundle) + preimage["artifactHashes"] = None + canonical = json.dumps(preimage, sort_keys=True, separators=(",", ":")).encode() + bundle["bundleHash"] = "sha256:" + hashlib.sha256(canonical).hexdigest() + with open(sys.argv[1], "w", encoding="utf-8") as output: + json.dump(bundle, output, sort_keys=True, separators=(",", ":")) + output.write("\n") + PY + ''; + in + { lib, pkgs, ... }: + { + imports = [ + ../../nixos-modules/component-session.nix + ../../nixos-modules/guest-broker.nix + { + _module.args = { + d2bInputs = { inherit self; }; + d2bHostTools = { + broker = self.packages.${pkgs.system}.d2b-broker-guest-static; + }; + d2bHostToolOverrides = self.lib.d2bHostToolOverrides; + }; + + d2b.componentSession = { + enable = lib.mkForce true; + guestConfigPath = lib.mkForce null; + }; + + # The Guest target agent binds an AF_VSOCK ComponentSession listener. + # The lane's QEMU ships vhost-vsock-pci and now passes /dev/vhost-vsock + # into the build sandbox, so this node can carry the same device the + # enrolled Guest gets, plus a fixture bundle and key pair installed at + # the production owner/mode the resolver verifies (root:d2bd 0640). + virtualisation.qemu.options = [ "-device" "vhost-vsock-pci,guest-cid=3" ]; + boot.kernelModules = [ "vmw_vsock_virtio_transport" ]; + + environment.etc."d2b/component-session/guest.key".source = + "${fixtureKeys}/guest.key"; + environment.etc."d2b/component-session/parent.pub".source = + "${fixtureKeys}/parent.pub"; + + d2b.componentSession.localPrivateKeyPath = + "/etc/d2b/component-session/guest.key"; + d2b.componentSession.parentPublicKeyPath = + "/etc/d2b/component-session/parent.pub"; + d2b.componentSession.bundlePath = "/var/lib/d2b/guest-bundle/bundle.json"; + d2b.guestBroker.bundlePath = "/var/lib/d2b/guest-bundle/bundle.json"; + + systemd.services.d2b-install-guest-bundle = { + requiredBy = [ "d2bd-guest.service" "d2b-broker-guest.service" ]; + before = [ "d2bd-guest.service" "d2b-broker-guest.service" ]; + serviceConfig.Type = "oneshot"; + script = '' + install -d -o root -g d2bd -m 0750 /var/lib/d2b/guest-bundle + for file in bundle.json host.json processes.json privileges.json; do + install -o root -g d2bd -m 0640 \ + ${guestBundle}/"$file" /var/lib/d2b/guest-bundle/"$file" + done + install -o root -g d2bd -m 0644 \ + ${guestBundle}/vms.json /var/lib/d2b/guest-bundle/vms.json + ''; + }; + + system.stateVersion = "25.11"; + } + ]; + }; + # The guest each fixture-less image evaluates, by the name the image action # asks for. A check's own guest is read out of the check's fixture; these are # the guests with no fixture to be read out of - the two reusable shapes the @@ -342,6 +458,10 @@ rec { node = d2bGuestAgentCapConfinementNode; testName = "d2b-guest-agent-cap-confinement"; }; + guest-shell-service = { + node = d2bGuestShellServiceNode; + testName = "d2b-guest-shell-service"; + }; }; } diff --git a/packages/d2b-test-vm-harness/src/checks/guest_shell_service.rs b/packages/d2b-test-vm-harness/src/checks/guest_shell_service.rs new file mode 100644 index 000000000..13c89e3e3 --- /dev/null +++ b/packages/d2b-test-vm-harness/src/checks/guest_shell_service.rs @@ -0,0 +1,82 @@ +//! The guest ComponentSession service wiring check, ported from its fixture. +//! +//! It applies the component-session module and the guest-broker module +//! directly to a NixOS node - no nested, d2b-managed VM - and asserts that +//! the Guest target agent boots from the enrolled inputs the fixture +//! installed (a ComponentSession key pair and a self-hashed v3 bundle) and +//! reaches its AF_VSOCK ComponentSession listener, which is the route the +//! shell family's per-session supervisor service and the other provider +//! services are served over. A bundle or key the agent cannot read fails +//! closed here instead of restart-looping unnoticed. +//! +//! The assertions are the fixture's, in the fixture's order and with the +//! fixture's own command text and bounds, including the two journal greps: +//! the listener line must be there, and the bundle-validation failure line +//! must not. The diagnostics the fixture's prelude printed for those waits +//! are this surface's own: a `diag_unit` reads a unit's status, and a +//! `diag_wait` reads the row set and the journal sources the fixture named. +//! +//! The fixture declared its guest inline, on top of the two d2b modules, so +//! the guest it boots - the modules, the enrolled inputs, the bundle install +//! unit, and the `vhost-vsock-pci` device its listener needs - is declared in +//! `nix/test-support/host-integration-node.nix` beside the reusable nodes. +//! +//! `start_all()` is not restated here: it is the lane's own boot of the +//! guest the check runs against. + +use std::time::Duration; + +use crate::legacy::{DiagRow, GuestControl, LegacyResult}; + +/// The bound the guest's own `multi-user.target` gets, the fixture's own. +const BOOT: Duration = Duration::from_secs(180); + +/// The bound the guest daemon gets to become active, the fixture's own. +const GUEST_DAEMON: Duration = Duration::from_secs(120); + +/// The bound the listener line gets to appear in the journal, the fixture's +/// own. +const LISTENER_BOUND: Duration = Duration::from_secs(60); + +/// The unit whose journal both greps read. +const GUEST_DAEMON_UNIT: &str = "d2bd-guest.service"; + +/// The row the listener wait explains itself with, the fixture's own: the +/// unit's status, dumped the way the prelude's `unit_dumps` dumped it. +const GUEST_DAEMON_STATUS: &str = "systemctl status d2bd-guest.service --no-pager 2>&1 | tail -n 40 || true"; + +/// The journal line the agent logs once its AF_VSOCK listener is bound. +const LISTENER_BOUND_COMMAND: &str = + "journalctl -u d2bd-guest.service --no-pager -b | grep -F 'Guest ComponentSession listener bound'"; + +/// The bundle-validation failure the agent must not have logged. +const BUNDLE_VALIDATION_FAILED_COMMAND: &str = + "journalctl --no-pager -b | grep -F 'Guest process bundle validation failed'"; + +/// The check's assertions, in the order its fixture made them. +pub fn assertions(control: &mut GuestControl) -> LegacyResult<()> { + control.stage("boot"); + control.wait_for_unit("multi-user.target", None, BOOT)?; + + // The Guest target agent boots from the enrolled bundle and key pair, + // and the unit that carries the device answers for it. + control.diag_unit("guest-daemon", GUEST_DAEMON_UNIT, GUEST_DAEMON)?; + control.succeed(&["systemctl is-active --quiet d2bd-guest.service"], None)?; + + let rows: [DiagRow<'_>; 1] = [(GUEST_DAEMON_UNIT, GUEST_DAEMON_STATUS)]; + let explain = [("d2bd-guest.service", "")]; + control.diag_wait( + "guest-listener-bound", + LISTENER_BOUND_COMMAND, + LISTENER_BOUND, + &rows, + &explain, + )?; + + // ... and the failure path the same reader would have reported is + // absent, so the listener line is evidence of a boot rather than of a + // restart loop that got lucky. + control.fail(&[BUNDLE_VALIDATION_FAILED_COMMAND], None)?; + + Ok(()) +} diff --git a/packages/d2b-test-vm-harness/src/checks/mod.rs b/packages/d2b-test-vm-harness/src/checks/mod.rs index 574e5aab3..4a2e7519d 100644 --- a/packages/d2b-test-vm-harness/src/checks/mod.rs +++ b/packages/d2b-test-vm-harness/src/checks/mod.rs @@ -26,6 +26,7 @@ pub mod bridge_isolation; pub mod daemon_smoke; pub mod guest_agent_cap_confinement; +pub mod guest_shell_service; use crate::legacy::{GuestControl, LegacyResult}; @@ -47,6 +48,7 @@ const PORTED: &[(&str, Assertions)] = &[ "guest-agent-cap-confinement", guest_agent_cap_confinement::assertions, ), + ("guest-shell-service", guest_shell_service::assertions), ]; /// The assertions of one ported check, or `None` for a check that has not From 1802c0b6fccd9d93886be039fbb9fd51dbc9c9b5 Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sat, 26 Sep 2026 22:37:50 -0700 Subject: [PATCH 17/51] refactor(vm): assert privilege-oracle in Rust The live broker privilege posture oracle moves into packages/d2b-test-vm-harness/src/checks/privilege_oracle.rs in the fixture's own order: 4 stage, 2 diag_unit (d2b-broker.socket 30s, d2bd.service 180s), nine succeed calls (the broker start, the main-pid wait, the rendered posture, the live status, cgroup and namespace reads, the kernel's cap_last_cap and the two identity reads) and the twelve assertions the fixture made: uid and gid equal the rendered User and Group, the uid is 0, CapBnd equals the rendered CapabilityBoundingSet and is not the full kernel mask, CapEff stays inside CapBnd, CapAmb equals the rendered AmbientCapabilities and is zero, NoNewPrivs equals the rendered NoNewPrivileges, seccomp is filter mode, and the cgroup path carries both the rendered Slice and d2b.slice. The fixture's two parsing helpers come with it, including their refusal of a capability this kernel does not number. The four lines the fixture printed about the posture are reported through the surface's own `announce`, so a run reads the way it read. The guest is the reusable daemon node the fixture already booted from the node module, so no guest configuration moved. Its fixture file was removed by 8ad9e01f1's broad stage; this commit adds the Rust that replaces it. Counts the port must satisfy: 4 stage, 2 diag_unit, 9 succeed, 12 assertions, before and after. --- bazel/checks/vm/BUILD.bazel | 1 + nix/test-support/host-integration-node.nix | 4 + .../d2b-test-vm-harness/src/checks/mod.rs | 2 + .../src/checks/privilege_oracle.rs | 455 ++++++++++++++++++ 4 files changed, 462 insertions(+) create mode 100644 packages/d2b-test-vm-harness/src/checks/privilege_oracle.rs diff --git a/bazel/checks/vm/BUILD.bazel b/bazel/checks/vm/BUILD.bazel index 18728845b..c98a8781e 100644 --- a/bazel/checks/vm/BUILD.bazel +++ b/bazel/checks/vm/BUILD.bazel @@ -151,6 +151,7 @@ _PORTED_CHECKS = [ "daemon-smoke", "guest-agent-cap-confinement", "guest-shell-service", + "privilege-oracle", ] [ diff --git a/nix/test-support/host-integration-node.nix b/nix/test-support/host-integration-node.nix index 7c2bcd428..f10437c05 100644 --- a/nix/test-support/host-integration-node.nix +++ b/nix/test-support/host-integration-node.nix @@ -462,6 +462,10 @@ rec { node = d2bGuestShellServiceNode; testName = "d2b-guest-shell-service"; }; + privilege-oracle = { + node = d2bDaemonNode { }; + testName = "d2b-privilege-oracle"; + }; }; } diff --git a/packages/d2b-test-vm-harness/src/checks/mod.rs b/packages/d2b-test-vm-harness/src/checks/mod.rs index 4a2e7519d..ba7823c16 100644 --- a/packages/d2b-test-vm-harness/src/checks/mod.rs +++ b/packages/d2b-test-vm-harness/src/checks/mod.rs @@ -27,6 +27,7 @@ pub mod bridge_isolation; pub mod daemon_smoke; pub mod guest_agent_cap_confinement; pub mod guest_shell_service; +pub mod privilege_oracle; use crate::legacy::{GuestControl, LegacyResult}; @@ -49,6 +50,7 @@ const PORTED: &[(&str, Assertions)] = &[ guest_agent_cap_confinement::assertions, ), ("guest-shell-service", guest_shell_service::assertions), + ("privilege-oracle", privilege_oracle::assertions), ]; /// The assertions of one ported check, or `None` for a check that has not diff --git a/packages/d2b-test-vm-harness/src/checks/privilege_oracle.rs b/packages/d2b-test-vm-harness/src/checks/privilege_oracle.rs new file mode 100644 index 000000000..fe2969da6 --- /dev/null +++ b/packages/d2b-test-vm-harness/src/checks/privilege_oracle.rs @@ -0,0 +1,455 @@ +//! The live broker privilege posture oracle, ported from its fixture. +//! +//! It boots a d2b daemon host, starts the socket-activated privileged broker, +//! derives the expected posture from the rendered systemd unit, and checks the +//! live `/proc/` state for the hardening invariants that matter at +//! runtime. It is the hermetic successor to the retired self-hosted L1c shell +//! oracle. The assertions are the fixture's, in the fixture's order and with +//! the fixture's own command text and bounds: the unit's `User`, `Group`, +//! `CapabilityBoundingSet`, `AmbientCapabilities`, `NoNewPrivileges` and +//! `Slice` are read back off systemd and compared with the process's own uid, +//! gid, capability sets, `NoNewPrivs`, seccomp mode and cgroup path. +//! +//! The fixture's two parsing helpers come with it: `parse_cap_set`, which +//! accepts either a hexadecimal mask or a list of capability names and treats +//! an empty bounding set as the full kernel mask, and `parse_unit_bool`. Both +//! refuse a value they do not understand rather than guessing, which is what +//! makes the oracle about the declaration rather than about the observation. +//! +//! The guest is the reusable daemon node, which the fixture already booted +//! from `nix/test-support/host-integration-node.nix`, so nothing about it +//! moved here. +//! +//! `start_all()` is not restated here: it is the lane's own boot of the +//! guest the check runs against. + +use std::{collections::BTreeMap, time::Duration}; + +use crate::legacy::{shlex_quote, GuestControl, LegacyError, LegacyResult}; + +/// The bound the broker socket gets, the fixture's own: it is +/// socket-activated, so it is up once systemd has bound and ACLed it. +const BROKER_SOCKET: Duration = Duration::from_secs(30); + +/// The bound the daemon gets before it reports readiness, the fixture's own. +const DAEMON_UP: Duration = Duration::from_secs(180); + +/// The capability names the kernel numbers, in the order the fixture listed +/// them. The index of a name in this list is the bit the kernel assigns it. +const CAPABILITY_NAMES: [&str; 41] = [ + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "KILL", + "SETGID", + "SETUID", + "SETPCAP", + "LINUX_IMMUTABLE", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_ADMIN", + "NET_RAW", + "IPC_LOCK", + "IPC_OWNER", + "SYS_MODULE", + "SYS_RAWIO", + "SYS_CHROOT", + "SYS_PTRACE", + "SYS_PACCT", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_NICE", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "MKNOD", + "LEASE", + "AUDIT_WRITE", + "AUDIT_CONTROL", + "SETFCAP", + "MAC_OVERRIDE", + "MAC_ADMIN", + "SYSLOG", + "WAKE_ALARM", + "BLOCK_SUSPEND", + "AUDIT_READ", + "PERFMON", + "BPF", + "CHECKPOINT_RESTORE", +]; + +/// The broker's main pid, waited for the way the fixture waited for it: the +/// unit publishes one, and a process it names is readable. +const WAIT_FOR_MAIN_PID: &str = concat!( + "for i in $(seq 1 100); do ", + "pid=$(systemctl show -p MainPID --value d2b-broker.service); ", + "if [ -n \"$pid\" ] && [ \"$pid\" != 0 ] && [ -r \"/proc/$pid/status\" ]; then ", + "echo \"$pid\"; exit 0; fi; ", + "sleep 0.2; ", + "done; ", + "echo 'd2b-broker.service did not publish a MainPID within 20s:'; ", + "systemctl status --no-pager d2b-broker.service; ", + "exit 1", +); + +/// The rendered posture, in the seven properties the oracle compares against +/// the live process. +const RENDERED_POSTURE: &str = concat!( + "systemctl show d2b-broker.service ", + "-p CapabilityBoundingSet ", + "-p AmbientCapabilities ", + "-p NoNewPrivileges ", + "-p User ", + "-p Group ", + "-p Slice ", + "-p SystemCallFilter", +); + +/// The live process's own namespace set, as the fixture read it: the kinds it +/// asked for, by name, with the link each one points at. +fn namespace_report(pid: &str) -> String { + format!( + concat!( + "for ns in cgroup ipc mnt net pid time time_for_children user uts; do ", + "[ -e /proc/{pid}/ns/$ns ] && printf '%s=%s\\n' \"$ns\" \"$(readlink /proc/{pid}/ns/$ns)\"; ", + "done", + ), + pid = pid, + ) +} + +/// The check's assertions, in the order its fixture made them. +pub fn assertions(control: &mut GuestControl) -> LegacyResult<()> { + control.stage("boot"); + control.diag_unit("broker-socket", "d2b-broker.socket", BROKER_SOCKET)?; + control.diag_unit("daemon-up", "d2bd.service", DAEMON_UP)?; + + // The broker is socket-activated, but starting the service directly keeps + // a live Type=notify process long enough to read its /proc posture. + control.stage("broker-start"); + control.succeed(&["systemctl start d2b-broker.service"], None)?; + let broker_pid = control.succeed(&[WAIT_FOR_MAIN_PID], None)?.trim().to_owned(); + control.announce(&format!("live d2b-broker PID: {broker_pid}")); + + control.stage("broker-posture"); + let unit_raw = control.succeed(&[RENDERED_POSTURE], None)?; + control.announce(&format!("rendered d2b-broker.service posture:\n{unit_raw}")); + + let status_raw = control.succeed(&[&format!("cat /proc/{broker_pid}/status")], None)?; + let cgroup_raw = control.succeed(&[&format!("cat /proc/{broker_pid}/cgroup")], None)?; + let ns_raw = control.succeed(&[&namespace_report(&broker_pid)], None)?; + control.announce(&format!( + "live /proc status subset:\n{}", + status_raw + .lines() + .filter(|line| { + [ + "Uid:", + "Gid:", + "Groups:", + "CapEff:", + "CapBnd:", + "CapAmb:", + "NoNewPrivs:", + "Seccomp:", + ] + .iter() + .any(|prefix| line.starts_with(prefix)) + }) + .collect::>() + .join("\n") + )); + control.announce(&format!("live cgroup:\n{cgroup_raw}")); + control.announce(&format!("live namespaces:\n{ns_raw}")); + + let unit = equals_properties(&unit_raw); + let status = colon_properties(&status_raw); + let cap_last_cap: u32 = control + .succeed(&["cat /proc/sys/kernel/cap_last_cap"], None)? + .trim() + .parse() + .map_err(|error| { + LegacyError::Assertion(format!( + "the kernel's cap_last_cap is not a number: {error}" + )) + })?; + let full_cap_mask = full_capability_mask(cap_last_cap); + + control.stage("posture-oracle"); + let user = required(&unit, "User")?; + let group = required(&unit, "Group")?; + let expected_uid: u64 = control + .succeed(&[&format!("id -u {}", shlex_quote(user))], None)? + .trim() + .parse() + .map_err(|error| LegacyError::Assertion(format!("the broker's uid is not a number: {error}")))?; + let expected_gid: u64 = control + .succeed( + &[&format!("getent group {} | cut -d: -f3", shlex_quote(group))], + None, + )? + .trim() + .parse() + .map_err(|error| LegacyError::Assertion(format!("the broker's gid is not a number: {error}")))?; + let expected_cap_bnd = parse_cap_set( + required(&unit, "CapabilityBoundingSet")?, + cap_last_cap, + full_cap_mask, + true, + )?; + let expected_cap_amb = parse_cap_set( + required(&unit, "AmbientCapabilities")?, + cap_last_cap, + full_cap_mask, + false, + )?; + let expected_nonewprivs = parse_unit_bool(required(&unit, "NoNewPrivileges")?)?; + let expected_slice = required(&unit, "Slice")?.trim().to_owned(); + + let actual_uids = integers(required(&status, "Uid")?)?; + let actual_gids = integers(required(&status, "Gid")?)?; + let actual_cap_eff = hex(required(&status, "CapEff")?)?; + let actual_cap_bnd = hex(required(&status, "CapBnd")?)?; + let actual_cap_amb = hex(required(&status, "CapAmb")?)?; + let actual_nonewprivs: u32 = required(&status, "NoNewPrivs")? + .trim() + .parse() + .map_err(|error| LegacyError::Assertion(format!("NoNewPrivs is not a number: {error}")))?; + let actual_seccomp: u32 = required(&status, "Seccomp")? + .trim() + .parse() + .map_err(|error| LegacyError::Assertion(format!("Seccomp is not a number: {error}")))?; + let cgroup_paths = cgroup_raw + .lines() + .filter(|line| line.contains(':')) + .map(|line| line.splitn(3, ':').nth(2).unwrap_or_default().to_owned()) + .collect::>(); + + // The process's identity is the unit's declaration of it. + if !actual_uids.iter().all(|uid| *uid == expected_uid) { + return Err(LegacyError::Assertion(format!( + "broker Uid must match rendered User={user} ({expected_uid}), got {}", + render_integers(&actual_uids) + ))); + } + if expected_uid != 0 { + return Err(LegacyError::Assertion(format!( + "broker must run as root uid 0, rendered User={user}" + ))); + } + if !actual_gids.iter().all(|gid| *gid == expected_gid) { + return Err(LegacyError::Assertion(format!( + "broker Gid must match rendered Group={group} ({expected_gid}), got {}", + render_integers(&actual_gids) + ))); + } + + // The capability sets: bounded exactly as declared, never the full kernel + // set, and effective bits inside the bounding set. + if actual_cap_bnd != expected_cap_bnd { + return Err(LegacyError::Assertion(format!( + "CapBnd must match rendered CapabilityBoundingSet: expected 0x{expected_cap_bnd:x}, got 0x{actual_cap_bnd:x}" + ))); + } + if actual_cap_bnd == full_cap_mask { + return Err(LegacyError::Assertion(format!( + "CapBnd is the full kernel capability mask 0x{full_cap_mask:x}, not the bounded broker set" + ))); + } + if actual_cap_eff & !actual_cap_bnd != 0 { + return Err(LegacyError::Assertion(format!( + "CapEff 0x{actual_cap_eff:x} contains bits outside CapBnd 0x{actual_cap_bnd:x}" + ))); + } + if actual_cap_amb != expected_cap_amb { + return Err(LegacyError::Assertion(format!( + "CapAmb must match rendered AmbientCapabilities: expected 0x{expected_cap_amb:x}, got 0x{actual_cap_amb:x}" + ))); + } + if actual_cap_amb != 0 { + return Err(LegacyError::Assertion(format!( + "broker must not carry ambient capabilities, got 0x{actual_cap_amb:x}" + ))); + } + + // The hardening flags: no-new-privileges as declared, and a seccomp + // filter actually installed rather than merely declared. + let rendered_nonewprivs = required(&unit, "NoNewPrivileges")?; + if actual_nonewprivs != expected_nonewprivs { + return Err(LegacyError::Assertion(format!( + "NoNewPrivs must match rendered NoNewPrivileges={rendered_nonewprivs}, got {actual_nonewprivs}" + ))); + } + if actual_seccomp != 2 { + return Err(LegacyError::Assertion(format!( + "broker must run in seccomp filter mode (2), got {actual_seccomp}" + ))); + } + + // The process lives in the slice its unit declared, and in the d2b slice + // the discipline is about. + if !cgroup_paths.iter().any(|path| path.contains(&expected_slice)) { + return Err(LegacyError::Assertion(format!( + "broker cgroup path must contain rendered Slice={expected_slice}, got {}", + render_strings(&cgroup_paths) + ))); + } + if !cgroup_paths.iter().any(|path| path.contains("d2b.slice")) { + return Err(LegacyError::Assertion(format!( + "broker cgroup path must contain d2b.slice, got {}", + render_strings(&cgroup_paths) + ))); + } + + Ok(()) +} + +/// The `Key=Value` lines of one command's output, as a map - the fixture's +/// own reader for the rendered unit. +fn equals_properties(output: &str) -> BTreeMap { + output + .lines() + .filter(|line| line.contains('=')) + .filter_map(|line| line.split_once('=')) + .map(|(key, value)| (key.to_owned(), value.to_owned())) + .collect() +} + +/// The `Key: Value` lines of one command's output, as a map - the fixture's +/// own reader for `/proc//status`. Each value is stripped; a line +/// without the separator is not a property and is skipped. +fn colon_properties(output: &str) -> BTreeMap { + output + .lines() + .filter(|line| line.contains(':')) + .filter_map(|line| line.split_once(':')) + .map(|(key, value)| (key.to_owned(), value.trim().to_owned())) + .collect() +} + +/// One property, or the failure a missing one is: the fixture's own `KeyError` +/// became a check failure rather than a lane failure, and it names the key. +fn required<'a>(properties: &'a BTreeMap, key: &str) -> LegacyResult<&'a String> { + properties.get(key).ok_or_else(|| { + LegacyError::Assertion(format!("the command that reports {key} did not report it")) + }) +} + +/// A whitespace-separated list of decimal numbers. +fn integers(value: &str) -> LegacyResult> { + value + .split_whitespace() + .map(|part| { + part.parse::().map_err(|error| { + LegacyError::Assertion(format!("{part:?} is not a number: {error}")) + }) + }) + .collect() +} + +/// A hexadecimal capability mask, as `/proc//status` prints one. +fn hex(value: &str) -> LegacyResult { + let trimmed = value.trim(); + let digits = trimmed + .strip_prefix("0x") + .or_else(|| trimmed.strip_prefix("0X")) + .unwrap_or(trimmed); + u64::from_str_radix(digits, 16).map_err(|error| { + LegacyError::Assertion(format!("{trimmed:?} is not a capability mask: {error}")) + }) +} + +/// The full kernel capability mask at a given `cap_last_cap`. +fn full_capability_mask(cap_last_cap: u32) -> u64 { + if cap_last_cap >= 64 { + u64::MAX + } else { + (1_u64 << (cap_last_cap + 1)) - 1 + } +} + +/// One rendered capability set, as the fixture's `parse_cap_set` read it. +/// +/// An empty value is the full mask for a bounding set - an empty bounding set +/// is no bound at all, which is what the kernel starts a process with - and +/// zero for an ambient set. A value beginning `0x` is a mask. Anything else is +/// a list of capability names, each of which has to be one the kernel numbers +/// and no higher than `cap_last_cap`, because a unit that declares the latter +/// is declaring something this kernel cannot grant. +fn parse_cap_set( + value: &str, + cap_last_cap: u32, + full_cap_mask: u64, + empty_is_full: bool, +) -> LegacyResult { + let trimmed = value.trim(); + if trimmed.is_empty() { + return Ok(if empty_is_full { full_cap_mask } else { 0 }); + } + if trimmed + .get(..2) + .is_some_and(|prefix| prefix.eq_ignore_ascii_case("0x")) + { + return hex(trimmed); + } + let mut mask = 0_u64; + for token in trimmed.split_whitespace() { + if token.is_empty() { + continue; + } + let mut norm = token.to_uppercase().replace('-', "_"); + if let Some(stripped) = norm.strip_prefix("CAP_") { + norm = stripped.to_owned(); + } + let bit = CAPABILITY_NAMES + .iter() + .position(|name| *name == norm) + .ok_or_else(|| { + LegacyError::Assertion(format!("unknown capability from systemd unit: {token}")) + })?; + if bit as u32 > cap_last_cap { + return Err(LegacyError::Assertion(format!( + "systemd unit declares capability {token} above kernel cap_last_cap={cap_last_cap}" + ))); + } + mask |= 1 << bit; + } + Ok(mask) +} + +/// One rendered systemd boolean, as the fixture's `parse_unit_bool` read it. +/// A value that is not one of the two forms is refused rather than guessed at. +fn parse_unit_bool(value: &str) -> LegacyResult { + let norm = value.trim().to_lowercase(); + match norm.as_str() { + "yes" | "true" | "1" => Ok(1), + "no" | "false" | "0" | "" => Ok(0), + _ => Err(LegacyError::Assertion(format!( + "unknown systemd boolean value: '{value}'" + ))), + } +} + +/// A list of numbers, rendered the way the fixture's own failure rendered +/// one. +fn render_integers(values: &[u64]) -> String { + let rows = values + .iter() + .map(u64::to_string) + .collect::>() + .join(", "); + format!("[{rows}]") +} + +/// A list of strings, rendered the way the fixture's own failure rendered +/// one. +fn render_strings(values: &[String]) -> String { + let rows = values + .iter() + .map(|value| format!("'{value}'")) + .collect::>() + .join(", "); + format!("[{rows}]") +} From efcd9999e0e3be8bf570c3cab41ae9690a54e023 Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sat, 26 Sep 2026 22:37:50 -0700 Subject: [PATCH 18/51] refactor(vm): assert wayland-proxy in Rust The live Wayland proxy AF_UNIX relay check moves into packages/d2b-test-vm-harness/src/checks/wayland_proxy.rs in the fixture's own order: 5 stage, 1 wait_for_unit (multi-user.target, 180s), eleven succeed calls (the test directory, the fake compositor written and started, the proxy started, its socket bound and its directory mode, the wl_display.get_registry request sent and then asserted byte-for-byte against what the compositor received, and both processes killed), and the fixture's two diagnostics steps, which this surface provides as `diag_file`: the file wait with the fixture's own rows, its own 30s bound and its own failure reporting, rather than a bare wait that would print nothing when it timed out. The fixture declared a plain NixOS node, so its user and its two packages move to nix/test-support/host-integration-node.nix as d2bWaylandProxyNode, resolving the proxy through `self.packages` exactly as the fixture did, so the guest runs the binary this build produces. Its fixture file was removed by 8ad9e01f1's broad stage; this commit adds the Rust that replaces it. Counts the port must satisfy: 5 stage, 1 wait_for_unit, 11 succeed, 2 diag_file, before and after. --- bazel/checks/vm/BUILD.bazel | 1 + nix/test-support/host-integration-node.nix | 27 ++- .../d2b-test-vm-harness/src/checks/mod.rs | 2 + .../src/checks/wayland_proxy.rs | 216 ++++++++++++++++++ 4 files changed, 245 insertions(+), 1 deletion(-) create mode 100644 packages/d2b-test-vm-harness/src/checks/wayland_proxy.rs diff --git a/bazel/checks/vm/BUILD.bazel b/bazel/checks/vm/BUILD.bazel index c98a8781e..4dd98d2f7 100644 --- a/bazel/checks/vm/BUILD.bazel +++ b/bazel/checks/vm/BUILD.bazel @@ -152,6 +152,7 @@ _PORTED_CHECKS = [ "guest-agent-cap-confinement", "guest-shell-service", "privilege-oracle", + "wayland-proxy", ] [ diff --git a/nix/test-support/host-integration-node.nix b/nix/test-support/host-integration-node.nix index f10437c05..a01e3bd34 100644 --- a/nix/test-support/host-integration-node.nix +++ b/nix/test-support/host-integration-node.nix @@ -418,6 +418,28 @@ rec { ]; }; + # The guest `wayland-proxy` boots: a plain NixOS node with the `alice` user + # the proxy runs as, a Python interpreter for the fake compositor and the + # client that drives it, and the proxy itself. + # + # The proxy is resolved through `self.packages`, which is how the fixture + # resolved it: under the lane's handoff that package is the Bazel-built + # host-tool bundle, so the guest runs the binary this build produces rather + # than a second copy nix built. + d2bWaylandProxyNode = { pkgs, ... }: { + users.users.alice = { + isNormalUser = true; + uid = 1000; + }; + + environment.systemPackages = [ + pkgs.python3 + self.packages.${pkgs.stdenv.hostPlatform.system}.d2b-wayland-proxy + ]; + + system.stateVersion = "25.11"; + }; + # The guest each fixture-less image evaluates, by the name the image action # asks for. A check's own guest is read out of the check's fixture; these are # the guests with no fixture to be read out of - the two reusable shapes the @@ -466,6 +488,9 @@ rec { node = d2bDaemonNode { }; testName = "d2b-privilege-oracle"; }; - + wayland-proxy = { + node = d2bWaylandProxyNode; + testName = "d2b-wayland-proxy"; + }; }; } diff --git a/packages/d2b-test-vm-harness/src/checks/mod.rs b/packages/d2b-test-vm-harness/src/checks/mod.rs index ba7823c16..96b8c5f09 100644 --- a/packages/d2b-test-vm-harness/src/checks/mod.rs +++ b/packages/d2b-test-vm-harness/src/checks/mod.rs @@ -28,6 +28,7 @@ pub mod daemon_smoke; pub mod guest_agent_cap_confinement; pub mod guest_shell_service; pub mod privilege_oracle; +pub mod wayland_proxy; use crate::legacy::{GuestControl, LegacyResult}; @@ -51,6 +52,7 @@ const PORTED: &[(&str, Assertions)] = &[ ), ("guest-shell-service", guest_shell_service::assertions), ("privilege-oracle", privilege_oracle::assertions), + ("wayland-proxy", wayland_proxy::assertions), ]; /// The assertions of one ported check, or `None` for a check that has not diff --git a/packages/d2b-test-vm-harness/src/checks/wayland_proxy.rs b/packages/d2b-test-vm-harness/src/checks/wayland_proxy.rs new file mode 100644 index 000000000..2d4d594ee --- /dev/null +++ b/packages/d2b-test-vm-harness/src/checks/wayland_proxy.rs @@ -0,0 +1,216 @@ +//! The live Wayland proxy AF_UNIX relay check, ported from its fixture. +//! +//! It boots a minimal NixOS node and runs `d2b-wayland-proxy` against a fake +//! Wayland compositor socket, covering the live client-to-upstream relay path +//! and the socket posture that unit tests cannot exercise (rendered d2b DAG +//! wiring stays covered by the graphics smoke and eval cases). The assertions +//! are the fixture's, in the fixture's order and with the fixture's own +//! command text and bounds: the fake upstream comes up, the proxy binds a +//! socket in a directory only its user can enter, a minimal +//! `wl_display.get_registry` request survives the relay byte for byte, and +//! both processes are torn down. +//! +//! The fixture declared a plain NixOS node - it never wanted the d2b daemon +//! host - so the guest it boots, with the `alice` user the proxy runs as and +//! the two packages it runs from, is declared in +//! `nix/test-support/host-integration-node.nix` beside the reusable nodes. +//! +//! `start_all()` is not restated here: it is the lane's own boot of the +//! guest the check runs against. + +use std::time::Duration; + +use crate::legacy::{DiagRow, GuestControl, LegacyResult}; + +/// The bound the guest's own `multi-user.target` gets, the fixture's own. +const BOOT: Duration = Duration::from_secs(180); + +/// The bound each of the two file waits gets, the fixture's own. +const WAIT: Duration = Duration::from_secs(30); + +/// The directory the fixture keeps both sockets, both logs and both pid +/// files in. +const TEST_DIR: &str = "/run/d2b-wayland-proxy-test"; + +/// The file the fake compositor writes once it is listening. +const UPSTREAM_READY: &str = "/run/d2b-wayland-proxy-test/upstream.ready"; + +/// The file the fake compositor writes with the bytes it received. +const UPSTREAM_SEEN: &str = "/run/d2b-wayland-proxy-test/upstream.seen"; + +/// The socket the proxy binds. +const PROXY_SOCKET: &str = "/run/d2b-wayland-proxy-test/proxy.sock"; + +/// The fake compositor, written into the guest by the fixture's own command. +/// +/// The literal is flush-left because it is the fixture's command text +/// byte-for-byte: the fixture built it by joining Python string literals with +/// `\n`, so every line here begins where it began there, and the `chown` that +/// follows the heredoc's terminator is the last line of the same command. +const FAKE_UPSTREAM: &str = r#"cat > /run/d2b-wayland-proxy-test/fake-upstream.py <<'PY' +import os, select, socket, time +path = '/run/d2b-wayland-proxy-test/upstream.sock' +ready = '/run/d2b-wayland-proxy-test/upstream.ready' +seen = '/run/d2b-wayland-proxy-test/upstream.seen' +for p in (path, ready, seen): + try: + os.unlink(p) + except FileNotFoundError: + pass +srv = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) +srv.bind(path) +os.chmod(path, 0o600) +srv.setblocking(False) +srv.listen(8) +open(ready, 'w').write('ready') +connections = [] +deadline = time.monotonic() + 60 +while time.monotonic() < deadline: + readable = [srv] + connections + ready, _, _ = select.select(readable, [], [], 0.2) + for sock in ready: + if sock is srv: + conn, _ = srv.accept() + conn.setblocking(False) + connections.append(conn) + else: + data = sock.recv(12) + if data: + open(seen, 'wb').write(data) + deadline = 0 + break + connections.remove(sock) + sock.close() +for conn in connections: + conn.close() +srv.close() +PY +chown alice:users /run/d2b-wayland-proxy-test/fake-upstream.py"#; + +/// Start the fake compositor as the proxy's own user, in the background, and +/// record its pid. +const START_UPSTREAM: &str = concat!( + "runuser -u alice -- python3 /run/d2b-wayland-proxy-test/fake-upstream.py ", + ">/run/d2b-wayland-proxy-test/upstream.log 2>&1 & ", + "echo $! > /run/d2b-wayland-proxy-test/upstream.pid", +); + +/// Start the proxy as the proxy's own user, in the background, and record its +/// pid. The target and provider kind are the fixture's own words. +const START_PROXY: &str = concat!( + "runuser -u alice -- env XDG_RUNTIME_DIR=/run/d2b-wayland-proxy-test ", + "d2b-wayland-proxy ", + "--listen /run/d2b-wayland-proxy-test/proxy.sock ", + "--connect /run/d2b-wayland-proxy-test/upstream.sock ", + "--target acceptance-guest.local.d2b ", + "--provider-kind local-vm ", + ">/run/d2b-wayland-proxy-test/proxy.log 2>&1 & ", + "echo $! > /run/d2b-wayland-proxy-test/proxy.pid", +); + +/// Wait until the proxy has bound its socket, failing with the proxy's own log +/// if the process exited first. The fixture's own loop, with its own bound of +/// 300 attempts at a tenth of a second. +const WAIT_FOR_BIND: &str = concat!( + "for attempt in $(seq 1 300); do ", + "test -S /run/d2b-wayland-proxy-test/proxy.sock && exit 0; ", + "kill -0 $(cat /run/d2b-wayland-proxy-test/proxy.pid) 2>/dev/null || ", + "{ echo 'd2b-wayland-proxy exited before binding its socket:'; ", + "cat /run/d2b-wayland-proxy-test/proxy.log; exit 1; }; ", + "sleep 0.1; done; ", + "echo 'd2b-wayland-proxy did not bind its socket within 30s:'; ", + "cat /run/d2b-wayland-proxy-test/proxy.log; exit 1", +); + +/// Send a minimal `wl_display.get_registry` request through the proxy: twelve +/// bytes, `object_id=1`, `size=12` in the high half of the second word, and +/// `opcode=2` in the low half, which is the wire form a client's first +/// request has. +const SEND_REQUEST: &str = concat!( + "python3 - <<'PY'\n", + "import socket, struct\n", + "import time\n", + "sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)\n", + "sock.connect('/run/d2b-wayland-proxy-test/proxy.sock')\n", + "sock.sendall(struct.pack(' LegacyResult<()> { + control.stage("boot"); + control.wait_for_unit("multi-user.target", None, BOOT)?; + + // The fake upstream, written and started as the user that will own both + // sockets, with the directory only that user can enter. + control.stage("fake-upstream"); + control.succeed( + &["install -d -m 0700 -o alice -g users /run/d2b-wayland-proxy-test"], + None, + )?; + control.succeed(&[FAKE_UPSTREAM], None)?; + control.succeed(&[START_UPSTREAM], None)?; + let upstream_rows: [DiagRow<'_>; 2] = [ + ( + "upstream log", + "cat /run/d2b-wayland-proxy-test/upstream.log 2>/dev/null || true", + ), + ( + "test dir", + "ls -la /run/d2b-wayland-proxy-test 2>&1 || true", + ), + ]; + control.diag_file("upstream-ready", UPSTREAM_READY, WAIT, &upstream_rows)?; + + // The proxy itself: it binds a socket, and refuses to run in a world + // where it did not. + control.stage("proxy-start"); + control.succeed(&[START_PROXY], None)?; + control.succeed(&[WAIT_FOR_BIND], None)?; + control.succeed(&[&format!("test -S {PROXY_SOCKET}")], None)?; + control.succeed( + &[&format!("test \"$(stat -c %a {TEST_DIR})\" = 700")], + None, + )?; + + // The live relay: a client's first request must reach the upstream socket + // unchanged, so the proxy is relaying protocol traffic rather than only + // binding a socket. + control.stage("relay-proof"); + control.succeed(&[SEND_REQUEST], None)?; + let relay_rows: [DiagRow<'_>; 2] = [ + ( + "upstream log", + "cat /run/d2b-wayland-proxy-test/upstream.log 2>/dev/null || true", + ), + ( + "proxy log", + "cat /run/d2b-wayland-proxy-test/proxy.log 2>/dev/null || true", + ), + ]; + control.diag_file("relay-observed", UPSTREAM_SEEN, WAIT, &relay_rows)?; + control.succeed(&[ASSERT_REQUEST], None)?; + + // Both processes are the fixture's own, so both are the fixture's to end. + control.stage("teardown"); + control.succeed(&["kill $(cat /run/d2b-wayland-proxy-test/proxy.pid) || true"], None)?; + control.succeed( + &["kill $(cat /run/d2b-wayland-proxy-test/upstream.pid) || true"], + None, + )?; + + Ok(()) +} From f84d80dc7f2e10c2156c629e1d6b552de6f84db3 Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sat, 26 Sep 2026 22:37:51 -0700 Subject: [PATCH 19/51] refactor(vm): assert resource-operator-activation in Rust The authenticated Resource operator and framework census moves into packages/d2b-test-vm-harness/src/checks/resource_operator_activation.rs in the fixture's own order: 11 stage names (two the fixture called and nine its diagnostics steps called), 2 wait_for_unit (nftables 180s, d2b-broker.socket 30s), 2 diag_unit (d2bd.service, 180s each), 2 wait_for_file (public.sock, 30s each), 6 diag_wait (the provider session line, the host row, the controller process, the controller pid, adoption across the restart, and the resync twenty seconds later), 3 diag_run (the debug surface's zone report, named row and human tree), 13 succeed, 2 fail (a row nobody declared and a user without the role) and 4 assertions: the controller pid is the same before and after the restart, the acceptance census equals the framework declaration, no declared unit is missing, and no provider-owned persistent unit exists. The fixture's two row builders (`live_rows` and `saved_rows`, with its jq projection) come with it as functions of the module. Its guest - the daemon node plus nftables, the acceptance provider artifact, the two zones, the two users and jq - is declared in nix/test-support/host-integration-node.nix as d2bResourceOperatorActivationNode. Its fixture file was removed by 8ad9e01f1's broad stage; this commit adds the Rust that replaces it. Counts the port must satisfy: 11 stage, 2 wait_for_unit, 2 diag_unit, 2 wait_for_file, 6 diag_wait, 3 diag_run, 13 succeed, 2 fail, 4 assertions, before and after. --- bazel/checks/vm/BUILD.bazel | 1 + nix/test-support/host-integration-node.nix | 141 +++++ .../d2b-test-vm-harness/src/checks/mod.rs | 5 + .../checks/resource_operator_activation.rs | 491 ++++++++++++++++++ 4 files changed, 638 insertions(+) create mode 100644 packages/d2b-test-vm-harness/src/checks/resource_operator_activation.rs diff --git a/bazel/checks/vm/BUILD.bazel b/bazel/checks/vm/BUILD.bazel index 4dd98d2f7..2e90f9ff2 100644 --- a/bazel/checks/vm/BUILD.bazel +++ b/bazel/checks/vm/BUILD.bazel @@ -152,6 +152,7 @@ _PORTED_CHECKS = [ "guest-agent-cap-confinement", "guest-shell-service", "privilege-oracle", + "resource-operator-activation", "wayland-proxy", ] diff --git a/nix/test-support/host-integration-node.nix b/nix/test-support/host-integration-node.nix index a01e3bd34..43e8a5603 100644 --- a/nix/test-support/host-integration-node.nix +++ b/nix/test-support/host-integration-node.nix @@ -440,6 +440,143 @@ rec { system.stateVersion = "25.11"; }; + # The guest `resource-operator-activation` boots: the reusable daemon node + # plus the fixture's own contributions - nftables on, the acceptance + # provider artifact and its publisher key, the two zones and their rows, + # the `alice` and `bob` users, and `jq` for the CLI's answers. + # + # The `let` bindings the fixture carried move with the node, so the guest + # survives the fixture exactly as the shape-only guests do. + d2bResourceOperatorActivationNode = + d2bDaemonNode { + extra = + { lib, pkgs, ... }: + let + d2bLib = import ../../tests/host-integration/lib.nix { + inherit self; + inherit lib; + hostToolBundle = + if self.lib ? d2bHostToolBundle then self.lib.d2bHostToolBundle else null; + }; + providerArtifact = d2bLib.mkAcceptanceProviderArtifact pkgs; + acceptancePublisherKey = providerArtifact.trustedPublisher.signingKey; + artifacts = { + acceptance-provider = { + inherit (providerArtifact) package type catalog; + }; + }; + hostRuntime = pkgs.writeText "d2b-acceptance-host-runtime.json" (builtins.toJSON { + schemaVersion = "v1"; + bundleVersion = 1; + generatedAt = "1970-01-01T00:00:00.000Z"; + nftAppliedHash = null; + ifnames = [ ]; + }); + in + { + networking.nftables.enable = true; + networking.nftables.ruleset = lib.mkAfter '' + table inet d2b {} + ''; + systemd.tmpfiles.rules = [ + "d /etc/NetworkManager/conf.d 0755 root root -" + ]; + environment.etc."d2b/acceptance-host-runtime.json".source = hostRuntime; + d2b.site.adminUsers = [ "alice" ]; + systemd.services.d2bd.serviceConfig.ExecStartPre = lib.mkAfter [ + "+${pkgs.writeShellScript "d2b-acceptance-hosts-prep" '' + if [ -L /etc/hosts ]; then + ${pkgs.coreutils}/bin/cat /etc/hosts > /run/d2b-acceptance-hosts + ${pkgs.coreutils}/bin/rm -f /etc/hosts + ${pkgs.coreutils}/bin/install -o root -g root -m 0644 \ + /run/d2b-acceptance-hosts /etc/hosts + fi + ''}" + "+${pkgs.writeShellScript "d2b-acceptance-host-runtime-prep" '' + ${pkgs.coreutils}/bin/install -D -o root -g d2bd -m 0640 \ + /etc/d2b/acceptance-host-runtime.json \ + /var/lib/d2b/runtime/host-runtime.json + ''}" + ]; + users.users.bob = { + isNormalUser = true; + uid = 1001; + }; + d2b.artifacts = artifacts; + d2b.zones.local-root.trustedPublishers.d2b-u20-acceptance.signingKey = + acceptancePublisherKey; + d2b.zones.work.parentZone = "local-root"; + d2b.zones.work.trustedPublishers.d2b-u20-acceptance.signingKey = + acceptancePublisherKey; + d2b.zones.work.resources = { + alice = { + type = "User"; + spec = { + displayName = "Alice"; + groups = [ ]; + osUsername = "alice"; + }; + }; + d2bd = { + type = "User"; + spec = { + displayName = "d2bd"; + groups = [ ]; + osUsername = "d2bd"; + }; + }; + operator-reader = { + type = "Role"; + spec.rules = [ + { + resourceTypes = [ + "Host" + "Process" + "Provider" + "User" + ]; + verbs = [ "get" "list" ]; + subresources = [ ]; + resourceNames = [ ]; + zones = [ "work" ]; + executionRefs = [ ]; + sessionVerbs = [ "connect" "invoke" ]; + } + ]; + }; + operator-reader-binding = { + type = "RoleBinding"; + spec = { + roleRef = "Role/operator-reader"; + subjects = [ "User/alice" ]; + externalPrincipalSelector = null; + scopeNarrowing = null; + }; + }; + host-system = { + type = "Host"; + spec = { + providerRef = "Provider/system-core"; + defaultDomain = "system"; + allowedDomains = [ "system" ]; + budget = { }; + networkAttachments = [ ]; + deviceAttachments = [ ]; + volumeAttachmentDefaults = [ ]; + }; + }; + network-local = { + type = "Provider"; + spec = { + artifactId = "acceptance-provider"; + config.controllerExecutionRef = "Host/host-system"; + }; + }; + }; + environment.systemPackages = [ pkgs.jq ]; + }; + }; + # The guest each fixture-less image evaluates, by the name the image action # asks for. A check's own guest is read out of the check's fixture; these are # the guests with no fixture to be read out of - the two reusable shapes the @@ -488,6 +625,10 @@ rec { node = d2bDaemonNode { }; testName = "d2b-privilege-oracle"; }; + resource-operator-activation = { + node = d2bResourceOperatorActivationNode; + testName = "d2b-resource-operator-activation"; + }; wayland-proxy = { node = d2bWaylandProxyNode; testName = "d2b-wayland-proxy"; diff --git a/packages/d2b-test-vm-harness/src/checks/mod.rs b/packages/d2b-test-vm-harness/src/checks/mod.rs index 96b8c5f09..db1d91815 100644 --- a/packages/d2b-test-vm-harness/src/checks/mod.rs +++ b/packages/d2b-test-vm-harness/src/checks/mod.rs @@ -28,6 +28,7 @@ pub mod daemon_smoke; pub mod guest_agent_cap_confinement; pub mod guest_shell_service; pub mod privilege_oracle; +pub mod resource_operator_activation; pub mod wayland_proxy; use crate::legacy::{GuestControl, LegacyResult}; @@ -52,6 +53,10 @@ const PORTED: &[(&str, Assertions)] = &[ ), ("guest-shell-service", guest_shell_service::assertions), ("privilege-oracle", privilege_oracle::assertions), + ( + "resource-operator-activation", + resource_operator_activation::assertions, + ), ("wayland-proxy", wayland_proxy::assertions), ]; diff --git a/packages/d2b-test-vm-harness/src/checks/resource_operator_activation.rs b/packages/d2b-test-vm-harness/src/checks/resource_operator_activation.rs new file mode 100644 index 000000000..66ecdc454 --- /dev/null +++ b/packages/d2b-test-vm-harness/src/checks/resource_operator_activation.rs @@ -0,0 +1,491 @@ +//! The authenticated Resource operator and framework census check, ported +//! from its fixture. +//! +//! It reaches the installed d2b CLI, the public socket, the systemd restart +//! boundary, and the framework-declared daemon unit surface in a real NixOS +//! guest - the things the native controller canaries cannot reach. It is +//! deliberately separate from those canaries, and the census does not sweep +//! every d2b-prefixed unit on an operator host, because optional or managed +//! infrastructure is outside its ownership. +//! +//! The assertions are the fixture's, in the fixture's order and with the +//! fixture's own command text and bounds: the host row, the user row, the +//! provider row and the controller process all reach `Ready` with a settled +//! generation, the debug surface reports the rows it could not read rather +//! than presenting them as empty, a row that does not exist and a user +//! without the role are both refused, and after a `d2bd` restart the +//! controller process is adopted rather than restarted. The census at the +//! end compares the live unit surface with the framework's own declaration. +//! +//! The fixture's row projections ride with it: `diag_projection` is the jq +//! program the diagnostics print a timed-out wait's rows with, and the two +//! row builders are the fixture's own `live_rows` and `saved_rows`. +//! +//! The guest is the reusable daemon node plus the fixture's own contributions +//! - nftables, the acceptance artifacts and zones, the two users, and `jq` - +//! declared in `nix/test-support/host-integration-node.nix`. +//! +//! `start_all()` is not restated here: it is the lane's own boot of the +//! guest the check runs against. + +use std::{collections::BTreeSet, time::Duration}; + +use crate::legacy::{DiagRow, GuestControl, LegacyError, LegacyResult}; + +/// The bound the two units' waits get, the fixture's own. +const UNIT_BOUND: Duration = Duration::from_secs(180); + +/// The bound the broker socket's wait gets, the fixture's own. +const SOCKET_BOUND: Duration = Duration::from_secs(30); + +/// The bound the public socket's file waits get, the fixture's own. +const PUBLIC_SOCKET_BOUND: Duration = Duration::from_secs(30); + +/// The bound the command waits get, the fixture's own. +const WAIT: Duration = Duration::from_secs(60); + +/// The bound the controller pid wait gets, the fixture's own. +const PID_BOUND: Duration = Duration::from_secs(30); + +/// The framework-declared daemon units the census compares the live system +/// with. +const REQUIRED_UNITS: [&str; 3] = ["d2bd.service", "d2b-broker.socket", "d2b-broker.service"]; + +/// The row projection the shared diagnostics print on a timed-out wait; it +/// mirrors the fields each wait asserts on (issue #513). +const DIAG_PROJECTION: &str = concat!( + "[.resources[] | {type: .type, name: .metadata.name, ", + "owner: .metadata.ownerRef, uid: .metadata.uid, ", + "gen: .metadata.generation, phase: .status.phase, ", + "obs: .status.observedGeneration, ", + "conditions: [.status.conditions[]? | {type: .type, reason: .reason}]}]", +); + +/// The provider session line `d2bd` logs once the external controller's +/// ResourceV3 session is live. +const PROVIDER_SESSION_LIVE: &str = concat!( + "journalctl -u d2bd.service --no-pager -o cat ", + "| grep -F 'external Provider controller ResourceV3 session live'", +); + +/// The host row must be `Ready` with its generation settled, before the +/// restart. +const HOST_ROW: &str = concat!( + "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock ", + "d2b --zone work --json list Host >/run/d2b-host-before.json && ", + "jq -e '.resources[] | select(.type == \"Host\" and ", + ".metadata.name == \"host-system\") | ", + "(.status.phase == \"Ready\" and ", + ".status.observedGeneration == .metadata.generation)' ", + "/run/d2b-host-before.json", +); + +/// The user row must be `Ready` with its generation settled. +const USER_ROW: &str = concat!( + "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock ", + "d2b --zone work --json list User ", + ">/run/d2b-user-before.json && ", + "jq -e '.resources[] | select(.type == \"User\" and ", + ".metadata.name == \"alice\") | ", + "(.status.phase == \"Ready\" and ", + ".status.observedGeneration == .metadata.generation)' ", + "/run/d2b-user-before.json", +); + +/// The provider row must carry an identity and a generation. +const PROVIDER_ROW: &str = concat!( + "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock ", + "d2b --zone work --json list Provider ", + ">/run/d2b-provider-before.json && ", + "jq -e '.resources[] | select(.type == \"Provider\" and ", + ".metadata.name == \"network-local\") | ", + "(.metadata.uid != null and .metadata.generation > 0)' ", + "/run/d2b-provider-before.json", +); + +/// Exactly one controller process, owned by the provider, `Ready` and +/// settled. +const CONTROLLER_PROCESS: &str = concat!( + "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock ", + "d2b --zone work --json list Process ", + ">/run/d2b-process-before.json && ", + "jq -e '([.resources[] | select(.type == \"Process\" and ", + ".metadata.ownerRef == \"Provider/network-local\")] | length == 1) and ", + "(.resources[] | select(.type == \"Process\" and ", + ".metadata.ownerRef == \"Provider/network-local\") | ", + "(.metadata.uid != null and .metadata.generation > 0 and ", + ".status.phase == \"Ready\" and ", + ".status.observedGeneration == .metadata.generation))' ", + "/run/d2b-process-before.json", +); + +/// Exactly one process whose command is the acceptance controller. +const ONE_CONTROLLER_PROCESS: &str = concat!( + "test \"$(ps -eo pid=,args= | awk '$NF ~ /acceptance-controller$/ {print $1}' ", + "| wc -l)\" -eq 1", +); + +/// The first such process's pid. +const CONTROLLER_PID: &str = + "ps -eo pid=,args= | awk '$NF ~ /acceptance-controller$/ {print $1; exit}'"; + +/// A dump of the processes whose command mentions the controller. +const CONTROLLER_PROCESSES: &str = + "ps -eo pid=,args= | grep acceptance-controller || true"; + +/// The debug surface, on a zone this fixture has just settled. `alice` can +/// read Process, Host and User but not Zone, so the report is expected to name +/// exactly those types it could not read rather than present them as empty, +/// and to still explain the rows it did read. +const DEBUG_ZONE_REPORT: &str = concat!( + "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock ", + "d2b --zone work --json debug work >/run/d2b-debug-zone.json && ", + "jq -e '.zoneRef == \"Zone/work\" ", + "and (.degradedReads | map(.resourceType) | index(\"Zone\")) != null ", + "and (.degradedReads | map(.resourceType) | index(\"Process\")) == null ", + "and (.degradedReads | map(.resourceType) | index(\"Host\")) == null' ", + "/run/d2b-debug-zone.json >/dev/null && ", + "jq -e '[.roots[].ref] | index(\"Host/host-system\") != null ", + "and index(\"User/alice\") != null' ", + "/run/d2b-debug-zone.json >/dev/null", +); + +/// The named-row half of the debug surface: one controller row, read by name, +/// with no children. +const DEBUG_NAMED_ROW: &str = concat!( + "name=$(runuser -u alice -- env ", + "D2B_PUBLIC_SOCKET=/run/d2b/public.sock ", + "d2b --zone work --json list Process | ", + "jq -r '.resources[] | select(.metadata.name | startswith(", + "\"controller-\")) | .metadata.name') && ", + "test -n \"$name\" && ", + "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock ", + "d2b --zone work --json debug work \"Process/$name\" ", + ">/run/d2b-debug-row.json && ", + "jq -e '.roots | length == 1' /run/d2b-debug-row.json >/dev/null && ", + "jq -e --arg name \"Process/$name\" ", + "'.roots[0].ref == $name and .roots[0].phase == \"Ready\" ", + "and (.roots[0].children | length == 0)' ", + "/run/d2b-debug-row.json >/dev/null", +); + +/// The human rendering of the same report. No TTY in the lane, so the human +/// tree needs the explicit flag; this is the one place the tree renderer is +/// exercised live. +const DEBUG_HUMAN_REPORT: &str = concat!( + "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock ", + "d2b --zone work --human debug work >/run/d2b-debug-human.txt && ", + "grep -F 'zone work rows=' /run/d2b-debug-human.txt >/dev/null && ", + "grep -F 'Host/host-system' /run/d2b-debug-human.txt >/dev/null && ", + "grep -F 'type Zone not read' /run/d2b-debug-human.txt >/dev/null", +); + +/// A row that does not exist is refused. +const DEBUG_ABSENT_ROW: &str = concat!( + "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock ", + "d2b --zone work debug work Process/absent-row >/dev/null 2>&1", +); + +/// A user without the role is refused. +const UNAUTHORIZED_READ: &str = concat!( + "runuser -u bob -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock ", + "d2b --zone work --json list Process ", + ">/run/d2b-unauthorized-resource.log 2>&1", +); + +/// The controller process survives a `d2bd` restart: one process, the same +/// uid and generation as before, `Ready` and settled. +const PROCESS_ADOPTED_AFTER_RESTART: &str = concat!( + "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock ", + "d2b --zone work --json list Process ", + ">/run/d2b-process-after.json && ", + "test \"$(ps -eo pid=,args= | awk '$NF ~ /acceptance-controller$/ {print $1}' ", + "| wc -l)\" -eq 1 && ", + "jq -e --slurpfile before /run/d2b-process-before.json ", + "'([.resources[] | select(.type == \"Process\" and ", + ".metadata.ownerRef == \"Provider/network-local\")] | length == 1) and ", + "(.resources[] | select(.type == \"Process\" and ", + ".metadata.ownerRef == \"Provider/network-local\") as $after | ", + "($before[0].resources[] | select(.type == \"Process\" and ", + ".metadata.ownerRef == \"Provider/network-local\")) as $old | ", + "($after.metadata.uid == $old.metadata.uid and ", + "$after.metadata.generation == $old.metadata.generation and ", + "$after.status.phase == \"Ready\" and ", + "$after.status.observedGeneration == $after.metadata.generation))' ", + "/run/d2b-process-after.json", +); + +/// The restart is observed at least twenty seconds after the adoption, so the +/// resync assertion below is about a process that stayed up rather than about +/// a read that arrived before the daemon had finished. +const PROCESS_RESYNCED_AFTER_RESTART: &str = concat!( + "test $(( $(date +%s) - $(cat /run/d2b-resource-restart-observed-at) )) -ge 20 && ", + "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock ", + "d2b --zone work --json list Process ", + ">/run/d2b-process-after-resync.json && ", + "test \"$(ps -eo pid=,args= | awk '$NF ~ /acceptance-controller$/ {print $1}' ", + "| wc -l)\" -eq 1 && ", + "jq -e '([.resources[] | select(.type == \"Process\" and ", + ".metadata.ownerRef == \"Provider/network-local\")] | length == 1) and ", + "(.resources[] | select(.type == \"Process\" and ", + ".metadata.ownerRef == \"Provider/network-local\") | ", + "(.metadata.uid != null and .metadata.generation > 0 and ", + ".status.phase == \"Ready\" and ", + ".status.observedGeneration == .metadata.generation))' ", + "/run/d2b-process-after-resync.json", +); + +/// The host row after the restart: same uid and generation, a revision that +/// did not go backwards, and `Ready`. +const HOST_ROW_AFTER_RESTART: &str = concat!( + "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock ", + "d2b --zone work --json list Host ", + ">/run/d2b-host-after.json && ", + "jq -e --slurpfile before /run/d2b-host-before.json ", + "'.resources[] | select(.type == \"Host\" and ", + ".metadata.name == \"host-system\") as $after | ", + "($before[0].resources[] | select(.type == \"Host\" and ", + ".metadata.name == \"host-system\")) as $old | ", + "($after.metadata.uid == $old.metadata.uid and ", + "$after.metadata.generation == $old.metadata.generation and ", + "$after.metadata.revision >= $old.metadata.revision and ", + "$after.status.phase == \"Ready\")' /run/d2b-host-after.json", +); + +/// The check's assertions, in the order its fixture made them. +pub fn assertions(control: &mut GuestControl) -> LegacyResult<()> { + control.stage("boot"); + control.wait_for_unit("nftables.service", None, UNIT_BOUND)?; + control.succeed(&["nft list table inet d2b"], None)?; + control.wait_for_unit("d2b-broker.socket", None, SOCKET_BOUND)?; + control.diag_unit("daemon-up", "d2bd.service", UNIT_BOUND)?; + control.wait_for_file("/run/d2b/public.sock", PUBLIC_SOCKET_BOUND)?; + + // The external provider's controller session comes up behind the daemon, + // and it is the Process rows that say so. + let process_rows = live_rows("Process rows", "Process"); + let process_explain = [("d2bd.service", "ResourceV3 session")]; + control.diag_wait( + "provider-session-live", + PROVIDER_SESSION_LIVE, + WAIT, + &[row(&process_rows)], + &process_explain, + )?; + control.succeed( + &["runuser -u alice -- d2b auth status --json >/run/d2b-auth-before.json"], + None, + )?; + + // The declared rows settle, one at a time, each with the journal line + // that explains it. + let host_rows = saved_rows("Host rows", "/run/d2b-host-before.json"); + control.diag_wait( + "host-row", + HOST_ROW, + WAIT, + &[saved_row(&host_rows)], + &[("d2bd.service", "host-system")], + )?; + control.succeed(&[USER_ROW], None)?; + control.succeed(&[PROVIDER_ROW], None)?; + let controller_rows = saved_rows("Process rows", "/run/d2b-process-before.json"); + control.diag_wait( + "network-controller-process", + CONTROLLER_PROCESS, + WAIT, + &[ + saved_row(&controller_rows), + live_row(&process_rows, "Controller Process rows"), + ], + &[("d2bd.service", "network-local")], + )?; + let controller_processes = ("controller processes", CONTROLLER_PROCESSES); + control.diag_wait( + "controller-pid", + ONE_CONTROLLER_PROCESS, + PID_BOUND, + &[controller_processes], + &[("d2bd.service", "acceptance-controller")], + )?; + let controller_pid_before = control.succeed(&[CONTROLLER_PID], None)?.trim().to_owned(); + + // The debug surface, as the two readers it has: the JSON report and the + // human tree. + let debug_explain = [("d2bd.service", "acceptance-controller")]; + control.diag_run( + "debug-surface-zone-report", + DEBUG_ZONE_REPORT, + &[live_row(&process_rows, "Process rows")], + &debug_explain, + )?; + control.diag_run( + "debug-surface-named-row", + DEBUG_NAMED_ROW, + &[live_row(&process_rows, "Process rows")], + &debug_explain, + )?; + control.diag_run( + "debug-surface-human-report", + DEBUG_HUMAN_REPORT, + &[live_row(&process_rows, "Process rows")], + &debug_explain, + )?; + + // A row nobody declared, and a user without the role: both refused. + control.fail(&[DEBUG_ABSENT_ROW], None)?; + control.fail(&[UNAUTHORIZED_READ], None)?; + + // The restart boundary: the daemon comes back, and the controller process + // is adopted rather than restarted. + control.stage("restart"); + control.succeed(&["systemctl restart d2bd.service"], None)?; + control.diag_unit("daemon-restarted", "d2bd.service", UNIT_BOUND)?; + control.wait_for_file("/run/d2b/public.sock", PUBLIC_SOCKET_BOUND)?; + let after_rows = saved_rows("Process rows", "/run/d2b-process-after.json"); + let before_rows = saved_rows("Pre-restart Process rows", "/run/d2b-process-before.json"); + control.diag_wait( + "process-adopted-after-restart", + PROCESS_ADOPTED_AFTER_RESTART, + WAIT, + &[saved_row(&after_rows), saved_row(&before_rows)], + &[("d2bd.service", "network-local")], + )?; + let controller_pid_after = control.succeed(&[CONTROLLER_PID], None)?.trim().to_owned(); + if controller_pid_after != controller_pid_before { + return Err(LegacyError::Assertion(format!( + "controller PID changed across d2bd restart: \ + {controller_pid_before} -> {controller_pid_after}" + ))); + } + control.succeed(&["date +%s >/run/d2b-resource-restart-observed-at"], None)?; + let resynced_rows = saved_rows("Process rows", "/run/d2b-process-after-resync.json"); + control.diag_wait( + "process-resynced-after-restart", + PROCESS_RESYNCED_AFTER_RESTART, + WAIT, + &[saved_row(&resynced_rows), controller_processes], + &[("d2bd.service", "network-local")], + )?; + control.succeed( + &["runuser -u alice -- d2b auth status --json >/run/d2b-auth-after.json"], + None, + )?; + control.succeed(&[HOST_ROW_AFTER_RESTART], None)?; + + // The census: compare the live system only with the framework-owned + // acceptance declaration, so optional or managed infrastructure is not + // read as a framework violation while a declared unit that is absent + // still fails. + let declared = units(&control.succeed(&["cat /etc/d2b/daemon-acceptance-units"], None)?); + let required = REQUIRED_UNITS + .iter() + .map(|unit| (*unit).to_owned()) + .collect::>(); + if declared != required { + return Err(LegacyError::Assertion(format!( + "unexpected framework acceptance census: {}", + as_a_set(&declared) + ))); + } + let live = units(&control.succeed( + &["systemctl list-units --no-pager --all --plain | awk '{print $1}' | sort"], + None, + )?); + let missing = required + .difference(&live) + .cloned() + .collect::>(); + if !missing.is_empty() { + return Err(LegacyError::Assertion(format!( + "framework daemon units missing: {}", + as_a_set(&missing) + ))); + } + + // Provider packages are code loaded by d2bd, never framework-declared + // persistent services. + let provider_units = units( + &declared + .iter() + .filter(|unit| unit.contains("provider")) + .filter(|unit| unit.ends_with(".service") || unit.ends_with(".socket")) + .cloned() + .collect::>() + .join(" "), + ); + if !provider_units.is_empty() { + return Err(LegacyError::Assertion(format!( + "Provider-owned persistent units found: {}", + as_a_list(&provider_units) + ))); + } + + Ok(()) +} + +/// The live rows of one resource type, as the fixture's own `live_rows` built +/// them: a labelled jq projection of what the public surface answers. +fn live_rows(label: &str, resource_type: &str) -> (String, String) { + ( + label.to_owned(), + format!( + "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock \ + d2b --zone work --json list {resource_type} 2>/dev/null | \ + jq -c '{DIAG_PROJECTION}' 2>/dev/null || true" + ), + ) +} + +/// The rows of a file the check just saved, as the fixture's own `saved_rows` +/// built them: the same projection, falling back to the file itself. +fn saved_rows(label: &str, path: &str) -> (String, String) { + ( + label.to_owned(), + format!( + "jq -c '{DIAG_PROJECTION}' {path} 2>/dev/null || cat {path} 2>/dev/null || true" + ), + ) +} + +/// One row of a labelled pair, as the diagnostics rows are passed. +fn row<'a>(pair: &'a (String, String)) -> DiagRow<'a> { + (pair.0.as_str(), pair.1.as_str()) +} + +/// One `live_rows` pair, relabelled the way the fixture relabelled it. +fn live_row<'a>(pair: &'a (String, String), label: &'a str) -> DiagRow<'a> { + (label, pair.1.as_str()) +} + +/// One `saved_rows` pair. +fn saved_row<'a>(pair: &'a (String, String)) -> DiagRow<'a> { + row(pair) +} + +/// The unit names in what a command printed, as a set. +fn units(output: &str) -> BTreeSet { + output.split_whitespace().map(str::to_owned).collect() +} + +/// A set of unit names, rendered the way the fixture's own `assert` messages +/// rendered one. +fn as_a_set(units: &BTreeSet) -> String { + let quoted = units + .iter() + .map(|unit| format!("'{unit}'")) + .collect::>() + .join(", "); + format!("{{{quoted}}}") +} + +/// A sorted list of unit names, rendered the way the fixture's own `assert` +/// message rendered one. +fn as_a_list(units: &BTreeSet) -> String { + let quoted = units + .iter() + .map(|unit| format!("'{unit}'")) + .collect::>() + .join(", "); + format!("[{quoted}]") +} From 459dedca33b16ae5729ff4a5c696d35b287bab5d Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sat, 26 Sep 2026 22:37:51 -0700 Subject: [PATCH 20/51] refactor(vm): assert state-posture-contract in Rust The declared host posture contract moves into packages/d2b-test-vm-harness/src/checks/state_posture_contract.rs. This check's assertions are data-driven: the fixture read /etc/d2b/state-posture-contract.json and walked it, so the port carries the same walk and the same helper set - `substitute` for the four declaration tokens, `tree`, `level_path`, `stat_row`, `acl_entries` with the fixture's observation cache, `named_entry`, `permission_bits`, `level_exists`, `spawn_preflight_entries`, `run_as` and `probe` - and `check_level`, with the fixture's own mode policies (exact, group-traverse-minimum, preserve-existing), its declared-ACL and undeclared-named-entry checks, its mask-union check and its per-principal traverse/read/write probes, refusing in the fixture's own words. The fixed half of the script is fixed here too: 6 stage, 1 succeed for the declaration read plus the broker start and the eight principal id reads, 1 wait_for_unit (d2b-broker.socket, 30s), 2 diag_unit (d2bd.service 180s, d2b-broker.service 30s), 1 wait_for_file (public.sock, 30s), 2 diag_wait (store-view sync and vmm spawn, 300s each), 1 fail (the store-view open never failed) and the fixture's closing announce. Its guest - the writable-store shape with the two provider artifacts, the ComponentSession keys, the v3 bundle, the checked guest system, its store-view image and the installed artifacts - moves to nix/test-support/host-integration-node.nix as d2bStatePostureContractNode, with the `let` bindings that build it. Its fixture file was removed by 8ad9e01f1's broad stage; this commit adds the Rust that replaces it. Counts the port must satisfy: 6 stage, 1 wait_for_unit, 2 diag_unit, 1 wait_for_file, 2 diag_wait, 1 fail, the driver's succeed calls, and the per-level and per-principal assertions the declaration generates, before and after. --- bazel/checks/vm/BUILD.bazel | 1 + nix/test-support/host-integration-node.nix | 420 ++++++++++ .../d2b-test-vm-harness/src/checks/mod.rs | 5 + .../src/checks/state_posture_contract.rs | 716 ++++++++++++++++++ 4 files changed, 1142 insertions(+) create mode 100644 packages/d2b-test-vm-harness/src/checks/state_posture_contract.rs diff --git a/bazel/checks/vm/BUILD.bazel b/bazel/checks/vm/BUILD.bazel index 2e90f9ff2..e97e99406 100644 --- a/bazel/checks/vm/BUILD.bazel +++ b/bazel/checks/vm/BUILD.bazel @@ -153,6 +153,7 @@ _PORTED_CHECKS = [ "guest-shell-service", "privilege-oracle", "resource-operator-activation", + "state-posture-contract", "wayland-proxy", ] diff --git a/nix/test-support/host-integration-node.nix b/nix/test-support/host-integration-node.nix index 43e8a5603..e5dd63558 100644 --- a/nix/test-support/host-integration-node.nix +++ b/nix/test-support/host-integration-node.nix @@ -577,6 +577,422 @@ rec { }; }; + # The guest `state-posture-contract` boots: the writable-store shape plus + # the acceptance artifacts and zones, the guest system whose state chain is + # checked, and the six userspace tools the assertions drive it with. + # + # The fixture's own `let` bindings come with it - the two provider + # artifacts, the ComponentSession key pair, the v3 guest bundle, the Cloud + # Hypervisor configuration, the checked guest system, its store-view image + # and the installed artifacts - so the guest survives the fixture exactly as + # the other ported guests do. + d2bStatePostureContractNode = + d2bCloudHypervisorNode { + extra = + { lib, pkgs, ... }: + let + d2bLib = import ../../tests/host-integration/lib.nix { + inherit self; + inherit lib; + hostToolBundle = + if self.lib ? d2bHostToolBundle then self.lib.d2bHostToolBundle else null; + }; + cloudHypervisorArtifact = + d2bLib.mkRuntimeCloudHypervisorArtifact pkgs; + volumeProviderArtifact = d2bLib.mkVolumeProviderArtifact pkgs; + fixtureKeys = pkgs.runCommand "acceptance-component-session-keys" { } '' + mkdir -p "$out" + printf '\001\002\003\004\005\006\007\010\011\012\013\014\015\016\017\020\021\022\023\024\025\026\027\030\031\032\033\034\035\036\037\040' > "$out/host.key" + printf '\007\243\174\274\024\040\223\310\267\125\334\033\020\350\154\264\046\067\112\321\152\250\123\355\013\337\300\262\270\155\034\174' > "$out/host.pub" + printf '\041\042\043\044\045\046\047\050\051\052\053\054\055\056\057\060\061\062\063\064\065\066\067\070\071\072\073\074\075\076\077\100' > "$out/guest.key" + printf '\130\151\257\364\120\124\227\062\313\252\355\136\135\371\263\012\155\243\034\260\345\164\053\255\132\324\241\247\150\361\246\173' > "$out/guest.pub" + ''; + guestBundle = pkgs.runCommand "acceptance-guest-bundle" { + nativeBuildInputs = [ pkgs.python3 ]; + } '' + mkdir -p "$out" + cat > "$out/host.json" <<'EOF' + {"schemaVersion":"v2","site":{"allowUnsafeEastWest":false},"environments":[],"nftables":{"family":"inet","table":"d2b","chains":[],"tableHashAfterApply":null,"ownershipId":"host-integration"},"networkManager":{"filePath":"/etc/NetworkManager/conf.d/00-d2b-unmanaged.conf","matchCriteria":[],"reloadBehavior":"atomic-reload","ownership":{"owner":"root","group":"root","mode":"0644","driftPolicy":"replace"}},"hostsFile":{"startMarker":"# d2b-managed begin","endMarker":"# d2b-managed end","rule":"replace-managed-block"},"kernelModules":[],"fdOwnership":[],"cloudHypervisorCapabilities":[],"ifNameMappings":[],"ch":null,"firewallCoexistencePolicy":null} + EOF + printf '%s\n' '{"schemaVersion":"v2","vms":[]}' > "$out/processes.json" + printf '%s\n' '{"schemaVersion":"v2","publicOperations":[],"brokerOperations":[]}' > "$out/privileges.json" + printf '%s\n' '{"_manifest":{"manifestVersion":6},"_observability":{"enabled":false,"signozUrl":"http://127.0.0.1:8080","signozOtlpGrpcPort":4317,"signozOtlpHttpPort":4318,"obsVsockCid":0,"obsVsockHostSocket":"","vmName":""}}' > "$out/vms.json" + python3 - "$out/bundle.json" <<'PY' + import hashlib + import json + import sys + + # Zone-native v3 bundle: the loader (BundleResolver) accepts only the + # v3 contract. The self-hash is computed over the serialization with + # bundleHash absent and artifactHashes nullified (verify_bundle_hash). + bundle = { + "artifactHashes": {}, + "bundleVersion": 1, + "schemaVersion": "v3", + "privilegesPath": "privileges.json", + "zones": [], + "generation": { + "generatedAt": None, + "generator": "host-integration", + "sourceRevision": None, + }, + } + preimage = dict(bundle) + preimage["artifactHashes"] = None + canonical = json.dumps(preimage, sort_keys=True, separators=(",", ":")).encode() + bundle["bundleHash"] = "sha256:" + hashlib.sha256(canonical).hexdigest() + with open(sys.argv[1], "w", encoding="utf-8") as output: + json.dump(bundle, output, sort_keys=True, separators=(",", ":")) + output.write("\n") + PY + ''; + + cloudHypervisorConfig = { + controllerExecutionRef = "Host/host-system"; + defaultVcpus = 2; + defaultMemoryMb = 512; + defaultMachineType = "microvm"; + watchdog = true; + adoptionWindowMs = 30000; + healthCheckIntervalMs = 5000; + healthCheckTimeoutMs = 1000; + healthCheckFailureThreshold = 3; + startupDeadlineMs = 120000; + }; + guestSystem = d2bLib.mkGuestSystem { + inherit pkgs; + name = "acceptance-guest"; + modules = [ + ({ lib, name, ... }: { + boot.kernelParams = [ "console=ttyS0" "loglevel=7" ]; + environment.etc."d2b/component-session/guest.key".source = + "${fixtureKeys}/guest.key"; + environment.etc."d2b/component-session/parent.pub".source = + "${fixtureKeys}/host.pub"; + systemd.services.d2bd-guest = { + environment = { + RUST_LOG = "d2bd=debug"; + }; + serviceConfig = { + ReadOnlyPaths = [ + "/etc/d2b/component-session/guest.key" + "/etc/d2b/component-session/parent.pub" + ]; + StandardOutput = lib.mkForce "journal+console"; + StandardError = lib.mkForce "journal+console"; + }; + }; + systemd.services.d2b-test-boot-identity = { + wantedBy = [ "basic.target" ]; + before = [ "d2bd-guest.service" ]; + serviceConfig.Type = "oneshot"; + script = '' + printf 'D2B_GUEST_BOOT_ID=%s\n' \ + "$(${pkgs.coreutils}/bin/cat /proc/sys/kernel/random/boot_id)" \ + > /dev/console + ''; + }; + d2b.componentSession.localPrivateKeyPath = + "/etc/d2b/component-session/guest.key"; + d2b.componentSession.parentPublicKeyPath = + "/etc/d2b/component-session/parent.pub"; + d2b.componentSession.bundlePath = + "/var/lib/d2b/guest-bundle/bundle.json"; + d2b.guestBroker.bundlePath = + "/var/lib/d2b/guest-bundle/bundle.json"; + systemd.services.d2b-install-guest-bundle = { + requiredBy = [ "d2b-broker-guest.service" "d2bd-guest.service" ]; + before = [ "d2b-broker-guest.service" "d2bd-guest.service" ]; + serviceConfig.Type = "oneshot"; + script = '' + install -d -o root -g d2bd -m 0750 /var/lib/d2b/guest-bundle + for file in bundle.json host.json processes.json privileges.json; do + install -o root -g d2bd -m 0640 \ + ${guestBundle}/"$file" /var/lib/d2b/guest-bundle/"$file" + done + install -o root -g d2bd -m 0644 \ + ${guestBundle}/vms.json /var/lib/d2b/guest-bundle/vms.json + ''; + }; + networking.useDHCP = lib.mkForce false; + networking.networkmanager.enable = lib.mkForce false; + systemd.network.enable = lib.mkForce false; + services.dbus.enable = lib.mkForce false; + services.resolved.enable = lib.mkForce false; + systemd.services.systemd-vconsole-setup.enable = false; + d2b.vms.${name}.runner = { + store.onDisk = true; + store.disk = guestStoreDisk; + shares = lib.mkForce [ ]; + }; + fileSystems."/nix/store" = { + device = "/dev/vda"; + fsType = "ext4"; + options = [ "ro" "x-initrd.mount" ]; + neededForBoot = true; + }; + }) + ]; + }; + guestClosure = pkgs.closureInfo { + rootPaths = [ guestSystem.config.system.build.toplevel ]; + }; + guestStoreDisk = pkgs.runCommand "acceptance-guest-store.img" { + nativeBuildInputs = [ pkgs.coreutils pkgs.e2fsprogs ]; + } '' + mkdir -p root + while IFS= read -r path; do + cp -r --no-preserve=ownership,xattr,context "$path" root/ + done < ${guestClosure}/store-paths + truncate -s 4096M "$out" + # Reproducible ext4 image: SOURCE_DATE_EPOCH pins the superblock times and + # a fixed UUID seed pins the htree hash seed (e2fsprogs ignores an all-zero + # seed and randomizes it), so every build is byte-identical. With a random + # seed each build differed, and the nixos-install closure spec (recorded + # from an earlier build) could never match the freshly built image. + SOURCE_DATE_EPOCH=0 mkfs.ext4 -q -F \ + -U 123e4567-e89b-12d3-a456-426614174000 \ + -E hash_seed=123e4567-e89b-12d3-a456-426614174000 \ + -d root "$out" + ''; + artifacts = { + runtime-cloud-hypervisor = { + inherit (cloudHypervisorArtifact) package type catalog; + }; + volume-acceptance-provider = { + inherit (volumeProviderArtifact) package type catalog; + }; + acceptance-system = { + package = guestSystem.config.system.build.toplevel; + type = "nixos-system"; + }; + }; + in + { + d2b.site.adminUsers = [ "alice" ]; + environment.systemPackages = with pkgs; [ + acl + iproute2 + jq + iputils + procps + util-linux + ]; + d2b.artifacts = artifacts; + # The declaration under test, installed verbatim from the repo so the + # fixture reads the same file the posture code embeds. + environment.etc."d2b/state-posture-contract.json".source = + ../../packages/d2b-broker/src/ops/state-posture-contract.json; + d2b.guestSystems.work.acceptance-guest = guestSystem; + d2b.zones.local-root.trustedPublishers.d2b-cloud-hypervisor.signingKey = + cloudHypervisorArtifact.trustedPublisher.signingKey; + d2b.zones.local-root.trustedPublishers.d2b-volume-acceptance.signingKey = + volumeProviderArtifact.trustedPublisher.signingKey; + d2b.zones.work.trustedPublishers.d2b-cloud-hypervisor.signingKey = + cloudHypervisorArtifact.trustedPublisher.signingKey; + d2b.zones.work.trustedPublishers.d2b-volume-acceptance.signingKey = + volumeProviderArtifact.trustedPublisher.signingKey; + d2b.zones.local-root.resources.host-system = { + type = "Host"; + spec = { + providerRef = "Provider/system-core"; + defaultDomain = "system"; + allowedDomains = [ "system" ]; + budget = { }; + networkAttachments = [ ]; + deviceAttachments = [ ]; + volumeAttachmentDefaults = [ ]; + }; + }; + d2b.zones.work = { + parentZone = "local-root"; + resources = { + alice = { + type = "User"; + spec = { + displayName = "Alice"; + groups = [ ]; + osUsername = "alice"; + }; + }; + d2bd = { + type = "User"; + spec = { + displayName = "d2bd"; + groups = [ ]; + osUsername = "d2bd"; + }; + }; + lifecycle-operator = { + type = "Role"; + spec.rules = [ + { + resourceTypes = [ "Endpoint" "Guest" "Host" "Process" "Provider" "Volume" "VolumeBinding" ]; + verbs = [ "get" "list" ]; + subresources = [ ]; + resourceNames = [ ]; + zones = [ "work" ]; + executionRefs = [ ]; + sessionVerbs = [ "connect" "invoke" ]; + } + { + resourceTypes = [ "Guest" ]; + verbs = [ "delete" ]; + subresources = [ ]; + resourceNames = [ "acceptance-guest" ]; + zones = [ "work" ]; + executionRefs = [ ]; + sessionVerbs = [ "connect" "invoke" ]; + } + { + resourceTypes = [ "Volume" ]; + verbs = [ "delete" ]; + subresources = [ ]; + resourceNames = [ "state" ]; + zones = [ "work" ]; + executionRefs = [ ]; + sessionVerbs = [ "connect" "invoke" ]; + } + ]; + }; + lifecycle-operator-binding = { + type = "RoleBinding"; + spec = { + roleRef = "Role/lifecycle-operator"; + subjects = [ "User/alice" ]; + externalPrincipalSelector = null; + scopeNarrowing = null; + }; + }; + host-system = { + type = "Host"; + spec = { + providerRef = "Provider/system-core"; + defaultDomain = "system"; + allowedDomains = [ "system" ]; + budget = { }; + networkAttachments = [ ]; + deviceAttachments = [ ]; + volumeAttachmentDefaults = [ ]; + }; + }; + volume-local = { + type = "Provider"; + spec = { + artifactId = "volume-acceptance-provider"; + config = { + controllerExecutionRef = "Host/host-system"; + sourcePolicies = [ + { + id = "default-state"; + class = "local-path"; + volumeKinds = [ "durable" "state" "cache" ]; + } + # U7: daemon-owned root the unprivileged daemon can + # lock and provision inline (path:daemon-state). + { + id = "daemon-state"; + class = "local-path"; + volumeKinds = [ "durable" "state" "cache" ]; + } + ]; + }; + }; + }; + volume-virtiofs = { + type = "Provider"; + spec = { + artifactId = "volume-acceptance-provider"; + config.controllerExecutionRef = "Host/host-system"; + }; + }; + state = { + type = "Volume"; + spec = { + providerRef = "Provider/volume-local"; + kind = "state"; + source = { + executionRef = "Host/host-system"; + settings = { + kind = "local-path"; + sourcePolicyId = "daemon-state"; + }; + }; + layout = [{ + path = "state"; + type = "directory"; + # U7: daemon-owned so the unprivileged daemon can + # provision inline; the guest share stays read-only. + ownerRef = "User/d2bd"; + groupRef = "User/d2bd"; + mode = "0700"; + target = null; + accessAcl = [ ]; + defaultAcl = [ ]; + foreignChildPolicy = "preserve"; + noFollow = true; + recursive = false; + sensitivity = "private"; + createPolicy = "create-if-never-provisioned"; + repairPolicy = "exact-owner"; + cleanupPolicy = "owner-controlled"; + adoptionPolicy = "quarantine-on-ambiguity"; + restartPolicy = "preserve-across-controller-restart"; + leaseClass = "none"; + invariants = [ "no-symlink" ]; + }]; + views.controller = { + path = ""; + rights = [ "read" "write" "traverse" ]; + }; + # KTD1: the attachment stays declared input only. The Volume + # side mints the durable VolumeBinding at reconcile; the + # deterministic binding identity below is + # vol-binding-6a8ea4307a30f7ceae6533f2 (volume, execution + # target, view, mount path). + attachments = [{ + executionRef = "Guest/acceptance-guest"; + transport = "virtiofs"; + view = "controller"; + access = "read-only"; + mountPath = "/state"; + settings = { + posixAcl = false; + xattr = false; + cache = "auto"; + inodeFileHandles = "never"; + threadPoolSize = null; + socketGroup = null; + }; + }]; + }; + }; + runtime-cloud-hypervisor = { + type = "Provider"; + spec = { + artifactId = "runtime-cloud-hypervisor"; + config = cloudHypervisorConfig; + }; + }; + acceptance-guest = { + type = "Guest"; + spec = { + providerRef = "Provider/runtime-cloud-hypervisor"; + executionRef = "Host/host-system"; + systemArtifactId = "acceptance-system"; + defaultDomain = "system"; + allowedDomains = [ "system" ]; + budget = { }; + volumeAttachmentDefaults = [ ]; + networkAttachments = [ ]; + deviceAttachments = [ ]; + }; + }; + }; + }; + }; + }; + # The guest each fixture-less image evaluates, by the name the image action # asks for. A check's own guest is read out of the check's fixture; these are # the guests with no fixture to be read out of - the two reusable shapes the @@ -629,6 +1045,10 @@ rec { node = d2bResourceOperatorActivationNode; testName = "d2b-resource-operator-activation"; }; + state-posture-contract = { + node = d2bStatePostureContractNode; + testName = "d2b-state-posture-contract"; + }; wayland-proxy = { node = d2bWaylandProxyNode; testName = "d2b-wayland-proxy"; diff --git a/packages/d2b-test-vm-harness/src/checks/mod.rs b/packages/d2b-test-vm-harness/src/checks/mod.rs index db1d91815..846598bac 100644 --- a/packages/d2b-test-vm-harness/src/checks/mod.rs +++ b/packages/d2b-test-vm-harness/src/checks/mod.rs @@ -28,6 +28,7 @@ pub mod daemon_smoke; pub mod guest_agent_cap_confinement; pub mod guest_shell_service; pub mod privilege_oracle; +pub mod state_posture_contract; pub mod resource_operator_activation; pub mod wayland_proxy; @@ -57,6 +58,10 @@ const PORTED: &[(&str, Assertions)] = &[ "resource-operator-activation", resource_operator_activation::assertions, ), + ( + "state-posture-contract", + state_posture_contract::assertions, + ), ("wayland-proxy", wayland_proxy::assertions), ]; diff --git a/packages/d2b-test-vm-harness/src/checks/state_posture_contract.rs b/packages/d2b-test-vm-harness/src/checks/state_posture_contract.rs new file mode 100644 index 000000000..4f86a3d7b --- /dev/null +++ b/packages/d2b-test-vm-harness/src/checks/state_posture_contract.rs @@ -0,0 +1,716 @@ +//! The declared host posture contract for state farms and shared +//! directories, ported from its fixture (issue #512). +//! +//! One file declares the posture +//! (`packages/d2b-broker/src/ops/state-posture-contract.json`); the broker +//! posture code embeds it, the Nix provisioning derives from it, and this +//! check asserts the live host against it: every declared level's +//! owner/group/mode/ACL, plus the allowed and denied operations for each +//! principal (root, d2bd, a d2b-group launcher, and nobody). The fixture +//! never restated a posture value - it read the declaration - so neither does +//! this module: the contract is parsed at run time and the same tree and +//! level walk is performed over it. +//! +//! What the fixture expressed as its own Python helpers comes with it, as +//! private functions of this module: `substitute` (the `` / +//! `` / `` / `` tokens), `tree`, `level_path`, `stat_row`, +//! `acl_entries` (with the fixture's observation cache), `named_entry`, +//! `permission_bits`, `level_exists`, `spawn_preflight_entries` (the +//! structural carve-out for the broker's spawn-time traversal grants, which +//! cannot be declared because the runner uids are minted per guest at run +//! time), `run_as`, `probe` and `check_level`. Every message is the +//! fixture's own, because a drift report that reads differently from the +//! fixture's report is a differently reported drift. +//! +//! The guest boots the same Zone-native Cloud Hypervisor Guest recipe the +//! acceptance fixture uses - the writable-store shape plus the acceptance +//! artifacts, zones and guest system - so the state chain, the store-view +//! farm, and the spawn-time traversal ACLs are all materialized by the +//! product path. That guest, and the `let` bindings that build it, are +//! declared in `nix/test-support/host-integration-node.nix`. +//! +//! `start_all()` is not restated here: it is the lane's own boot of the +//! guest the check runs against. + +use std::{collections::{BTreeMap, BTreeSet}, time::Duration}; + +use serde_json::Value; + +use crate::legacy::{shlex_quote, DiagRow, GuestControl, LegacyError, LegacyResult}; + +/// The zone the check drives. +const ZONE: &str = "work"; + +/// The guest inside that zone whose state chain is checked. +const GUEST: &str = "acceptance-guest"; + +/// The state root every declared path hangs off. +const STATE_ROOT: &str = "/var/lib/d2b"; + +/// The bound the daemon's activation gets, the fixture's own. +const DAEMON_UP: Duration = Duration::from_secs(180); + +/// The bound the broker socket gets, the fixture's own. +const BROKER_SOCKET: Duration = Duration::from_secs(30); + +/// The bound the public socket's file wait gets, the fixture's own. +const PUBLIC_SOCKET: Duration = Duration::from_secs(30); + +/// The bound the broker service's activation gets, the fixture's own. +const BROKER_SERVICE: Duration = Duration::from_secs(30); + +/// The bound each of the two store-view waits gets, the fixture's own. +const STORE_WAIT: Duration = Duration::from_secs(300); + +/// The principals whose declared rights are probed, and the linux user each +/// one is. +const PRINCIPAL_USER: [(&str, &str); 4] = [ + ("root", "root"), + ("d2bd", "d2bd"), + ("d2b", "alice"), + ("nobody", "nobody"), +]; + +/// The contract trees the check compares the live host with. +const CHECKED_TREES: [&str; 5] = [ + "state-root", + "guest-state-chain", + "guest-state-dir", + "guest-store-view", + "shared-run-dir", +]; + +/// One `stat -c` row of a declared level. +struct Stat { + owner: String, + group: String, + mode: String, +} + +/// The check's assertions, in the order its fixture made them. +pub fn assertions(control: &mut GuestControl) -> LegacyResult<()> { + control.stage("daemon-up"); + + // The declared contract, from the same file the broker posture code + // embeds and `nixos-modules/host-daemon.nix` derives its provisioning + // from. + let contract_text = control.succeed(&["cat /etc/d2b/state-posture-contract.json"], None)?; + let contract: Value = serde_json::from_str(&contract_text).map_err(|error| { + LegacyError::Assertion(format!( + "the state posture contract is not readable JSON: {error}" + )) + })?; + + control.diag_unit("daemon-up", "d2bd.service", DAEMON_UP)?; + control.wait_for_unit("d2b-broker.socket", None, BROKER_SOCKET)?; + control.wait_for_file("/run/d2b/public.sock", PUBLIC_SOCKET)?; + control.succeed(&["systemctl start d2b-broker.service"], None)?; + control.diag_unit("broker-service", "d2b-broker.service", BROKER_SERVICE)?; + + // The ids each principal's probes run as. + let mut principal_ids = BTreeMap::new(); + for (principal, user) in PRINCIPAL_USER { + let uid = control + .succeed(&[&format!("id -u {user}")], None)? + .trim() + .to_owned(); + let gid = control + .succeed(&[&format!("id -g {user}")], None)? + .trim() + .to_owned(); + principal_ids.insert(principal.to_owned(), (uid, gid)); + } + + let guest_state = format!("{STATE_ROOT}/zones/{ZONE}/guests/{GUEST}"); + let store_view = format!("{guest_state}/store-view"); + + // The store-view farm syncs behind the guest start, and the vmm binds the + // guest's own socket. + control.stage("store-view-sync"); + let guest_state_row = ( + "guest state tree".to_owned(), + format!( + "find {guest_state} -maxdepth 1 -exec stat -c '%A %U %G %n' {{}} + 2>/dev/null | sort || true" + ), + ); + let store_view_row = ( + "store-view tree".to_owned(), + format!( + "find {store_view} -maxdepth 2 -exec stat -c '%A %U %G %n' {{}} + 2>/dev/null | sort || true" + ), + ); + let store_view_command = format!( + "test -L {store_view}/state/current && test -L {store_view}/meta/current" + ); + control.diag_wait( + "store-view-sync", + &store_view_command, + STORE_WAIT, + &[as_row(&guest_state_row), as_row(&store_view_row)], + &[ + ("d2bd.service", "store"), + ("d2b-broker.service", "StoreSync"), + ], + )?; + + control.stage("vmm-spawn"); + let vmm_socket_command = format!("test -S {guest_state}/{GUEST}.sock"); + control.diag_wait( + "vmm-spawn", + &vmm_socket_command, + STORE_WAIT, + &[as_row(&guest_state_row)], + &[ + ("d2bd.service", "cloud-hypervisor"), + ("d2bd.service", "component-session"), + ], + )?; + + // Every checked level, observed before any of it is compared: the + // spawn-preflight carve-out relates a worker's ancestor traversal entries + // to the leaf grant below them, so the expected set cannot be decided one + // level at a time. + control.stage("posture-contract"); + let mut observed_acls: BTreeMap> = BTreeMap::new(); + for tree_id in CHECKED_TREES { + let entry = tree(&contract, tree_id)?; + for level in levels(entry)? { + let path = level_path(entry, level)?; + if level_exists(control, &path)? { + observe_acl(control, &mut observed_acls, &path)?; + } + } + } + let spawn_entries = spawn_preflight_entries(control, &observed_acls)?; + for tree_id in CHECKED_TREES { + let entry = tree(&contract, tree_id)?; + for level in levels(entry)? { + check_level( + control, + entry, + level, + &mut observed_acls, + &spawn_entries, + &principal_ids, + )?; + } + } + + // The guest start above ran the daemon's anchored store-view walk while + // the chain was capped at search-only by the spawn-time `u:d2bd:--x` + // ACL. Two live proofs: the store-view open never failed, and the resolved + // view reached a virtiofsd worker through the daemon's fd handoff. + control.stage("anchor-open-rule"); + control.fail( + &["journalctl -u d2bd.service --no-pager -b -n 5000 | grep -F 'store-view-open'"], + None, + )?; + if control.execute("pgrep -x virtiofsd >/dev/null", None)?.status != 0 { + return Err(LegacyError::Assertion( + "the store-view directory must reach a virtiofsd worker".to_owned(), + )); + } + // Live denial, from the same contract rows: the daemon may search the + // per-Guest state dir it does not own but may not read it. + let daemon_read = run_as( + control, + &principal_ids, + "d2bd", + &format!("ls {} >/dev/null 2>&1", shlex_quote(&guest_state)), + )?; + if daemon_read { + return Err(LegacyError::Assertion( + "the daemon must not read the per-Guest state dir it only traverses".to_owned(), + )); + } + + control.stage("done"); + control.announce("[d2b] declared state posture contract holds on the live host"); + Ok(()) +} + +/// One declared tree, which must exist exactly once. +fn tree<'a>(contract: &'a Value, tree_id: &str) -> LegacyResult<&'a Value> { + let matches = contract + .get("trees") + .and_then(Value::as_array) + .map(|trees| { + trees + .iter() + .filter(|entry| entry.get("id").and_then(Value::as_str) == Some(tree_id)) + .collect::>() + }) + .unwrap_or_default(); + if matches.len() != 1 { + return Err(LegacyError::Assertion(format!( + "contract tree {tree_id} must exist exactly once" + ))); + } + Ok(matches[0]) +} + +/// One tree's declared levels. +fn levels(entry: &Value) -> LegacyResult> { + entry + .get("levels") + .and_then(Value::as_array) + .map(|levels| levels.iter().collect()) + .ok_or_else(|| LegacyError::Assertion("a declared tree carries no levels".to_owned())) +} + +/// One field of a declaration, as the fixture read it: a missing field is a +/// check failure rather than a lane failure, in the fixture's own words. +fn field<'a>(value: &'a Value, key: &str, where_: &str) -> LegacyResult<&'a Value> { + value + .get(key) + .ok_or_else(|| LegacyError::Assertion(format!("{where_} declares no {key}"))) +} + +/// The `` substitutions the declaration uses. +fn substitute(value: &str) -> String { + let tokens = [ + ("state-root", STATE_ROOT), + ("zone", ZONE), + ("guest", GUEST), + ("vm", GUEST), + ]; + let mut substituted = value.to_owned(); + for (token, replacement) in tokens { + substituted = substituted.replace(&format!("<{token}>"), replacement); + } + substituted +} + +/// The live path one declared level names. +fn level_path(entry: &Value, level: &Value) -> LegacyResult { + let where_ = format!( + "{}:{}", + entry.get("id").and_then(Value::as_str).unwrap_or("?"), + level.get("path").and_then(Value::as_str).unwrap_or("?") + ); + let root = substitute( + field(entry, "root", &where_)? + .as_str() + .ok_or_else(|| LegacyError::Assertion(format!("{where_} declares no root")))?, + ); + let path = field(level, "path", &where_)? + .as_str() + .ok_or_else(|| LegacyError::Assertion(format!("{where_} declares no path")))?; + if path == "." { + return Ok(root); + } + Ok(format!("{root}/{}", substitute(path))) +} + +/// A label a level is reported under, the fixture's own `where` text. +fn where_(entry: &Value, level: &Value, path: &str) -> String { + format!( + "{}:{} ({path})", + entry.get("id").and_then(Value::as_str).unwrap_or("?"), + level.get("path").and_then(Value::as_str).unwrap_or("?"), + ) +} + +/// The owner, group and mode of a live path. +fn stat_row(control: &mut GuestControl, path: &str) -> LegacyResult { + let output = control.succeed(&[&format!("stat -c '%U %G %a' {}", shlex_quote(path))], None)?; + let mut fields = output.split_whitespace(); + match (fields.next(), fields.next(), fields.next(), fields.next()) { + (Some(owner), Some(group), Some(mode), None) => Ok(Stat { + owner: owner.to_owned(), + group: group.to_owned(), + mode: mode.to_owned(), + }), + _ => Err(LegacyError::Assertion(format!( + "stat reported no owner, group and mode for {path}: {output:?}" + ))), + } +} + +/// Whether a path exists, without refusing when it does not. +fn level_exists(control: &mut GuestControl, path: &str) -> LegacyResult { + Ok(control + .execute(&format!("test -e {}", shlex_quote(path)), None)? + .status + == 0) +} + +/// Observe one path's ACL entries, once, and keep them. +/// +/// The cache is the fixture's own: the spawn-preflight carve-out is +/// structural, so the whole observed set is read before any of it is judged. +fn observe_acl( + control: &mut GuestControl, + observed: &mut BTreeMap>, + path: &str, +) -> LegacyResult<()> { + if observed.contains_key(path) { + return Ok(()); + } + let output = control.succeed( + &[&format!("getfacl -cp {} 2>/dev/null || true", shlex_quote(path))], + None, + )?; + let mut entries = BTreeSet::new(); + for line in output.lines() { + let parts = line.trim().split(':').collect::>(); + if parts.len() != 3 { + continue; + } + let (kind, name, permissions) = (parts[0], parts[1], parts[2]); + let short = match kind { + "user" => Some("u"), + "group" => Some("g"), + "other" => Some("o"), + "mask" => Some("m"), + _ => None, + }; + if let Some(short) = short { + entries.insert(format!("{short}:{name}:{permissions}")); + } + } + observed.insert(path.to_owned(), entries); + Ok(()) +} + +/// One ACL entry's permissions, as the set of bits it grants. +fn permission_bits(permissions: &str) -> BTreeSet { + "rwx".chars().filter(|bit| permissions.contains(*bit)).collect() +} + +/// Whether an ACL entry names a principal rather than a class. +fn named_entry(spec: &str) -> bool { + let parts = spec.splitn(3, ':').collect::>(); + parts.len() == 3 && (parts[0] == "u" || parts[0] == "g") && !parts[1].is_empty() +} + +/// Named entries the spawn preflight is expected to add, per level. +/// +/// The broker's spawn preflight opens the ancestor chain above a +/// runner-owned tree with search (`u::--x`) and grants the runner its +/// own leaf (`rwx` for a private state tree, `r-x` for a read-only served +/// view root): `runner_tree_acl_targets` in +/// `packages/d2b-broker/src/live_handlers.rs`, reached from +/// `refresh_spawn_runner_acls` / `grant_serving_worker_launch_acls` / +/// `grant_device_worker_launch_acls`. The runner principals are uids minted +/// per Guest at runtime, so the declaration cannot name them; the check +/// derives them from the live worker processes and then allows an undeclared +/// entry only in that structural shape - `--x` on a level that has a deeper +/// grant of the same uid, or the uid's own topmost grant (`--x` when its leaf +/// is outside the checked levels, else the leaf spelling). A foreign uid, or +/// a wider grant on a level above the worker's own leaf, still fails. +fn spawn_preflight_entries( + control: &mut GuestControl, + observed: &BTreeMap>, +) -> LegacyResult>> { + let processes = control.succeed(&["ps -eo uid=,comm= --no-headers"], None)?; + let mut worker_uids = BTreeSet::new(); + for line in processes.lines() { + let trimmed = line.trim(); + let (uid, comm) = trimmed.split_once(' ').unwrap_or((trimmed, "")); + let comm = comm.trim(); + if comm.starts_with("cloud-hyperviso") || comm.starts_with("virtiofsd") { + worker_uids.insert(uid.to_owned()); + } + } + let mut allowed: BTreeMap> = BTreeMap::new(); + for uid in worker_uids { + let prefix = format!("u:{uid}:"); + let grant_paths = observed + .iter() + .filter(|(_, entries)| entries.iter().any(|entry| entry.starts_with(&prefix))) + .map(|(path, _)| path.clone()) + .collect::>(); + for path in &grant_paths { + let prefix_path = format!("{}/", path.trim_end_matches('/')); + let deeper = grant_paths + .iter() + .any(|other| other != path && other.starts_with(&prefix_path)); + let mut spellings = BTreeSet::new(); + spellings.insert(format!("{prefix}--x")); + if !deeper { + spellings.insert(format!("{prefix}rwx")); + spellings.insert(format!("{prefix}r-x")); + } + allowed.entry(path.clone()).or_default().extend(spellings); + } + } + Ok(allowed) +} + +/// Run one command as one principal, and whether it succeeded. +fn run_as( + control: &mut GuestControl, + principal_ids: &BTreeMap, + principal: &str, + command: &str, +) -> LegacyResult { + let (uid, gid) = principal_ids.get(principal).ok_or_else(|| { + LegacyError::Assertion(format!("the check has no ids for principal {principal}")) + })?; + let status = control + .execute( + &format!( + "setpriv --reuid={uid} --regid={gid} --init-groups /bin/sh -c {}", + shlex_quote(command) + ), + None, + )? + .status; + Ok(status == 0) +} + +/// Whether one principal has one right on one path. +fn probe( + control: &mut GuestControl, + principal_ids: &BTreeMap, + principal: &str, + right: &str, + path: &str, +) -> LegacyResult { + let flag = match right { + "traverse" => "x", + "read" => "r", + "write" => "w", + _ => { + return Err(LegacyError::Assertion(format!( + "unknown right {right} in the declaration" + ))); + } + }; + run_as( + control, + principal_ids, + principal, + &format!("test -{flag} {}", shlex_quote(path)), + ) +} + +/// One declared string field of a level, as the fixture read it. +fn declared_str<'a>(level: &'a Value, key: &str, where_: &str) -> LegacyResult<&'a str> { + field(level, key, where_)? + .as_str() + .ok_or_else(|| LegacyError::Assertion(format!("{where_} declares no {key}"))) +} + +/// Compare one declared level with the live host, in the fixture's own order. +fn check_level( + control: &mut GuestControl, + entry: &Value, + level: &Value, + observed: &mut BTreeMap>, + spawn_entries: &BTreeMap>, + principal_ids: &BTreeMap, +) -> LegacyResult<()> { + let path = level_path(entry, level)?; + let where_ = where_(entry, level, &path); + if !level_exists(control, &path)? { + if level.get("required").and_then(Value::as_bool) == Some(false) { + return Ok(()); + } + return Err(LegacyError::Assertion(format!( + "declared level is missing: {where_}" + ))); + } + + let stat = stat_row(control, &path)?; + let mode = u32::from_str_radix(&stat.mode, 8).map_err(|error| { + LegacyError::Assertion(format!("the mode at {where_} is not octal: {error}")) + })?; + let policy = level + .get("modePolicy") + .and_then(Value::as_str) + .unwrap_or("exact"); + match policy { + "exact" => { + let declared_mode = field(level, "mode", &where_)? + .as_str() + .ok_or_else(|| LegacyError::Assertion(format!("{where_} declares no mode")))?; + let declared = u32::from_str_radix(declared_mode, 8).map_err(|error| { + LegacyError::Assertion(format!("the declared mode at {where_} is not octal: {error}")) + })?; + if mode != declared & 0o7777 { + return Err(LegacyError::Assertion(format!( + "mode drift at {where_}: declared {declared_mode}, observed {}", + stat.mode + ))); + } + if stat.owner != declared_str(level, "owner", &where_)? { + return Err(LegacyError::Assertion(format!( + "owner drift at {where_}: declared {}, observed {}", + declared_str(level, "owner", &where_)?, + stat.owner + ))); + } + if stat.group != declared_str(level, "group", &where_)? { + return Err(LegacyError::Assertion(format!( + "group drift at {where_}: declared {}, observed {}", + declared_str(level, "group", &where_)?, + stat.group + ))); + } + } + "group-traverse-minimum" => { + if stat.owner != declared_str(level, "owner", &where_)? { + return Err(LegacyError::Assertion(format!( + "owner drift at {where_}: declared {}, observed {}", + declared_str(level, "owner", &where_)?, + stat.owner + ))); + } + if stat.group != declared_str(level, "group", &where_)? { + return Err(LegacyError::Assertion(format!( + "group drift at {where_}: declared {}, observed {}", + declared_str(level, "group", &where_)?, + stat.group + ))); + } + if mode & 0o010 == 0 { + return Err(LegacyError::Assertion(format!( + "group search missing at {where_}" + ))); + } + if mode & 0o020 != 0 { + return Err(LegacyError::Assertion(format!( + "group write must never be granted at {where_}" + ))); + } + } + "preserve-existing" => {} + other => { + return Err(LegacyError::Assertion(format!( + "unknown modePolicy {other} at {where_}" + ))); + } + } + + observe_acl(control, observed, &path)?; + let entries = observed + .get(&path) + .cloned() + .ok_or_else(|| LegacyError::Assertion(format!("no ACL entries were read at {where_}")))?; + let declared_acl = level + .get("acl") + .and_then(Value::as_array) + .map(|acl| { + acl.iter() + .filter_map(|declared| { + declared + .get("spec") + .and_then(Value::as_str) + .map(str::to_owned) + }) + .collect::>() + }) + .unwrap_or_default(); + for spec in &declared_acl { + if !entries.contains(spec) { + return Err(LegacyError::Assertion(format!( + "declared ACL missing at {where_}: {spec}" + ))); + } + } + + // A declaration pins its named entries completely: an entry nobody + // declared is drift, not a harmless extra. The only undeclared entries + // the live host may carry are the spawn-preflight traversal grants derived + // for the runner uids; the base entries (u::/g::/o::/m::) are pinned only + // when the declaration names them. + let declared_named = declared_acl + .iter() + .filter(|spec| named_entry(spec)) + .cloned() + .collect::>(); + let observed_named = entries + .iter() + .filter(|entry| named_entry(entry)) + .cloned() + .collect::>(); + let mut expected_named = declared_named.clone(); + if let Some(spawned) = spawn_entries.get(&path) { + expected_named.extend(spawned.iter().cloned()); + } + let undeclared = observed_named + .difference(&expected_named) + .cloned() + .collect::>(); + if !undeclared.is_empty() { + return Err(LegacyError::Assertion(format!( + "undeclared ACL entry at {where_}: {}", + undeclared.join(", ") + ))); + } + let missing = declared_named + .difference(&observed_named) + .cloned() + .collect::>(); + if !missing.is_empty() { + return Err(LegacyError::Assertion(format!( + "declared ACL entry missing at {where_}: {}", + missing.join(", ") + ))); + } + + // An unpinned mask must be the union of the group entry and the named + // grants (setfacl semantics): a mask that drifts from that silently + // re-scopes the whole group class. + let declared_mask = declared_acl + .iter() + .any(|spec| spec.splitn(3, ':').next() == Some("m")); + let mask_entry = entries.iter().find(|entry| entry.starts_with("m::")); + if !declared_mask { + if let Some(mask_entry) = mask_entry { + let group_entry = entries.iter().find(|entry| entry.starts_with("g::")); + let mut expected_mask = group_entry + .map(|entry| permission_bits(entry.splitn(3, ':').nth(2).unwrap_or(""))) + .unwrap_or_default(); + for spec in &observed_named { + expected_mask.extend(permission_bits(spec.splitn(3, ':').nth(2).unwrap_or(""))); + } + let observed_mask = permission_bits(mask_entry.splitn(3, ':').nth(2).unwrap_or("")); + if observed_mask != expected_mask { + let rendered = ["r", "w", "x"] + .into_iter() + .filter(|bit| expected_mask.contains(&bit.chars().next().unwrap_or(' '))) + .collect::(); + return Err(LegacyError::Assertion(format!( + "ACL mask drift at {where_}: expected {rendered} from the group entry and \ + named grants, observed {mask_entry}" + ))); + } + } + } + + // The declared rights: an expectation of `preserve` or `not-required` is + // not probed, and root's allow rows are documentation because root + // bypasses DAC. + let rights = level + .get("rights") + .and_then(Value::as_object) + .ok_or_else(|| LegacyError::Assertion(format!("{where_} declares no rights")))?; + for (principal, rights) in rights { + for right in ["traverse", "read", "write"] { + let expectation = rights.get(right).and_then(Value::as_str).unwrap_or("preserve"); + if expectation == "preserve" || expectation == "not-required" { + continue; + } + if principal == "root" { + continue; + } + let observed_right = probe(control, principal_ids, principal, right, &path)?; + if observed_right != (expectation == "allow") { + return Err(LegacyError::Assertion(format!( + "{where_}: {principal} {right} expected {expectation}, observed {}", + if observed_right { "allow" } else { "deny" } + ))); + } + } + } + + Ok(()) +} + +/// One owned row, as the diagnostics rows are passed. +fn as_row<'a>(pair: &'a (String, String)) -> DiagRow<'a> { + (pair.0.as_str(), pair.1.as_str()) +} From 85627a4712dd0d23d38fe06782bfe734bed6d9ec Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sat, 26 Sep 2026 22:43:09 -0700 Subject: [PATCH 21/51] refactor(vm): assert device-worker-launch in Rust and retire its fixture The Device-worker launch path (U17 slice 3) moves into packages/d2b-test-vm-harness/src/checks/device_worker_launch.rs in the fixture's own order: 9 stage, 1 wait_for_unit (d2b-broker.socket, 30s), 1 wait_for_file (public.sock, 30s), 1 diag_unit (d2bd.service, 180s), 6 diag_wait (rows-ingested 180, tpm-worker-ready 180, tpm-worker-process 60, tpm-sockets 60, tpm-flush-outcome 180, tpm-teardown 180), 19 succeed, 3 execute (the fixture's own `d2b` and process reads) and twenty assertion sites - the fixture's own `check(...)` calls, which run fifty-four times over the declared-row loops. No fail call exists in the fixture and none exists here, and the fixture's fourteen printed lines are reported through the surface's own `announce`, so a run reads the way it read. The Python the fixture built - the `d2b`/`list_json`/`flush_get` command builders, the `DECLARED_ROWS` and `BINDING_OWNER` tables and the row dumps its diagnostics print - are private ports in the module. The guest it boots - the daemon shape plus the swtpm and GPU device-worker artifacts, the crosvm stand-in, the Cloud Hypervisor configuration and the declared zones, Devices and provider rows - is declared in nix/test-support/host-integration-node.nix as d2bDeviceWorkerLaunchNode. This commit also repairs the lane's own check inventory: `_CHECKS` in bazel/checks/vm/BUILD.bazel carried `state-posture-contract` twice, which would have declared one `guest_image_state-posture-contract` target twice. The duplicate is dropped, so the inventory is the eleven checks again. Counts the port must satisfy: 9 stage, 1 wait_for_unit, 1 wait_for_file, 1 diag_unit, 6 diag_wait, 19 succeed, 3 execute, 20 assertion sites (54 invocations), 14 announce lines, before and after. --- bazel/checks/vm/BUILD.bazel | 2 +- nix/test-support/host-integration-node.nix | 498 ++++++- .../src/checks/device_worker_launch.rs | 1157 +++++++++++++++++ .../d2b-test-vm-harness/src/checks/mod.rs | 5 + .../host-integration/device-worker-launch.nix | 1065 --------------- 5 files changed, 1657 insertions(+), 1070 deletions(-) create mode 100644 packages/d2b-test-vm-harness/src/checks/device_worker_launch.rs delete mode 100644 tests/host-integration/device-worker-launch.nix diff --git a/bazel/checks/vm/BUILD.bazel b/bazel/checks/vm/BUILD.bazel index e97e99406..397c7a5a0 100644 --- a/bazel/checks/vm/BUILD.bazel +++ b/bazel/checks/vm/BUILD.bazel @@ -133,7 +133,6 @@ _CHECKS = [ "resource-operator-activation", "state-posture-contract", "runtime-cloud-hypervisor-guest-preflight", - "state-posture-contract", "virtiofsd-volume-runtime", "wayland-proxy", ] @@ -149,6 +148,7 @@ _CHECKS = [ _PORTED_CHECKS = [ "bridge-isolation", "daemon-smoke", + "device-worker-launch", "guest-agent-cap-confinement", "guest-shell-service", "privilege-oracle", diff --git a/nix/test-support/host-integration-node.nix b/nix/test-support/host-integration-node.nix index e5dd63558..ae5efc750 100644 --- a/nix/test-support/host-integration-node.nix +++ b/nix/test-support/host-integration-node.nix @@ -993,6 +993,495 @@ rec { }; }; + # The guest `device-worker-launch` boots: the reusable daemon node + # plus the fixture's own contributions. + # + # The fixture declared no machine size, disk or device of its own - the shape + # carries those - so what moves here is its `let` bindings (the swtpm and GPU + # device-worker artifacts, the crosvm stand-in shim, the Cloud Hypervisor + # configuration and the declared artifacts) and the extra module that declares + # the provider rows and the Devices whose worker rows the check launches. + d2bDeviceWorkerLaunchNode = + d2bDaemonNode { + extra = + { lib, pkgs, ... }: + let + hostToolBundle = + if self.lib ? d2bHostToolBundle then self.lib.d2bHostToolBundle else null; + d2bLib = import ../../tests/host-integration/lib.nix { + inherit self; + inherit lib; + inherit hostToolBundle; + }; + cloudHypervisorArtifact = d2bLib.mkRuntimeCloudHypervisorArtifact pkgs; + volumeProviderArtifact = d2bLib.mkVolumeProviderArtifact pkgs; + + # A Provider artifact that packages the Device worker executables the + # declared rows name. Shape mirrors `mkVolumeProviderArtifact`: a signed + # manifest whose executable set is computed from the packaged `bin/` files, + # a Device-exporting catalog entry, and a deterministic publisher key. + mkDeviceWorkerProviderArtifact = + { artifactId + , publisher + , binaries + , controllerBinary + }: + let + signer = pkgs.python3.withPackages + (pythonPackages: [ pythonPackages.cryptography ]); + manifest = ../../tests/fixtures/provider-acceptance/provider-manifest.json; + schema = ../../tests/fixtures/provider-acceptance/config-schema.json; + controller = if hostToolBundle == null then + "${self.packages.${pkgs.stdenv.hostPlatform.system}.d2b-provider-test-controller}/bin/d2b-provider-test-controller" + else + "${hostToolBundle}/bin/d2b-provider-test-controller"; + package = pkgs.runCommand "d2b-${artifactId}" { + nativeBuildInputs = [ pkgs.coreutils signer ]; + } '' + mkdir -p "$out/bin" + ${lib.concatStringsSep "\n" (lib.mapAttrsToList + (name: path: '' + cp "${path}" "$out/bin/${name}" + chmod 0755 "$out/bin/${name}" + '') + binaries)} + cp "${controller}" "$out/bin/${controllerBinary}" + chmod 0755 "$out/bin/${controllerBinary}" + ${signer}/bin/python3 - "${manifest}" "$out" \ + "${artifactId}" "${publisher}" "${controllerBinary}" \ + ${lib.escapeShellArg (lib.concatStringsSep " " (lib.attrNames binaries))} <<'PY' + import hashlib + import json + import pathlib + import sys + from cryptography.hazmat.primitives import serialization + from cryptography.hazmat.primitives.asymmetric.ed25519 import ( + Ed25519PrivateKey, + ) + + ( + manifest_path, + output_path, + artifact_id, + publisher, + controller_binary, + binary_names, + ) = sys.argv[1:] + output = pathlib.Path(output_path) + manifest = json.loads(pathlib.Path(manifest_path).read_text()) + # The executable set the compiler recomputes covers every regular file + # in bin/: the controller binary plus the declared worker binaries. + names = sorted(set(binary_names.split()) | {controller_binary}) + + # Device-only manifest: the declared Device worker rows are the only + # rows this artifact's Provider serves in this fixture. + resource_types = {"Device"} + manifest["apiBindings"] = [ + binding + for binding in manifest.get("apiBindings", []) + if binding.get("resourceType") in resource_types + ] + for component in manifest.get("components", []): + component["exportedResourceTypes"] = [ + resource_type + for resource_type in component.get("exportedResourceTypes", []) + if resource_type in resource_types + ] + + manifest["artifactId"] = artifact_id + manifest["trust"]["publisher"] = publisher + executable_map = json.dumps( + { + name: "sha256:" + hashlib.sha256( + (output / "bin" / name).read_bytes() + ).hexdigest() + for name in names + }, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode() + first = hashlib.sha256( + b"d2b:v3:provider-executable-set\0" + executable_map + ).digest() + executable_digest = "sha256:" + hashlib.sha256(first).hexdigest() + controller_digest = "sha256:" + hashlib.sha256( + (output / "bin" / controller_binary).read_bytes() + ).hexdigest() + manifest["digests"]["executable"] = executable_digest + for component in manifest.get("components", []): + for capability in component.get("targetCapabilities", []): + capability["artifactDigest"] = controller_digest + manifest_bytes = json.dumps( + manifest, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode() + seed = hashlib.sha256( + b"d2b-u17-device-worker-provider-signing-key-v1" + + artifact_id.encode() + ).digest() + private_key = Ed25519PrivateKey.from_private_bytes(seed) + public_key = private_key.public_key().public_bytes( + serialization.Encoding.PEM, + serialization.PublicFormat.SubjectPublicKeyInfo, + ) + metadata = output / "share/d2b/provider" + metadata.mkdir(parents=True) + (metadata / "provider-manifest.json").write_bytes(manifest_bytes) + (metadata / "provider-manifest.json.sig").write_bytes( + private_key.sign(manifest_bytes) + ) + (metadata / "config-schema.json").write_bytes( + pathlib.Path("${schema}").read_bytes() + ) + (output / "publisher-public-key.pem").write_bytes(public_key) + (output / "executable-set-digest").write_text(executable_digest) + (output / "manifest-digest").write_text( + "sha256:" + hashlib.sha256(manifest_bytes).hexdigest() + ) + PY + ''; + packageDigestPath = pkgs.runCommand + "d2b-${artifactId}-nar-digest" { + nativeBuildInputs = [ pkgs.nix ]; + } '' + printf 'sha256:%s' \ + "$(${pkgs.nix}/bin/nix --extra-experimental-features nix-command \ + hash path --type sha256 --base16 "${package}")" > "$out" + ''; + baseManifest = builtins.fromJSON (builtins.readFile manifest); + catalog = { + providerName = artifactId; + packageName = "d2b-${artifactId}"; + version = "0.0.0"; + systems = [ pkgs.stdenv.hostPlatform.system ]; + platform = pkgs.stdenv.hostPlatform.system; + apiCompatibility = "d2b.zone.v3"; + serviceCompatibility = "d2bd.resource"; + signature = { signatureId = "default"; }; + rootEpoch = 1; + revocationStatus = "clear"; + denyStatus = "clear"; + provenanceEvidence = "accepted"; + sbomEvidence = "accepted"; + licenseEvidence = "accepted"; + vulnerabilityEvidence = "accepted"; + conformanceAttestation = "accepted"; + supportChannel = "stable"; + supportContact = "d2b-u17-device-worker@localhost"; + publisher = publisher; + packageDigest = lib.removeSuffix "\n" + (builtins.readFile packageDigestPath); + executableDigest = lib.removeSuffix "\n" + (builtins.readFile "${package}/executable-set-digest"); + manifestDigest = lib.removeSuffix "\n" + (builtins.readFile "${package}/manifest-digest"); + componentDigest = "sha256:${builtins.hashString + "sha256" (builtins.toJSON baseManifest.components)}"; + descriptorDigest = "sha256:${builtins.hashString + "sha256" (builtins.toJSON baseManifest.apiBindings)}"; + configDigest = "sha256:${builtins.hashString + "sha256" (builtins.readFile schema)}"; + }; + in { + inherit package catalog; + type = "provider"; + trustedPublisher = { + publisherRef = publisher; + signingKey = builtins.readFile "${package}/publisher-public-key.pem"; + }; + }; + + # The GPU artifact's crosvm stand-in: a real ELF (via the shim) so the + # artifact validates as a Provider executable set, whose behavior is to + # record its argv and refuse. The fixture never asks a GPU worker to serve. + crosvmStandIn = self.lib.buildProviderElfShim { + inherit pkgs; + name = "crosvm"; + interpreterPkg = pkgs.bash; + interpreterPath = "bin/bash"; + program = pkgs.writeText "d2b-u17-crosvm-stand-in.sh" '' + # Fixture stand-in for the GPU Provider's crosvm. It must never run in + # a passing fixture (the launch path is proved up to the broker's own + # refusal); if it does run, it records the argv the Process controller + # composed and refuses, so a fabricated success is impossible. + set -eu + log="/run/d2b/u17-device-worker-standin.argv" + if [ -d /run/d2b ]; then + printf '%s\n' "crosvm-stand-in:$*" >> "$log" 2>/dev/null || true + fi + printf 'd2b-u17: crosvm stand-in invoked with %s\n' "$*" >&2 + exit 79 + ''; + }; + + # One artifact serves both Device Providers: a Provider artifact exports its + # ResourceTypes, and two artifacts both exporting `Device` collide in one + # Zone (`provider-resourcetype-collision`). + deviceWorkerArtifact = mkDeviceWorkerProviderArtifact { + artifactId = "device-worker-acceptance-provider"; + publisher = "d2b-u17-device-worker"; + controllerBinary = "acceptance-controller"; + binaries = { + swtpm = "${pkgs.swtpm}/bin/swtpm"; + swtpm-ioctl = "${pkgs.swtpm}/bin/swtpm_ioctl"; + crosvm = "${crosvmStandIn}/bin/crosvm"; + }; + }; + + cloudHypervisorConfig = { + controllerExecutionRef = "Host/host-system"; + defaultVcpus = 2; + defaultMemoryMb = 512; + defaultMachineType = "microvm"; + watchdog = true; + adoptionWindowMs = 30000; + healthCheckIntervalMs = 5000; + healthCheckTimeoutMs = 1000; + healthCheckFailureThreshold = 3; + startupDeadlineMs = 120000; + }; + + artifacts = { + runtime-cloud-hypervisor = { + inherit (cloudHypervisorArtifact) package type catalog; + }; + volume-acceptance-provider = { + inherit (volumeProviderArtifact) package type catalog; + }; + device-worker-acceptance-provider = { + inherit (deviceWorkerArtifact) package type catalog; + }; + }; + in + { + d2b.site.adminUsers = [ "alice" ]; + environment.systemPackages = with pkgs; [ + jq + procps + util-linux + acl + iproute2 + # The fixture binds its stale video socket from the VM side; python3 + # is the smallest reliable AF_UNIX binder available in the VM. + python3 + ]; + d2b.artifacts = artifacts; + d2b.zones.local-root.resources.host-system = { + type = "Host"; + spec = { + providerRef = "Provider/system-core"; + defaultDomain = "system"; + allowedDomains = [ "system" ]; + budget = { }; + networkAttachments = [ ]; + deviceAttachments = [ ]; + volumeAttachmentDefaults = [ ]; + }; + }; + d2b.zones.work.parentZone = "local-root"; + # Every Zone the host compiles a bundle for declares the publishers of + # the artifacts its rows select; `local-root` is a compiled Zone too + # (the other Cloud Hypervisor fixtures declare the same pair there). + d2b.zones.local-root.trustedPublishers.d2b-cloud-hypervisor.signingKey = + cloudHypervisorArtifact.trustedPublisher.signingKey; + d2b.zones.local-root.trustedPublishers.d2b-volume-acceptance.signingKey = + volumeProviderArtifact.trustedPublisher.signingKey; + d2b.zones.local-root.trustedPublishers.d2b-u17-device-worker.signingKey = + deviceWorkerArtifact.trustedPublisher.signingKey; + d2b.zones.work.trustedPublishers.d2b-cloud-hypervisor.signingKey = + cloudHypervisorArtifact.trustedPublisher.signingKey; + d2b.zones.work.trustedPublishers.d2b-volume-acceptance.signingKey = + volumeProviderArtifact.trustedPublisher.signingKey; + d2b.zones.work.trustedPublishers.d2b-u17-device-worker.signingKey = + deviceWorkerArtifact.trustedPublisher.signingKey; + d2b.zones.work.resources = { + alice = { + type = "User"; + spec = { + displayName = "Alice"; + groups = [ ]; + osUsername = "alice"; + }; + }; + d2bd = { + type = "User"; + spec = { + displayName = "d2bd"; + groups = [ ]; + osUsername = "d2bd"; + }; + }; + device-operator = { + type = "Role"; + spec.rules = [ + { + resourceTypes = [ + "Device" + "Endpoint" + "EphemeralProcess" + "Guest" + "Host" + "Process" + "Provider" + "Volume" + ]; + verbs = [ "get" "list" ]; + subresources = [ ]; + resourceNames = [ ]; + zones = [ "work" ]; + executionRefs = [ ]; + sessionVerbs = [ "connect" "invoke" ]; + } + { + resourceTypes = [ "Device" ]; + verbs = [ "delete" ]; + subresources = [ ]; + resourceNames = [ "tpm0" ]; + zones = [ "work" ]; + executionRefs = [ ]; + sessionVerbs = [ "connect" "invoke" ]; + } + ]; + }; + device-operator-binding = { + type = "RoleBinding"; + spec = { + roleRef = "Role/device-operator"; + subjects = [ "User/alice" ]; + externalPrincipalSelector = null; + scopeNarrowing = null; + }; + }; + host-system = { + type = "Host"; + spec = { + providerRef = "Provider/system-core"; + defaultDomain = "system"; + allowedDomains = [ "system" ]; + budget = { }; + networkAttachments = [ ]; + deviceAttachments = [ ]; + volumeAttachmentDefaults = [ ]; + }; + }; + # The Device owners. The Guest stays declared input and is never + # booted: the Device worker rows are bundle-declared `Process` rows of + # the Process controller, and no guest system artifact is declared, so + # no VMM is ever launched here. Its name is the VM identity of the + # Devices it owns (`Device.metadata.ownerRef`). + acceptance-guest = { + type = "Guest"; + spec = { + providerRef = "Provider/volume-virtiofs"; + executionRef = "Host/host-system"; + defaultDomain = "system"; + allowedDomains = [ "system" ]; + budget = { }; + volumeAttachmentDefaults = [ ]; + networkAttachments = [ ]; + deviceAttachments = [ ]; + }; + }; + volume-local = { + type = "Provider"; + spec = { + artifactId = "volume-acceptance-provider"; + config = { + controllerExecutionRef = "Host/host-system"; + sourcePolicies = [ + { + id = "daemon-state"; + class = "local-path"; + volumeKinds = [ "durable" "state" "cache" ]; + } + # The TPM state Volume's source policy + # (`build_tpm_state_volume_spec`, opaque policy id). + { + id = "tpm-state"; + class = "local-path"; + volumeKinds = [ "state" ]; + } + ]; + }; + }; + }; + volume-virtiofs = { + type = "Provider"; + spec = { + artifactId = "volume-acceptance-provider"; + config.controllerExecutionRef = "Host/host-system"; + }; + }; + runtime-cloud-hypervisor = { + type = "Provider"; + spec = { + artifactId = "runtime-cloud-hypervisor"; + config = cloudHypervisorConfig; + }; + }; + device-tpm = { + type = "Provider"; + spec = { + artifactId = "device-worker-acceptance-provider"; + config.controllerExecutionRef = "Host/host-system"; + }; + }; + device-gpu = { + type = "Provider"; + spec = { + artifactId = "device-worker-acceptance-provider"; + config.controllerExecutionRef = "Host/host-system"; + }; + }; + # The Device under test: an emulated TPM claimed by the Guest. The + # Provider's projection declares `Process/swtpm-tpm0`, + # `EphemeralProcess/swtpm-flush-tpm0`, `Endpoint/tpm-tpm0` and + # `Endpoint/tpm-ctrl-tpm0` as this Device's children. + tpm0 = { + type = "Device"; + metadata.ownerRef = "Guest/acceptance-guest"; + spec = { + providerRef = "Provider/device-tpm"; + deviceClass = "emulated"; + arbitration = "exclusive"; + maxConcurrentClaims = 1; + inventory.selector = { }; + }; + }; + # The GPU/video Devices: a full GPU with its video sidecar + # (`gpu-worker` + `video-worker` rows) and a render-node-only Device + # (`gpu-render-node` row, the shape whose render node the broker + # pre-opens itself). Both are physical DRM Devices by declaration; the + # VM has no GPU, which is exactly what the fixture measures. + gpu0 = { + type = "Device"; + metadata.ownerRef = "Guest/acceptance-guest"; + spec = { + providerRef = "Provider/device-gpu"; + deviceClass = "physical"; + arbitration = "exclusive"; + maxConcurrentClaims = 1; + inventory.selector = { busClass = "drm"; label = "u17-gpu0"; }; + }; + }; + gpu1 = { + type = "Device"; + metadata.ownerRef = "Guest/acceptance-guest"; + spec = { + providerRef = "Provider/device-gpu"; + deviceClass = "physical"; + arbitration = "exclusive"; + maxConcurrentClaims = 1; + inventory.selector = { busClass = "drm"; label = "u17-gpu1"; }; + }; + }; + }; + }; + }; + # The guest each fixture-less image evaluates, by the name the image action # asks for. A check's own guest is read out of the check's fixture; these are # the guests with no fixture to be read out of - the two reusable shapes the @@ -1029,6 +1518,10 @@ rec { node = d2bDaemonSmokeNode; testName = "d2b-daemon-smoke"; }; + device-worker-launch = { + node = d2bDeviceWorkerLaunchNode; + testName = "d2b-device-worker-launch"; + }; guest-agent-cap-confinement = { node = d2bGuestAgentCapConfinementNode; testName = "d2b-guest-agent-cap-confinement"; @@ -1049,9 +1542,6 @@ rec { node = d2bStatePostureContractNode; testName = "d2b-state-posture-contract"; }; - wayland-proxy = { - node = d2bWaylandProxyNode; - testName = "d2b-wayland-proxy"; - }; + }; } diff --git a/packages/d2b-test-vm-harness/src/checks/device_worker_launch.rs b/packages/d2b-test-vm-harness/src/checks/device_worker_launch.rs new file mode 100644 index 000000000..e7a6adb44 --- /dev/null +++ b/packages/d2b-test-vm-harness/src/checks/device_worker_launch.rs @@ -0,0 +1,1157 @@ +//! The Device-worker launch path check (U17 slice 3), ported from its +//! fixture. +//! +//! It boots the daemon host the reusable node declares and proves, on a live +//! host, the path a Device's declared worker rows take. One Device declares +//! the Provider's swtpm rows and two declare the GPU rows; the Zone bundle +//! must carry exactly one declared row per Device worker template, owned by +//! its Device and bound to the Provider's digest-pinned executable with +//! launch arguments admitted, and the manager must ingest those rows so the +//! Process controller is the component that launches them. +//! +//! The TPM half runs end to end: `Process/swtpm-tpm0` reaches `Ready` with +//! the composed swtpm argv, the per-VM server socket and the +//! controller-created state dir exist, and the one-shot +//! `EphemeralProcess/swtpm-flush-tpm0` publishes its outcome as the row's +//! status projection. Deleting the Device then retires both rows and the +//! worker, children first. +//! +//! The GPU half is asserted the way the fixture asserted it, because the VM +//! has no GPU: both declared rows resolve and their launch is attempted, and +//! every GPU row must end on the named refusal +//! (`process-start-budget-exhausted` at `reconcile/launch`) - never `Ready` +//! and never a bare launch. +//! +//! The assertions below are the fixture's, in the fixture's order, with the +//! fixture's own command text and bounds. What the fixture expressed as +//! `machine.*` calls is [`GuestControl`]'s own operations, what it expressed +//! as the prelude's `stage`/`diag_unit`/`diag_wait` calls are the same +//! primitives, and what it built in Python - the `d2b`/`list_json`/ +//! `flush_get` command builders and the row dumps its diagnostics print - are +//! private ports in this module, so a failure reported here reads the way the +//! fixture's failure read. The guest it boots is declared in +//! `nix/test-support/host-integration-node.nix`, and `start_all()` is not +//! restated here: it is the lane's own boot of the guest the check runs +//! against. + +use std::{ + collections::BTreeMap, + thread, + time::{Duration, Instant}, +}; + +use serde_json::{json, Value}; + +use crate::legacy::{DiagRow, GuestControl, LegacyError, LegacyResult}; + +/// The zone the check drives, the fixture's own. +const ZONE: &str = "work"; + +/// The Guest that owns the Devices under test, the fixture's own. +const GUEST: &str = "acceptance-guest"; + +/// The declared flush row, read by exact ref rather than by a zone-wide +/// `list EphemeralProcess`. +const FLUSH_ROW: &str = "swtpm-flush-tpm0"; + +/// The row uid the fixture requires: a real v4 uuid. +const UUID_V4: &str = + "^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$"; + +/// The bound the daemon's activation gets, the fixture's own. +const DAEMON_UP: Duration = Duration::from_secs(180); + +/// The bound the broker socket's wait gets, the fixture's own. +const BROKER_SOCKET: Duration = Duration::from_secs(30); + +/// The bound the public socket's file wait gets, the fixture's own. +const PUBLIC_SOCKET: Duration = Duration::from_secs(30); + +/// The bound the rows-ingested wait gets, the fixture's own. +const ROWS_INGESTED: Duration = Duration::from_secs(180); + +/// The bound the declared TPM row's readiness wait gets, the fixture's own. +const TPM_READY: Duration = Duration::from_secs(180); + +/// The bound the live swtpm process's wait gets, the fixture's own. +const TPM_WORKER: Duration = Duration::from_secs(60); + +/// The bound the swtpm sockets' wait gets, the fixture's own. +const TPM_SOCKETS: Duration = Duration::from_secs(60); + +/// The bound the flush row's outcome wait gets, the fixture's own. +const FLUSH_OUTCOME: Duration = Duration::from_secs(180); + +/// The bound the teardown wait gets, the fixture's own. +const TEARDOWN: Duration = Duration::from_secs(180); + +/// The window the launch-outcome evidence loop polls for, the fixture's own. +const OUTCOME_WINDOW: Duration = Duration::from_secs(150); + +/// The window the GPU refusal loop polls for, the fixture's own. +const GPU_WINDOW: Duration = Duration::from_secs(180); + +/// How long each of the two polling windows waits between attempts, the +/// fixture's own `_time.sleep(0.5)`. +const POLL_INTERVAL: Duration = Duration::from_millis(500); + +/// The broker service the fixture starts by hand, after the broker socket. +const START_BROKER: &str = "systemctl start d2b-broker.service"; + +/// The Zone bundle every compile-time assertion in the first stage reads. +const BUNDLE: &str = "cat /etc/d2b/zones/work/resource-bundle.json"; + +/// The row fields the fixture's dumps and projections read, its own +/// `ROW_FIELDS`. +const ROW_FIELDS: &str = concat!( + "{type: .type, name: .metadata.name, owner: .metadata.ownerRef, ", + "uid: .metadata.uid, gen: .metadata.generation, ", + "obs: .status.observedGeneration, phase: .status.phase, ", + "template: .spec.template, resource: .status.resource}", +); + +/// The dump the ingested Process rows are read out of. +const INGEST_PROCESS: &str = "/run/d2b-u17-ingest-process.json"; + +/// The dump the ingested flush row is read out of. +const INGEST_FLUSH: &str = "/run/d2b-u17-ingest-flush.json"; + +/// The dump the outcome window polls. +const OUTCOME_PROCESS: &str = "/run/d2b-u17-outcome-process.json"; + +/// The dump the outcome window's flush read polls. +const OUTCOME_FLUSH: &str = "/run/d2b-u17-outcome-flush.json"; + +/// The dump the GPU window polls. +const GPU_PROCESS: &str = "/run/d2b-u17-gpu-process.json"; + +/// The dump the TPM row's readiness wait reads. +const TPM_PROCESS: &str = "/run/d2b-u17-tpm-process.json"; + +/// The dump the flush row's outcome wait reads. +const FLUSH_PROCESS: &str = "/run/d2b-u17-flush-process.json"; + +/// The dump the teardown wait reads. +const TEARDOWN_PROCESS: &str = "/run/d2b-u17-teardown-process.json"; + +/// The dump the retired flush row is read out of. +const TEARDOWN_FLUSH: &str = "/run/d2b-u17-teardown-flush.json"; + +/// The dump the Device's revision is read out of. +const DEVICE_PRE_DELETE: &str = "/run/d2b-u17-device-pre-delete.json"; + +/// The dump the Device delete writes. +const DEVICE_DELETE: &str = "/run/d2b-u17-device-delete.json"; + +/// The resource types the first-stage diagnostics dump a row set for, the +/// fixture's own list. +const ROW_DUMP_KINDS: [&str; 5] = ["Device", "Process", "Endpoint", "Volume", "Provider"]; + +/// The `rows-ingested` wait's jq program, the fixture's own composition: three +/// declared Process rows, each with its Device owner, its declared template +/// and a real v4 uid, plus the flush row read by exact ref with the same +/// shape. +fn rows_ingested_jq() -> String { + format!( + concat!( + "([.resources[] | select(.type == \"Process\" and ", + "(.metadata.name | test(\"^(swtpm-tpm0|gpu-gpu0|gpu-gpu1)$\")))] ", + "| length) == 3 and ", + "([.resources[] | select(.type == \"Process\" and ", + "(.metadata.name | test(\"^(swtpm-tpm0|gpu-gpu0|gpu-gpu1)$\")) ", + "and (.metadata.uid | test(\"{uuid}\")))] | length) == 3 and ", + "([.resources[] | select(.type == \"Process\" and ", + "({triples}))] | length) == 3 and ", + "([$flush[0] | select(.type == \"EphemeralProcess\" and ", + ".metadata.name == \"swtpm-flush-tpm0\" and ", + ".metadata.ownerRef == \"Device/tpm0\" and ", + ".spec.template == \"swtpm-init-flush\" and ", + "(.metadata.uid | test(\"{uuid}\")))] | length) == 1", + ), + uuid = UUID_V4, + triples = declared_process_triples(), + ) +} + +/// The `Process/swtpm-tpm0` row must be `Ready` with its generation settled, +/// owned by its Device, on its declared template. +const TPM_ROW_READY: &str = concat!( + "jq -e '([.resources[] | select(.type == \"Process\" and ", + ".metadata.name == \"swtpm-tpm0\") | select(", + ".status.phase == \"Ready\" and ", + ".status.observedGeneration == .metadata.generation and ", + ".metadata.ownerRef == \"Device/tpm0\" and ", + ".spec.template == \"swtpm-socket\")] | length) == 1' ", + "/run/d2b-u17-tpm-process.json", +); + +/// The swtpm worker really runs, under the Process controller. +const SWTPM_ALIVE: &str = + "test \"$(ps -eo args= | awk '/[s]wtpm socket/ {c++} END {print c+0}')\" -ge 1"; + +/// The live swtpm workers' argv, one line each, the fixture's own read. +const SWTPM_ARGV: &str = concat!( + "for pid in $(pgrep -f '[s]wtpm socket'); do tr '\\0' ' ' < /proc/$pid/cmdline; ", + "echo; done", +); + +/// The `--pid file=` value of the live swtpm worker, reduced to its +/// directory: the controller-created Volume root. +const SWTPM_STATE_DIR: &str = concat!( + "for pid in $(pgrep -f '[s]wtpm socket'); do ", + "tr '\\0' '\\n' < /proc/$pid/cmdline | sed -n '/--pid/{n;p}'; done ", + "| head -n1 | sed 's|^file=||' | xargs -r dirname", +); + +/// The one-shot flush publishes its outcome as the row's status projection, +/// which is what the TPM port's flush gate reads. +const FLUSH_OUTCOME_READY: &str = concat!( + "jq -e '([select(.type == \"EphemeralProcess\" and ", + ".metadata.name == \"swtpm-flush-tpm0\" and ", + ".metadata.ownerRef == \"Device/tpm0\" and ", + ".spec.template == \"swtpm-init-flush\" and ", + ".status.resource.ephemeral.state == \"succeeded\" and ", + ".status.resource.ephemeral.code == \"process-exited\")] | length) == 1' ", + "/run/d2b-u17-flush-process.json", +); + +/// The outcome window's row-status probe: the three declared Process rows +/// before the fixture's `---` marker, and the flush row after it. +const OUTCOME_PROBE: &str = concat!( + "jq -c '[.resources[] | select(.metadata.name | ", + "test(\"^(swtpm-tpm0|gpu-gpu0|gpu-gpu1)$\")) | ", + "{name: .metadata.name, phase: .status.phase, ", + "resource: .status.resource}]' /run/d2b-u17-outcome-process.json", + " && echo '---' && jq -c '[select(.metadata.name == ", + "\"swtpm-flush-tpm0\") | {name: .metadata.name, ", + "phase: .status.phase, resource: .status.resource}]' ", + "/run/d2b-u17-outcome-flush.json", +); + +/// The three declared Process rows as the outcome print reads them. +const OUTCOME_ROWS: &str = concat!( + "jq -c '[.resources[] | select(.metadata.name | ", + "test(\"^(swtpm-tpm0|gpu-gpu0|gpu-gpu1)$\")) | ", + "{name: .metadata.name, owner: .metadata.ownerRef, ", + "phase: .status.phase, resource: .status.resource}]' ", + "/run/d2b-u17-outcome-process.json", +); + +/// The flush row as the outcome print reads it. +const OUTCOME_FLUSH_ROW: &str = concat!( + "jq -c '[select(.metadata.name == \"swtpm-flush-tpm0\") | ", + "{phase: .status.phase, resource: .status.resource}]' ", + "/run/d2b-u17-outcome-flush.json", +); + +/// The launch evidence the fixture prints from the daemon journal. +const LAUNCH_REFUSAL_LINES: &str = concat!( + "journalctl -u d2bd.service --no-pager -o cat -b -n 4000 ", + "| grep -E 'device-worker|process-resolution-refused|", + "provider-ticket|swtpm|w1-gpu' | tail -n 40 || true", +); + +/// The flush outcome projection the fixture prints. +const FLUSH_PROJECTION_ROWS: &str = "jq -c '[.status.resource]' /run/d2b-u17-flush-process.json"; + +/// Both GPU rows' phase and resource, grouped by Device. +const GPU_ROWS: &str = concat!( + "jq -c '{gpu0: [.resources[] | select(.metadata.name == \"gpu-gpu0\") ", + "| {phase: .status.phase, resource: .status.resource}], ", + "gpu1: [.resources[] | select(.metadata.name == \"gpu-gpu1\") ", + "| {phase: .status.phase, resource: .status.resource}]}' ", + "/run/d2b-u17-gpu-process.json", +); + +/// The GPU evidence the fixture prints from the daemon journal. +const GPU_REFUSAL_LINES: &str = concat!( + "journalctl -u d2bd.service --no-pager -o cat -b -n 4000 ", + "| grep -E 'device-worker|process-resolution-refused|gpu-runner-shape|", + "render|w1-gpu|video' | tail -n 40 || true", +); + +/// The Device's revision, which the delete carries. +const DEVICE_REVISION: &str = concat!( + "jq -er '.resources[] | select(.type == \"Device\" and ", + ".metadata.name == \"tpm0\") | .metadata.revision' ", + "/run/d2b-u17-device-pre-delete.json", +); + +/// Deleting the Device retires its declared rows through the Process +/// controller: the declared `Process/swtpm-tpm0` row is gone. +const TPM_ROWS_RETIRED: &str = concat!( + "jq -e 'all(.resources[]; ", + "(.type == \"Process\" and .metadata.name == \"swtpm-tpm0\") | not)' ", + "/run/d2b-u17-teardown-process.json", +); + +/// Whether a swtpm worker is still alive, without refusing when it is not. +const PGREP_SWTPM: &str = "pgrep -f '[s]wtpm socket' >/dev/null"; + +/// One declared Device worker row, the fixture's own `DECLARED_ROWS` entry: +/// its name, its resource type, its declared template, and the Device that +/// owns it. +struct DeclaredRow { + name: &'static str, + kind: &'static str, + template: &'static str, + owner: &'static str, +} + +/// The declared rows, in the fixture's own order. +const DECLARED_ROWS: [DeclaredRow; 4] = [ + DeclaredRow { + name: "swtpm-tpm0", + kind: "Process", + template: "swtpm-socket", + owner: "Device/tpm0", + }, + DeclaredRow { + name: "swtpm-flush-tpm0", + kind: "EphemeralProcess", + template: "swtpm-init-flush", + owner: "Device/tpm0", + }, + DeclaredRow { + name: "gpu-gpu0", + kind: "Process", + template: "gpu-worker", + owner: "Device/gpu0", + }, + DeclaredRow { + name: "gpu-gpu1", + kind: "Process", + template: "gpu-worker", + owner: "Device/gpu1", + }, +]; + +/// The template binding's owner is the Device *Provider* that signs the +/// template (the declared row's own owner is the Device), the fixture's own +/// `BINDING_OWNER`. +const BINDING_OWNERS: [(&str, &str); 3] = [ + ("Device/tpm0", "Provider/device-tpm"), + ("Device/gpu0", "Provider/device-gpu"), + ("Device/gpu1", "Provider/device-gpu"), +]; + +/// The worker executable each declared template names, the fixture's own +/// expected-binary table. +const EXPECTED_BINARIES: [(&str, &str); 5] = [ + ("swtpm-socket", "swtpm"), + ("swtpm-init-flush", "swtpm-ioctl"), + ("gpu-worker", "crosvm"), + ("gpu-render-node", "crosvm"), + ("video-worker", "crosvm"), +]; + +/// The check's assertions, in the order its fixture made them. +pub fn assertions(control: &mut GuestControl) -> LegacyResult<()> { + let dumps = row_dumps(); + let rows = as_rows(&dumps); + + control.stage("daemon-up"); + control.diag_unit("daemon-up", "d2bd.service", DAEMON_UP)?; + control.wait_for_unit("d2b-broker.socket", None, BROKER_SOCKET)?; + control.wait_for_file("/run/d2b/public.sock", PUBLIC_SOCKET)?; + control.succeed(&[START_BROKER], None)?; + + // 1. Slice 1's compile-time half, on the live host: the Zone bundle + // carries exactly one declared row per Device worker template, owned by + // its Device, each bound to the Provider's digest-pinned executable + // with launch arguments admitted. No hardware and no launch involved. + control.stage("bundle-projection"); + let bundle = parse_json( + &control.succeed(&[BUNDLE], None)?, + "the Zone resource bundle", + )?; + let declared = declared_rows(&bundle); + for declared_row in &DECLARED_ROWS { + let (name, kind, template, owner) = ( + declared_row.name, + declared_row.kind, + declared_row.template, + declared_row.owner, + ); + let Some(row) = declared.get(&(kind, name)) else { + return Err(LegacyError::Assertion(format!( + "declared row {kind}/{name} missing from the bundle" + ))); + }; + let declared_owner = row + .get("metadata") + .and_then(|metadata| metadata.get("ownerRef")); + if declared_owner.and_then(Value::as_str) != Some(owner) { + return Err(LegacyError::Assertion(format!( + "{kind}/{name}: declared owner {} != {}", + python_repr(declared_owner), + python_repr_str(owner), + ))); + } + let declared_template = row.get("spec").and_then(|spec| spec.get("template")); + if declared_template.and_then(Value::as_str) != Some(template) { + return Err(LegacyError::Assertion(format!( + "{kind}/{name}: declared template {} != {}", + python_repr(declared_template), + python_repr_str(template), + ))); + } + if declares_argv(row) { + return Err(LegacyError::Assertion(format!( + "{kind}/{name}: the declared row must stay argv-free" + ))); + } + } + let bindings = template_bindings(&bundle); + let compiled = process_templates(&bundle) + .iter() + .map(|binding| { + json!({ + "processRef": binding.get("processRef").cloned().unwrap_or(Value::Null), + "ownerRef": binding.get("ownerRef").cloned().unwrap_or(Value::Null), + "template": binding.get("template").cloned().unwrap_or(Value::Null), + "binaryRef": binding.get("binaryRef").cloned().unwrap_or(Value::Null), + "launchArgs": binding.get("launchArgs").cloned().unwrap_or(Value::Bool(false)), + }) + }) + .collect::>(); + control.announce(&format!( + "[d2b] compiled processTemplates: {}", + python_dumps(&Value::Array(compiled)), + )); + for declared_row in &DECLARED_ROWS { + let (name, kind, template, owner) = ( + declared_row.name, + declared_row.kind, + declared_row.template, + declared_row.owner, + ); + let reference = format!("{kind}/{name}"); + let Some(binding) = bindings.get(reference.as_str()) else { + return Err(LegacyError::Assertion(format!( + "no Device worker template binding for {reference}" + ))); + }; + let binding_template = binding.get("template"); + if binding_template.and_then(Value::as_str) != Some(template) { + return Err(LegacyError::Assertion(format!( + "{reference}: binding template {} != {}", + python_repr(binding_template), + python_repr_str(template), + ))); + } + let launch_args = binding.get("launchArgs"); + if launch_args != Some(&Value::Bool(true)) { + return Err(LegacyError::Assertion(format!( + "{reference}: binding must admit launch arguments (saw {})", + python_repr(launch_args), + ))); + } + let expected_owner = binding_owner(owner); + let binding_owner_ref = binding.get("ownerRef"); + if binding_owner_ref.and_then(Value::as_str) != Some(expected_owner) { + return Err(LegacyError::Assertion(format!( + "{reference}: binding owner {} != {}", + python_repr(binding_owner_ref), + python_repr_str(expected_owner), + ))); + } + let expected = expected_binary(template); + let binding_binary = binding.get("binaryRef"); + if binding_binary.and_then(Value::as_str) != Some(expected) { + return Err(LegacyError::Assertion(format!( + "{reference}: binding binary {} != {}", + python_repr(binding_binary), + python_repr_str(expected), + ))); + } + } + let declared_bindings = bindings + .iter() + .map(|(reference, binding)| { + json!({ + "processRef": reference, + "template": binding.get("template").cloned().unwrap_or(Value::Null), + "binaryRef": binding.get("binaryRef").cloned().unwrap_or(Value::Null), + "launchArgs": binding.get("launchArgs").cloned().unwrap_or(Value::Bool(false)), + }) + }) + .collect::>(); + control.announce(&format!( + "[d2b] declared device worker bindings: {}", + python_dumps(&Value::Array(declared_bindings)), + )); + + // 2. The rows are ingested into the manager with their Device owner, so the + // Process controller is the component that launches them (KTD13). The + // three declared Process rows are the whole Process set this stage + // asserts (the fourth declared row is the flush EphemeralProcess below), + // each present with its Device owner, declared template, and a real v4 + // uid. + control.stage("rows-ingested"); + control.diag_wait( + "rows-ingested", + &rows_ingested_command(), + ROWS_INGESTED, + &rows, + &[("d2bd.service", "device-worker")], + )?; + for declared_row in &DECLARED_ROWS { + let (name, kind, template, owner) = ( + declared_row.name, + declared_row.kind, + declared_row.template, + declared_row.owner, + ); + let (source, row_source) = if kind == "Process" { + ( + INGEST_PROCESS, + format!(".resources[] | select(.type == \"{kind}\" and .metadata.name == \"{name}\")"), + ) + } else { + (INGEST_FLUSH, ".".to_owned()) + }; + let output = control.succeed( + &[&format!( + "jq -c '[{row_source} | {{owner: .metadata.ownerRef, \ + template: .spec.template, phase: .status.phase}}]' {source}" + )], + None, + )?; + let ingested = parse_json(&output, "the ingested rows dump")?; + let ingested: &[Value] = match ingested.as_array() { + Some(ingested) => ingested.as_slice(), + None => &[], + }; + if ingested.len() != 1 { + return Err(LegacyError::Assertion(format!( + "{kind}/{name}: expected one ingested row, got {output}" + ))); + } + let ingested_owner = ingested[0].get("owner"); + let ingested_template = ingested[0].get("template"); + if ingested_owner.and_then(Value::as_str) != Some(owner) + || ingested_template.and_then(Value::as_str) != Some(template) + { + return Err(LegacyError::Assertion(format!( + "{kind}/{name}: ingested shape {output}" + ))); + } + } + + // Evidence for the read path above (not an assertion): a zone-wide + // `list EphemeralProcess` is exec-routed and never reaches the manager, so + // record its exit status and stderr once per run. The declared row itself + // is read by exact ref through `flush_get` instead. + let list_probe = control.execute( + &d2b( + "list EphemeralProcess 2>&1", + "/run/d2b-u17-ephemeralprocess-probe.json", + ), + None, + )?; + control.announce(&format!( + "[d2b] zone-wide list EphemeralProcess probe: exit {}; {}", + list_probe.status, + list_probe.output.trim().replace('\n', " | "), + )); + + // 3. Launch-outcome evidence (diagnostic, not an assertion): every + // declared row reaches either Ready or a terminal classification within + // the bounded window, and the log carries the classification the row + // and the daemon publish. A row still Pending here is a launch that + // never resolved; the stages below assert the target behavior. + control.stage("worker-launch-outcome"); + let outcome_deadline = Instant::now() + OUTCOME_WINDOW; + while Instant::now() < outcome_deadline { + control.succeed(&[&format!("{}; echo OK", list_json("Process", OUTCOME_PROCESS))], None)?; + control.succeed(&[&format!("{}; echo OK", flush_get(OUTCOME_FLUSH))], None)?; + let probed = control.succeed(&[OUTCOME_PROBE], None)?; + // The fixture read the half before its `---` marker, which is the + // three declared Process rows. + let rows_now = parse_json( + probed.split("---").next().unwrap_or_default(), + "the declared row outcomes", + )?; + let flat_now: &[Value] = match rows_now.as_array() { + Some(rows_now) => rows_now.as_slice(), + None => &[], + }; + let settled = flat_now.iter().filter(|row| is_settled(row)).count(); + if !flat_now.is_empty() && settled == flat_now.len() { + break; + } + pause(); + } + let declared_outcomes = parse_json( + &control.succeed(&[OUTCOME_ROWS], None)?, + "the declared row outcomes", + )?; + if let Some(declared_outcomes) = declared_outcomes.as_array() { + for row in declared_outcomes { + control.announce(&format!("[d2b] declared row outcome: {}", python_dumps(row))); + } + } + let flush_outcome = control.succeed(&[OUTCOME_FLUSH_ROW], None)?; + control.announce(&format!("[d2b] flush row outcome: {flush_outcome}")); + let launch_lines = control.succeed(&[LAUNCH_REFUSAL_LINES], None)?; + control.announce(&format!("[d2b] launch refusal lines:\n{launch_lines}")); + + // 4. TPM end to end. The declared `Process/swtpm-tpm0` row is launched by + // the Process controller with the parameters the Device row, the + // declared template, and the daemon runtime paths supply: it reaches + // Ready, the real swtpm process is alive with that argv, and its + // sockets exist. + control.stage("tpm-worker"); + control.diag_wait( + "tpm-worker-ready", + &format!("{} && {TPM_ROW_READY}", list_json("Process", TPM_PROCESS)), + TPM_READY, + &rows, + &[("d2bd.service", "swtpm"), ("d2b-broker.service", "w1-swtpm")], + )?; + control.diag_wait( + "tpm-worker-process", + SWTPM_ALIVE, + TPM_WORKER, + &rows, + &[("d2bd.service", "swtpm")], + )?; + let argv = control.succeed(&[SWTPM_ARGV], None)?.trim().to_owned(); + control.announce(&format!("[d2b] live swtpm argv: {argv}")); + if !(argv.contains("--tpm2") + && argv.contains("--ctrl") + && argv.contains("--server") + && argv.contains("--tpmstate")) + { + return Err(LegacyError::Assertion(format!( + "the live swtpm argv must be the composed swtpm shape: {}", + python_repr_str(&argv), + ))); + } + if !argv.contains(&format!("path=/run/d2b/vms/{GUEST}/tpm.sock")) { + return Err(LegacyError::Assertion(format!( + "the live swtpm argv must carry the per-VM server socket: {}", + python_repr_str(&argv), + ))); + } + if !(argv.contains("device-") && argv.contains("tpm-state/ctrl.sock")) { + return Err(LegacyError::Assertion(format!( + "the live swtpm argv must carry the controller-created state dir: {}", + python_repr_str(&argv), + ))); + } + let state_dir = control.succeed(&[SWTPM_STATE_DIR], None)?.trim().to_owned(); + if !(state_dir.starts_with("/var/lib/d2b/") && state_dir.ends_with("tpm-state")) { + return Err(LegacyError::Assertion(format!( + "the swtpm state dir must be the controller-created Volume root: {}", + python_repr_str(&state_dir), + ))); + } + control.diag_wait( + "tpm-sockets", + &format!("test -S /run/d2b/vms/{GUEST}/tpm.sock && test -S {state_dir}/ctrl.sock"), + TPM_SOCKETS, + &rows, + &[("d2bd.service", "swtpm")], + )?; + let sockets = control.succeed( + &[&format!( + "stat -c '%F %a %U:%G %n' /run/d2b/vms/{GUEST}/tpm.sock {state_dir}/ctrl.sock" + )], + None, + )?; + control.announce(&format!( + "[d2b] swtpm state dir: {state_dir}; sockets: {}", + sockets.replace('\n', " | "), + )); + + // 4. The one-shot flush publishes its outcome as the row's status + // projection, which is what the TPM port's flush gate reads. + control.stage("tpm-flush"); + control.diag_wait( + "tpm-flush-outcome", + &format!("{} && {FLUSH_OUTCOME_READY}", flush_get(FLUSH_PROCESS)), + FLUSH_OUTCOME, + &rows, + &[ + ("d2bd.service", "swtpm-flush"), + ("d2bd.service", "ephemeral"), + ], + )?; + let flush_projection = control.succeed(&[FLUSH_PROJECTION_ROWS], None)?; + control.announce(&format!("[d2b] flush outcome projection: {flush_projection}")); + + // 5. GPU: the launch path, honestly. The VM has no GPU, so the fixture + // states what must happen instead of faking a device: the declared rows + // resolve, their launch is attempted through the Process controller and + // the broker, and every GPU row ends on a named refusal - never Ready, + // never a bare launch. + control.stage("gpu-launch"); + // The GPU rows must end on a named refusal, never Ready and never a fake + // device. The row status (and the daemon journal) name the stage. + let mut observed: Option = None; + let deadline = Instant::now() + GPU_WINDOW; + while Instant::now() < deadline { + control.succeed(&[&format!("{}; echo OK", list_json("Process", GPU_PROCESS))], None)?; + let statuses = parse_json(&control.succeed(&[GPU_ROWS], None)?, "the GPU row statuses")?; + let flat = gpu_rows(&statuses); + let terminal = flat + .iter() + .filter(|row| matches!(row_phase(row), Some("Failed" | "Quarantined"))) + .count(); + if terminal == 2 || flat.iter().any(|row| row_phase(row) == Some("Ready")) { + observed = Some(statuses); + break; + } + pause(); + } + let Some(observed) = observed else { + return Err(LegacyError::Assertion( + "every GPU/video row must reach a terminal refusal without a GPU".to_owned(), + )); + }; + let flat = gpu_rows(&observed); + if flat.len() != 2 || !flat.iter().all(|row| row_phase(row) == Some("Failed")) { + return Err(LegacyError::Assertion(format!( + "every GPU/video row must end Failed without a GPU: {}", + python_dumps(&observed), + ))); + } + for row in &flat { + // The refusal is the closed classification the plan documents: the + // restart ceiling makes a persistently refused launch terminal as + // `process-start-budget-exhausted` at the launch stage + // (`reconcile/launch`), never Ready and never a bare failure. + let failure = row + .get("resource") + .and_then(|resource| resource.get("driverFailure")); + let code = failure + .and_then(|failure| failure.get("code")) + .and_then(Value::as_str); + let stage = failure + .and_then(|failure| failure.get("stage")) + .and_then(Value::as_str); + if code != Some("process-start-budget-exhausted") || stage != Some("reconcile/launch") { + return Err(LegacyError::Assertion(format!( + "the GPU/video refusal must be the budget-exhausted launch refusal \ + (process-start-budget-exhausted at reconcile/launch): {}", + python_dumps(row), + ))); + } + } + control.announce(&format!( + "[d2b] GPU/video terminal refusals: {}", + python_dumps(&observed), + )); + let codes = control.succeed(&[GPU_REFUSAL_LINES], None)?; + control.announce(&format!("[d2b] GPU/video refusal evidence:\n{codes}")); + + // 6. Teardown: deleting the Device retires its declared rows through the + // Process controller - the process stops and the rows go away, children + // first, with nothing of the Device left behind. + control.stage("tpm-teardown"); + control.succeed(&[&list_json("Device", DEVICE_PRE_DELETE)], None)?; + let revision = control.succeed(&[DEVICE_REVISION], None)?.trim().to_owned(); + control.succeed( + &[&d2b(&format!("delete Device/tpm0 --revision {revision}"), DEVICE_DELETE)], + None, + )?; + control.diag_wait( + "tpm-teardown", + &format!("{} && {TPM_ROWS_RETIRED}", list_json("Process", TEARDOWN_PROCESS)), + TEARDOWN, + &rows, + &[("d2bd.service", "swtpm"), ("d2bd.service", "delete")], + )?; + let flush_gone = control.execute(&flush_get(TEARDOWN_FLUSH), None)?; + if flush_gone.status == 0 { + return Err(LegacyError::Assertion(format!( + "the Device delete must retire EphemeralProcess/swtpm-flush-tpm0 \ + (Get must refuse, saw exit {})", + flush_gone.status, + ))); + } + control.announce(&format!( + "[d2b] flush row after teardown: {}", + flush_gone.output.trim().replace('\n', " | "), + )); + if control.execute(PGREP_SWTPM, None)?.status == 0 { + return Err(LegacyError::Assertion( + "no swtpm worker may outlive the Device that declared it".to_owned(), + )); + } + control.announce("[d2b] Device/tpm0 teardown retired its declared rows and the worker"); + + control.stage("done"); + control.announce("[d2b] U17 device-worker launch path holds on the live host"); + Ok(()) +} + +/// The public CLI invocation every read in this check makes, the fixture's +/// own `d2b(command, out)`. +fn d2b(command: &str, out: &str) -> String { + format!( + "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock \ + d2b --zone {ZONE} --json {command} >{out}" + ) +} + +/// One zone-wide list of a resource type, the fixture's own `list_json`. +fn list_json(kind: &str, out: &str) -> String { + d2b(&format!("list {kind}"), out) +} + +/// The declared flush row read by exact ref, the fixture's own `flush_get`. +fn flush_get(out: &str) -> String { + d2b(&format!("get EphemeralProcess/{FLUSH_ROW}"), out) +} + +/// The bundle's declared rows by `(type, name)`, the fixture's own `declared` +/// map: the Process and EphemeralProcess rows the Zone bundle carries. +fn declared_rows(bundle: &Value) -> BTreeMap<(&str, &str), &Value> { + let mut declared = BTreeMap::new(); + for row in bundle_resources(bundle) { + let kind = row.get("type").and_then(Value::as_str); + let name = row + .get("metadata") + .and_then(|metadata| metadata.get("name")) + .and_then(Value::as_str); + if let (Some(kind), Some(name)) = (kind, name) { + if matches!(kind, "Process" | "EphemeralProcess") { + declared.insert((kind, name), row); + } + } + } + declared +} + +/// The bundle's `resources`, the rows its declared map is built from. +fn bundle_resources(bundle: &Value) -> &[Value] { + match bundle.get("resources").and_then(Value::as_array) { + Some(resources) => resources.as_slice(), + None => &[], + } +} + +/// The bundle's `processTemplates`, or none when the bundle carries none. +fn process_templates(bundle: &Value) -> &[Value] { + match bundle.get("processTemplates").and_then(Value::as_array) { + Some(bindings) => bindings.as_slice(), + None => &[], + } +} + +/// The bundle's process template bindings by `processRef`, the fixture's own +/// `bindings` map. +fn template_bindings(bundle: &Value) -> BTreeMap<&str, &Value> { + process_templates(bundle) + .iter() + .filter_map(|binding| { + binding + .get("processRef") + .and_then(Value::as_str) + .map(|process_ref| (process_ref, binding)) + }) + .collect() +} + +/// Whether a declared row carries launch arguments of its own, which the +/// fixture refused: a declared row names a template, never an argv. +fn declares_argv(row: &Value) -> bool { + match row.get("spec").and_then(Value::as_object) { + Some(spec) => spec.contains_key("command") || spec.contains_key("argv"), + None => false, + } +} + +/// The `or`-joined triples the rows-ingested wait requires, the fixture's own +/// composition over the declared Process rows. +fn declared_process_triples() -> String { + DECLARED_ROWS + .iter() + .filter(|row| row.kind == "Process") + .map(|row| { + format!( + "(.metadata.name == \"{}\" and .metadata.ownerRef == \"{}\" \ + and .spec.template == \"{}\")", + row.name, row.owner, row.template, + ) + }) + .collect::>() + .join(" or ") +} + +/// The rows-ingested wait's command, the fixture's own: the Process list, the +/// flush row read by exact ref, and the jq program that asserts both. +fn rows_ingested_command() -> String { + format!( + "{} && {} && jq -e --slurpfile flush /run/d2b-u17-ingest-flush.json '{}' \ + /run/d2b-u17-ingest-process.json", + list_json("Process", INGEST_PROCESS), + flush_get(INGEST_FLUSH), + rows_ingested_jq(), + ) +} + +/// Both Devices' row arrays, concatenated the way the fixture's `flat` was. +fn gpu_rows(observed: &Value) -> Vec<&Value> { + let mut rows = Vec::new(); + for key in ["gpu0", "gpu1"] { + if let Some(array) = observed.get(key).and_then(Value::as_array) { + rows.extend(array.iter()); + } + } + rows +} + +/// A row's phase, or nothing when it carries none. +fn row_phase(row: &Value) -> Option<&str> { + row.get("phase").and_then(Value::as_str) +} + +/// Whether a row is out of the launch path: `Ready`, or terminally Failed or +/// Quarantined. +fn is_settled(row: &Value) -> bool { + matches!(row_phase(row), Some("Ready" | "Failed" | "Quarantined")) +} + +/// The executable a declared template names. Every template this fixture +/// declares on a Device is in the table; a template outside it is reported as +/// the empty executable, so the binding assertion below fails honestly. +fn expected_binary(template: &str) -> &str { + EXPECTED_BINARIES + .iter() + .find(|(declared, _)| *declared == template) + .map(|(_, binary)| *binary) + .unwrap_or("") +} + +/// The Device *Provider* that signs one Device's templates, the fixture's own +/// `BINDING_OWNER` lookup. +fn binding_owner(owner: &str) -> &str { + BINDING_OWNERS + .iter() + .find(|(device, _)| *device == owner) + .map(|(_, provider)| *provider) + .unwrap_or("") +} + +/// One command's JSON output, as the fixture's `_json.loads` read it. +fn parse_json(output: &str, what: &str) -> LegacyResult { + serde_json::from_str(output) + .map_err(|error| LegacyError::Assertion(format!("{what} is not readable JSON: {error}"))) +} + +/// The interval the fixture's two bounded polling windows slept between +/// attempts: `_time.sleep(0.5)`, from the host that drives the guest. +/// +/// The lint's replacement vocabulary is for executor workers; a ported +/// check's assertions run on the lane's own synchronous check path, which is +/// the path every [`GuestControl`] operation runs on, so this takes the +/// per-site allow with the sanctioned reason rather than an async timer. +#[allow(clippy::disallowed_methods, reason = "synchronous path")] +fn pause() { + thread::sleep(POLL_INTERVAL); +} + +/// The rows the fixture's diagnostics dump on a timed-out wait, in its own +/// order: one per resource type, the exec-routed flush/list probe, the zone +/// bundle, the storage rows, the site artifact, and the worker sockets. +fn row_dumps() -> Vec<(String, String)> { + let row_projection = format!("[.resources[] | {ROW_FIELDS}]"); + let flush_projection = format!("[. | {ROW_FIELDS}]"); + let mut dumps = Vec::new(); + for kind in ROW_DUMP_KINDS { + let out = format!("/run/d2b-u17-{}.json", kind.to_lowercase()); + dumps.push(( + format!("{kind} rows"), + format!( + "{} && jq -c '{row_projection}' {out} || true", + list_json(kind, &out), + ), + )); + } + dumps.push(( + "declared flush row (Get) and zone-wide list EphemeralProcess (exec-routed)".to_owned(), + format!( + "{} && jq -c '{flush_projection}' \ + /run/d2b-u17-ephemeralprocess-get.json; rc=0; {} \ + || rc=$?; echo list-exit=$rc; true", + flush_get("/run/d2b-u17-ephemeralprocess-get.json"), + d2b( + "list EphemeralProcess 2>&1", + "/run/d2b-u17-ephemeralprocess.json", + ), + ), + )); + dumps.push(( + "zone resource bundle".to_owned(), + concat!( + "jq -c '{resources: [.resources[] | select(.type == \"Process\" or ", + ".type == \"EphemeralProcess\") | {type, name: .metadata.name, ", + "owner: .metadata.ownerRef, template: .spec.template}], ", + "bindings: [.processTemplates[] | {processRef, ownerRef, template, ", + "launchArgs, binaryRef}]}' /etc/d2b/zones/work/resource-bundle.json ", + "|| true", + ) + .to_owned(), + )); + dumps.push(( + "swtpm storage rows".to_owned(), + concat!( + "jq -c '[.paths[] | select(.id | test(\"swtpm\")) | {id, scope, ", + "pathTemplate}]' /etc/d2b/storage.json || true", + ) + .to_owned(), + )); + dumps.push(( + "site artifact".to_owned(), + "cat /etc/d2b/site.json 2>/dev/null || echo 'no site.json'".to_owned(), + )); + dumps.push(( + "device worker sockets".to_owned(), + concat!( + "find /run/d2b/vms /run/d2b-video -maxdepth 3 ", + "-printf '%M %u:%g %p\\n' 2>/dev/null | sort | head -n 40 || true", + ) + .to_owned(), + )); + dumps +} + +/// The row dumps as the diagnostics take them. +fn as_rows(dumps: &[(String, String)]) -> Vec> { + dumps + .iter() + .map(|(label, command)| (label.as_str(), command.as_str())) + .collect() +} + +/// `json.dumps(value, sort_keys=True)`, the rendering the fixture's `print` +/// lines used: keys sorted, `, ` and `: ` separators, and non-ASCII escaped +/// the way `ensure_ascii` escapes it. +fn python_dumps(value: &Value) -> String { + let mut out = String::new(); + dumps_into(value, &mut out); + out +} + +/// One value appended to a `json.dumps` rendering. +fn dumps_into(value: &Value, out: &mut String) { + match value { + Value::Null => out.push_str("null"), + Value::Bool(true) => out.push_str("true"), + Value::Bool(false) => out.push_str("false"), + Value::Number(number) => out.push_str(&number.to_string()), + Value::String(text) => dumps_string(text, out), + Value::Array(items) => { + out.push('['); + for (index, item) in items.iter().enumerate() { + if index > 0 { + out.push_str(", "); + } + dumps_into(item, out); + } + out.push(']'); + } + Value::Object(fields) => { + let mut keys = fields.keys().collect::>(); + keys.sort_unstable(); + out.push('{'); + for (index, key) in keys.iter().enumerate() { + if index > 0 { + out.push_str(", "); + } + dumps_string(key, out); + out.push_str(": "); + dumps_into(&fields[(*key).as_str()], out); + } + out.push('}'); + } + } +} + +/// One string in `json.dumps` form: double-quoted, with the escapes +/// `ensure_ascii` writes. +fn dumps_string(text: &str, out: &mut String) { + out.push('"'); + for character in text.chars() { + match character { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + '\u{08}' => out.push_str("\\b"), + '\u{0c}' => out.push_str("\\f"), + character if character < ' ' || character > '~' => { + let mut buffer = [0_u16; 2]; + for unit in character.encode_utf16(&mut buffer) { + out.push_str(&format!("\\u{unit:04x}")); + } + } + character => out.push(character), + } + } + out.push('"'); +} + +/// `repr(...)` of a value the fixture interpolated with `!r`; a field a row +/// does not carry is `None`. +fn python_repr(value: Option<&Value>) -> String { + match value { + None | Some(Value::Null) => "None".to_owned(), + Some(Value::Bool(true)) => "True".to_owned(), + Some(Value::Bool(false)) => "False".to_owned(), + Some(Value::Number(number)) => number.to_string(), + Some(Value::String(text)) => python_repr_str(text), + Some(Value::Array(items)) => { + let items = items + .iter() + .map(|item| python_repr(Some(item))) + .collect::>(); + format!("[{}]", items.join(", ")) + } + Some(Value::Object(fields)) => { + let fields = fields + .iter() + .map(|(key, value)| { + format!("{}: {}", python_repr_str(key), python_repr(Some(value))) + }) + .collect::>(); + format!("{{{}}}", fields.join(", ")) + } + } +} + +/// `repr(...)` of a string, the way the fixture's `!r` interpolations +/// rendered one. +fn python_repr_str(text: &str) -> String { + let quote = if text.contains('\'') && !text.contains('"') { + '"' + } else { + '\'' + }; + let mut out = String::new(); + out.push(quote); + for character in text.chars() { + if character == quote { + out.push('\\'); + out.push(character); + } else { + match character { + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + character if character < ' ' => { + out.push_str(&format!("\\x{:02x}", character as u32)); + } + character => out.push(character), + } + } + } + out.push(quote); + out +} diff --git a/packages/d2b-test-vm-harness/src/checks/mod.rs b/packages/d2b-test-vm-harness/src/checks/mod.rs index 846598bac..84fb77785 100644 --- a/packages/d2b-test-vm-harness/src/checks/mod.rs +++ b/packages/d2b-test-vm-harness/src/checks/mod.rs @@ -25,6 +25,7 @@ pub mod bridge_isolation; pub mod daemon_smoke; +pub mod device_worker_launch; pub mod guest_agent_cap_confinement; pub mod guest_shell_service; pub mod privilege_oracle; @@ -48,6 +49,10 @@ pub type Assertions = fn(&mut GuestControl) -> LegacyResult<()>; const PORTED: &[(&str, Assertions)] = &[ ("bridge-isolation", bridge_isolation::assertions), ("daemon-smoke", daemon_smoke::assertions), + ( + "device-worker-launch", + device_worker_launch::assertions, + ), ( "guest-agent-cap-confinement", guest_agent_cap_confinement::assertions, diff --git a/tests/host-integration/device-worker-launch.nix b/tests/host-integration/device-worker-launch.nix deleted file mode 100644 index 377ccc7fe..000000000 --- a/tests/host-integration/device-worker-launch.nix +++ /dev/null @@ -1,1065 +0,0 @@ -# Type-G runNixOSTest: the U17 Device-worker launch path (slice 3). -# -# One Device declares the Provider's swtpm rows, one declares the GPU/video -# rows. The fixture proves, on a live host: -# -# 1. TPM (no hardware needed). `Process/swtpm-` and -# `EphemeralProcess/swtpm-flush-` are the declared rows of the -# owning Device; the swtpm worker really runs with the argv the Process -# controller composed (state dir + ctrl/server sockets + principal), binds -# its sockets, and the one-shot flush publishes its outcome as the row's -# status projection; deleting the Device retires the declared rows through -# the Process controller (children first). -# 2. GPU (path only). No GPU exists in the VM, so the fixture pins the launch -# PATH: the declared `Process/gpu-` rows resolve, their launch is -# attempted through the Process controller and the broker, and the outcome -# is a named refusal on the row - never a bare launch, never Ready, and -# never a fake device. -# -# The worker executables come from the fixture's own signed Provider -# artifacts: `swtpm`/`swtpm-ioctl` are the real binaries (nixpkgs swtpm); the -# GPU artifact's `crosvm` is a fixture stand-in (an ELF shim that records its -# argv and refuses), because the fixture never intends a GPU worker to serve - -# it stands in for the executable a real GPU Provider artifact packages so the -# compiler emits the same digest-pinned binding. -{ pkgs, self }: - -let - inherit (pkgs) lib; - hostToolBundle = - if self.lib ? d2bHostToolBundle then self.lib.d2bHostToolBundle else null; - d2bLib = import ./lib.nix { - inherit self; - inherit lib; - inherit hostToolBundle; - }; - # The reusable guest configuration lives outside this directory so it - # survives the fixture (see `nix/test-support/host-integration-node.nix`). - d2bNode = import ../../nix/test-support/host-integration-node.nix { - inherit self; - inherit lib; - }; - cloudHypervisorArtifact = d2bLib.mkRuntimeCloudHypervisorArtifact pkgs; - volumeProviderArtifact = d2bLib.mkVolumeProviderArtifact pkgs; - - # A Provider artifact that packages the Device worker executables the - # declared rows name. Shape mirrors `mkVolumeProviderArtifact`: a signed - # manifest whose executable set is computed from the packaged `bin/` files, - # a Device-exporting catalog entry, and a deterministic publisher key. - mkDeviceWorkerProviderArtifact = - { artifactId - , publisher - , binaries - , controllerBinary - }: - let - signer = pkgs.python3.withPackages - (pythonPackages: [ pythonPackages.cryptography ]); - manifest = ../../tests/fixtures/provider-acceptance/provider-manifest.json; - schema = ../../tests/fixtures/provider-acceptance/config-schema.json; - controller = if hostToolBundle == null then - "${self.packages.${pkgs.stdenv.hostPlatform.system}.d2b-provider-test-controller}/bin/d2b-provider-test-controller" - else - "${hostToolBundle}/bin/d2b-provider-test-controller"; - package = pkgs.runCommand "d2b-${artifactId}" { - nativeBuildInputs = [ pkgs.coreutils signer ]; - } '' - mkdir -p "$out/bin" - ${lib.concatStringsSep "\n" (lib.mapAttrsToList - (name: path: '' - cp "${path}" "$out/bin/${name}" - chmod 0755 "$out/bin/${name}" - '') - binaries)} - cp "${controller}" "$out/bin/${controllerBinary}" - chmod 0755 "$out/bin/${controllerBinary}" - ${signer}/bin/python3 - "${manifest}" "$out" \ - "${artifactId}" "${publisher}" "${controllerBinary}" \ - ${lib.escapeShellArg (lib.concatStringsSep " " (lib.attrNames binaries))} <<'PY' - import hashlib - import json - import pathlib - import sys - from cryptography.hazmat.primitives import serialization - from cryptography.hazmat.primitives.asymmetric.ed25519 import ( - Ed25519PrivateKey, - ) - - ( - manifest_path, - output_path, - artifact_id, - publisher, - controller_binary, - binary_names, - ) = sys.argv[1:] - output = pathlib.Path(output_path) - manifest = json.loads(pathlib.Path(manifest_path).read_text()) - # The executable set the compiler recomputes covers every regular file - # in bin/: the controller binary plus the declared worker binaries. - names = sorted(set(binary_names.split()) | {controller_binary}) - - # Device-only manifest: the declared Device worker rows are the only - # rows this artifact's Provider serves in this fixture. - resource_types = {"Device"} - manifest["apiBindings"] = [ - binding - for binding in manifest.get("apiBindings", []) - if binding.get("resourceType") in resource_types - ] - for component in manifest.get("components", []): - component["exportedResourceTypes"] = [ - resource_type - for resource_type in component.get("exportedResourceTypes", []) - if resource_type in resource_types - ] - - manifest["artifactId"] = artifact_id - manifest["trust"]["publisher"] = publisher - executable_map = json.dumps( - { - name: "sha256:" + hashlib.sha256( - (output / "bin" / name).read_bytes() - ).hexdigest() - for name in names - }, - ensure_ascii=False, - sort_keys=True, - separators=(",", ":"), - ).encode() - first = hashlib.sha256( - b"d2b:v3:provider-executable-set\0" + executable_map - ).digest() - executable_digest = "sha256:" + hashlib.sha256(first).hexdigest() - controller_digest = "sha256:" + hashlib.sha256( - (output / "bin" / controller_binary).read_bytes() - ).hexdigest() - manifest["digests"]["executable"] = executable_digest - for component in manifest.get("components", []): - for capability in component.get("targetCapabilities", []): - capability["artifactDigest"] = controller_digest - manifest_bytes = json.dumps( - manifest, - ensure_ascii=False, - sort_keys=True, - separators=(",", ":"), - ).encode() - seed = hashlib.sha256( - b"d2b-u17-device-worker-provider-signing-key-v1" - + artifact_id.encode() - ).digest() - private_key = Ed25519PrivateKey.from_private_bytes(seed) - public_key = private_key.public_key().public_bytes( - serialization.Encoding.PEM, - serialization.PublicFormat.SubjectPublicKeyInfo, - ) - metadata = output / "share/d2b/provider" - metadata.mkdir(parents=True) - (metadata / "provider-manifest.json").write_bytes(manifest_bytes) - (metadata / "provider-manifest.json.sig").write_bytes( - private_key.sign(manifest_bytes) - ) - (metadata / "config-schema.json").write_bytes( - pathlib.Path("${schema}").read_bytes() - ) - (output / "publisher-public-key.pem").write_bytes(public_key) - (output / "executable-set-digest").write_text(executable_digest) - (output / "manifest-digest").write_text( - "sha256:" + hashlib.sha256(manifest_bytes).hexdigest() - ) - PY - ''; - packageDigestPath = pkgs.runCommand - "d2b-${artifactId}-nar-digest" { - nativeBuildInputs = [ pkgs.nix ]; - } '' - printf 'sha256:%s' \ - "$(${pkgs.nix}/bin/nix --extra-experimental-features nix-command \ - hash path --type sha256 --base16 "${package}")" > "$out" - ''; - baseManifest = builtins.fromJSON (builtins.readFile manifest); - catalog = { - providerName = artifactId; - packageName = "d2b-${artifactId}"; - version = "0.0.0"; - systems = [ pkgs.stdenv.hostPlatform.system ]; - platform = pkgs.stdenv.hostPlatform.system; - apiCompatibility = "d2b.zone.v3"; - serviceCompatibility = "d2bd.resource"; - signature = { signatureId = "default"; }; - rootEpoch = 1; - revocationStatus = "clear"; - denyStatus = "clear"; - provenanceEvidence = "accepted"; - sbomEvidence = "accepted"; - licenseEvidence = "accepted"; - vulnerabilityEvidence = "accepted"; - conformanceAttestation = "accepted"; - supportChannel = "stable"; - supportContact = "d2b-u17-device-worker@localhost"; - publisher = publisher; - packageDigest = lib.removeSuffix "\n" - (builtins.readFile packageDigestPath); - executableDigest = lib.removeSuffix "\n" - (builtins.readFile "${package}/executable-set-digest"); - manifestDigest = lib.removeSuffix "\n" - (builtins.readFile "${package}/manifest-digest"); - componentDigest = "sha256:${builtins.hashString - "sha256" (builtins.toJSON baseManifest.components)}"; - descriptorDigest = "sha256:${builtins.hashString - "sha256" (builtins.toJSON baseManifest.apiBindings)}"; - configDigest = "sha256:${builtins.hashString - "sha256" (builtins.readFile schema)}"; - }; - in { - inherit package catalog; - type = "provider"; - trustedPublisher = { - publisherRef = publisher; - signingKey = builtins.readFile "${package}/publisher-public-key.pem"; - }; - }; - - # The GPU artifact's crosvm stand-in: a real ELF (via the shim) so the - # artifact validates as a Provider executable set, whose behavior is to - # record its argv and refuse. The fixture never asks a GPU worker to serve. - crosvmStandIn = self.lib.buildProviderElfShim { - inherit pkgs; - name = "crosvm"; - interpreterPkg = pkgs.bash; - interpreterPath = "bin/bash"; - program = pkgs.writeText "d2b-u17-crosvm-stand-in.sh" '' - # Fixture stand-in for the GPU Provider's crosvm. It must never run in - # a passing fixture (the launch path is proved up to the broker's own - # refusal); if it does run, it records the argv the Process controller - # composed and refuses, so a fabricated success is impossible. - set -eu - log="/run/d2b/u17-device-worker-standin.argv" - if [ -d /run/d2b ]; then - printf '%s\n' "crosvm-stand-in:$*" >> "$log" 2>/dev/null || true - fi - printf 'd2b-u17: crosvm stand-in invoked with %s\n' "$*" >&2 - exit 79 - ''; - }; - - # One artifact serves both Device Providers: a Provider artifact exports its - # ResourceTypes, and two artifacts both exporting `Device` collide in one - # Zone (`provider-resourcetype-collision`). - deviceWorkerArtifact = mkDeviceWorkerProviderArtifact { - artifactId = "device-worker-acceptance-provider"; - publisher = "d2b-u17-device-worker"; - controllerBinary = "acceptance-controller"; - binaries = { - swtpm = "${pkgs.swtpm}/bin/swtpm"; - swtpm-ioctl = "${pkgs.swtpm}/bin/swtpm_ioctl"; - crosvm = "${crosvmStandIn}/bin/crosvm"; - }; - }; - - cloudHypervisorConfig = { - controllerExecutionRef = "Host/host-system"; - defaultVcpus = 2; - defaultMemoryMb = 512; - defaultMachineType = "microvm"; - watchdog = true; - adoptionWindowMs = 30000; - healthCheckIntervalMs = 5000; - healthCheckTimeoutMs = 1000; - healthCheckFailureThreshold = 3; - startupDeadlineMs = 120000; - }; - - artifacts = { - runtime-cloud-hypervisor = { - inherit (cloudHypervisorArtifact) package type catalog; - }; - volume-acceptance-provider = { - inherit (volumeProviderArtifact) package type catalog; - }; - device-worker-acceptance-provider = { - inherit (deviceWorkerArtifact) package type catalog; - }; - }; -in -pkgs.testers.runNixOSTest { - name = "d2b-device-worker-launch"; - - nodes.machine = d2bNode.d2bDaemonNode { - extra = { ... }: { - d2b.site.adminUsers = [ "alice" ]; - environment.systemPackages = with pkgs; [ - jq - procps - util-linux - acl - iproute2 - # The fixture binds its stale video socket from the VM side; python3 - # is the smallest reliable AF_UNIX binder available in the VM. - python3 - ]; - d2b.artifacts = artifacts; - d2b.zones.local-root.resources.host-system = { - type = "Host"; - spec = { - providerRef = "Provider/system-core"; - defaultDomain = "system"; - allowedDomains = [ "system" ]; - budget = { }; - networkAttachments = [ ]; - deviceAttachments = [ ]; - volumeAttachmentDefaults = [ ]; - }; - }; - d2b.zones.work.parentZone = "local-root"; - # Every Zone the host compiles a bundle for declares the publishers of - # the artifacts its rows select; `local-root` is a compiled Zone too - # (the other Cloud Hypervisor fixtures declare the same pair there). - d2b.zones.local-root.trustedPublishers.d2b-cloud-hypervisor.signingKey = - cloudHypervisorArtifact.trustedPublisher.signingKey; - d2b.zones.local-root.trustedPublishers.d2b-volume-acceptance.signingKey = - volumeProviderArtifact.trustedPublisher.signingKey; - d2b.zones.local-root.trustedPublishers.d2b-u17-device-worker.signingKey = - deviceWorkerArtifact.trustedPublisher.signingKey; - d2b.zones.work.trustedPublishers.d2b-cloud-hypervisor.signingKey = - cloudHypervisorArtifact.trustedPublisher.signingKey; - d2b.zones.work.trustedPublishers.d2b-volume-acceptance.signingKey = - volumeProviderArtifact.trustedPublisher.signingKey; - d2b.zones.work.trustedPublishers.d2b-u17-device-worker.signingKey = - deviceWorkerArtifact.trustedPublisher.signingKey; - d2b.zones.work.resources = { - alice = { - type = "User"; - spec = { - displayName = "Alice"; - groups = [ ]; - osUsername = "alice"; - }; - }; - d2bd = { - type = "User"; - spec = { - displayName = "d2bd"; - groups = [ ]; - osUsername = "d2bd"; - }; - }; - device-operator = { - type = "Role"; - spec.rules = [ - { - resourceTypes = [ - "Device" - "Endpoint" - "EphemeralProcess" - "Guest" - "Host" - "Process" - "Provider" - "Volume" - ]; - verbs = [ "get" "list" ]; - subresources = [ ]; - resourceNames = [ ]; - zones = [ "work" ]; - executionRefs = [ ]; - sessionVerbs = [ "connect" "invoke" ]; - } - { - resourceTypes = [ "Device" ]; - verbs = [ "delete" ]; - subresources = [ ]; - resourceNames = [ "tpm0" ]; - zones = [ "work" ]; - executionRefs = [ ]; - sessionVerbs = [ "connect" "invoke" ]; - } - ]; - }; - device-operator-binding = { - type = "RoleBinding"; - spec = { - roleRef = "Role/device-operator"; - subjects = [ "User/alice" ]; - externalPrincipalSelector = null; - scopeNarrowing = null; - }; - }; - host-system = { - type = "Host"; - spec = { - providerRef = "Provider/system-core"; - defaultDomain = "system"; - allowedDomains = [ "system" ]; - budget = { }; - networkAttachments = [ ]; - deviceAttachments = [ ]; - volumeAttachmentDefaults = [ ]; - }; - }; - # The Device owners. The Guest stays declared input and is never - # booted: the Device worker rows are bundle-declared `Process` rows of - # the Process controller, and no guest system artifact is declared, so - # no VMM is ever launched here. Its name is the VM identity of the - # Devices it owns (`Device.metadata.ownerRef`). - acceptance-guest = { - type = "Guest"; - spec = { - providerRef = "Provider/volume-virtiofs"; - executionRef = "Host/host-system"; - defaultDomain = "system"; - allowedDomains = [ "system" ]; - budget = { }; - volumeAttachmentDefaults = [ ]; - networkAttachments = [ ]; - deviceAttachments = [ ]; - }; - }; - volume-local = { - type = "Provider"; - spec = { - artifactId = "volume-acceptance-provider"; - config = { - controllerExecutionRef = "Host/host-system"; - sourcePolicies = [ - { - id = "daemon-state"; - class = "local-path"; - volumeKinds = [ "durable" "state" "cache" ]; - } - # The TPM state Volume's source policy - # (`build_tpm_state_volume_spec`, opaque policy id). - { - id = "tpm-state"; - class = "local-path"; - volumeKinds = [ "state" ]; - } - ]; - }; - }; - }; - volume-virtiofs = { - type = "Provider"; - spec = { - artifactId = "volume-acceptance-provider"; - config.controllerExecutionRef = "Host/host-system"; - }; - }; - runtime-cloud-hypervisor = { - type = "Provider"; - spec = { - artifactId = "runtime-cloud-hypervisor"; - config = cloudHypervisorConfig; - }; - }; - device-tpm = { - type = "Provider"; - spec = { - artifactId = "device-worker-acceptance-provider"; - config.controllerExecutionRef = "Host/host-system"; - }; - }; - device-gpu = { - type = "Provider"; - spec = { - artifactId = "device-worker-acceptance-provider"; - config.controllerExecutionRef = "Host/host-system"; - }; - }; - # The Device under test: an emulated TPM claimed by the Guest. The - # Provider's projection declares `Process/swtpm-tpm0`, - # `EphemeralProcess/swtpm-flush-tpm0`, `Endpoint/tpm-tpm0` and - # `Endpoint/tpm-ctrl-tpm0` as this Device's children. - tpm0 = { - type = "Device"; - metadata.ownerRef = "Guest/acceptance-guest"; - spec = { - providerRef = "Provider/device-tpm"; - deviceClass = "emulated"; - arbitration = "exclusive"; - maxConcurrentClaims = 1; - inventory.selector = { }; - }; - }; - # The GPU/video Devices: a full GPU with its video sidecar - # (`gpu-worker` + `video-worker` rows) and a render-node-only Device - # (`gpu-render-node` row, the shape whose render node the broker - # pre-opens itself). Both are physical DRM Devices by declaration; the - # VM has no GPU, which is exactly what the fixture measures. - gpu0 = { - type = "Device"; - metadata.ownerRef = "Guest/acceptance-guest"; - spec = { - providerRef = "Provider/device-gpu"; - deviceClass = "physical"; - arbitration = "exclusive"; - maxConcurrentClaims = 1; - inventory.selector = { busClass = "drm"; label = "u17-gpu0"; }; - }; - }; - gpu1 = { - type = "Device"; - metadata.ownerRef = "Guest/acceptance-guest"; - spec = { - providerRef = "Provider/device-gpu"; - deviceClass = "physical"; - arbitration = "exclusive"; - maxConcurrentClaims = 1; - inventory.selector = { busClass = "drm"; label = "u17-gpu1"; }; - }; - }; - }; - }; - }; - - testScript = '' - ${d2bLib.fixtureDiagnostics} - - import json as _json - import time as _time - from typing import Any - - ZONE = "work" - GUEST = "acceptance-guest" - DEVICES = ["tpm0", "gpu0", "gpu1"] - DECLARED_ROWS = { - # row ref -> (resource type, declared template, owning Device) - "swtpm-tpm0": ("Process", "swtpm-socket", "Device/tpm0"), - "swtpm-flush-tpm0": ("EphemeralProcess", "swtpm-init-flush", "Device/tpm0"), - "gpu-gpu0": ("Process", "gpu-worker", "Device/gpu0"), - "gpu-gpu1": ("Process", "gpu-worker", "Device/gpu1"), - } - # The template binding's owner is the Device *Provider* that signs the - # template (the declared row's own owner is the Device). - BINDING_OWNER = { - "Device/tpm0": "Provider/device-tpm", - "Device/gpu0": "Provider/device-gpu", - "Device/gpu1": "Provider/device-gpu", - } - UUID_V4 = ( - r"^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-" - r"[0-9a-f]{12}$" - ) - - def check(condition, message): - if not condition: - raise AssertionError(message) - - def d2b(command, out): - return ( - "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - f"d2b --zone {ZONE} --json {command} >{out}" - ) - - def list_json(kind, out): - return d2b(f"list {kind}", out) - - # The declared flush row (`EphemeralProcess/swtpm-flush-tpm0`) is read by - # exact ref instead of by a zone-wide `list EphemeralProcess`: the daemon - # routes every EphemeralProcess List into its exec-session surface - # (packages/d2bd/src/composition.rs, `process_resource_management_request` - # -> `dispatch_resource_exec_request`) before the manager sees it, and - # that surface derives an execution ref from a Guest executionRef or from - # a resourceRef, neither of which a zone-wide list carries - so the - # command refuses and never reaches the manager. Get is not intercepted - # and serves the same row projection; the rows-ingested stage records the - # refused list once as evidence. - FLUSH_ROW = "swtpm-flush-tpm0" - - def flush_get(out): - return d2b(f"get EphemeralProcess/{FLUSH_ROW}", out) - - ROW_FIELDS = ( - "{type: .type, name: .metadata.name, owner: .metadata.ownerRef, " - "uid: .metadata.uid, gen: .metadata.generation, " - "obs: .status.observedGeneration, phase: .status.phase, " - "template: .spec.template, resource: .status.resource}" - ) - ROW_PROJECTION = f"[.resources[] | {ROW_FIELDS}]" - # `Get` returns the row itself, so its projection is the same fields - # without the List envelope. - FLUSH_PROJECTION = f"[. | {ROW_FIELDS}]" - row_dumps = [ - ( - f"{kind} rows", - list_json(kind, f"/run/d2b-u17-{kind.lower()}.json") - + f" && jq -c '{ROW_PROJECTION}' /run/d2b-u17-{kind.lower()}.json " - + "|| true", - ) - for kind in [ - "Device", - "Process", - "Endpoint", - "Volume", - "Provider", - ] - ] + [ - ( - "declared flush row (Get) and zone-wide list EphemeralProcess (exec-routed)", - flush_get("/run/d2b-u17-ephemeralprocess-get.json") - + f" && jq -c '{FLUSH_PROJECTION}' " - "/run/d2b-u17-ephemeralprocess-get.json; " - "rc=0; " - + d2b("list EphemeralProcess 2>&1", "/run/d2b-u17-ephemeralprocess.json") - + " || rc=$?; echo list-exit=$rc; true", - ), - ( - "zone resource bundle", - "jq -c '{resources: [.resources[] | select(.type == \"Process\" or " - ".type == \"EphemeralProcess\") | {type, name: .metadata.name, " - "owner: .metadata.ownerRef, template: .spec.template}], " - "bindings: [.processTemplates[] | {processRef, ownerRef, template, " - "launchArgs, binaryRef}]}' /etc/d2b/zones/work/resource-bundle.json " - "|| true", - ), - ( - "swtpm storage rows", - "jq -c '[.paths[] | select(.id | test(\"swtpm\")) | {id, scope, " - "pathTemplate}]' /etc/d2b/storage.json || true", - ), - ( - "site artifact", - "cat /etc/d2b/site.json 2>/dev/null || echo 'no site.json'", - ), - ( - "device worker sockets", - "find /run/d2b/vms /run/d2b-video -maxdepth 3 " - "-printf '%M %u:%g %p\\n' 2>/dev/null | sort | head -n 40 || true", - ), - ] - - start_all() - stage("daemon-up") - diag_unit("daemon-up", "d2bd.service", 180) - machine.wait_for_unit("d2b-broker.socket", timeout=30) - machine.wait_for_file("/run/d2b/public.sock", timeout=30) - machine.succeed("systemctl start d2b-broker.service") - - # 1. Slice 1's compile-time half, on the live host: the Zone bundle carries - # exactly one declared row per Device worker template, owned by its - # Device, each bound to the Provider's digest-pinned executable with - # launch arguments admitted. No hardware and no launch involved. - stage("bundle-projection") - bundle_rows: Any = _json.loads( - machine.succeed("cat /etc/d2b/zones/work/resource-bundle.json") - ) - declared: Any = { - (row["type"], row["metadata"]["name"]): row - for row in bundle_rows["resources"] - if row["type"] in ("Process", "EphemeralProcess") - } - for name, (kind, template, owner) in DECLARED_ROWS.items(): - row = declared.get((kind, name)) - check(row is not None, f"declared row {kind}/{name} missing from the bundle") - check( - row["metadata"].get("ownerRef") == owner, - f"{kind}/{name}: declared owner {row['metadata'].get('ownerRef')!r} " - f"!= {owner!r}", - ) - check( - row["spec"]["template"] == template, - f"{kind}/{name}: declared template {row['spec']['template']!r} " - f"!= {template!r}", - ) - check( - "command" not in row["spec"] and "argv" not in row["spec"], - f"{kind}/{name}: the declared row must stay argv-free", - ) - bindings: Any = { - binding["processRef"]: binding - for binding in bundle_rows.get("processTemplates", []) - } - print( - "[d2b] compiled processTemplates: " - + _json.dumps( - [ - { - "processRef": binding.get("processRef"), - "ownerRef": binding.get("ownerRef"), - "template": binding.get("template"), - "binaryRef": binding.get("binaryRef"), - "launchArgs": binding.get("launchArgs", False), - } - for binding in bundle_rows.get("processTemplates", []) - ], - sort_keys=True, - ) - ) - for name, (kind, template, owner) in DECLARED_ROWS.items(): - ref = f"{kind}/{name}" - binding = bindings.get(ref) - check(binding is not None, f"no Device worker template binding for {ref}") - check( - binding["template"] == template, - f"{ref}: binding template {binding['template']!r} != {template!r}", - ) - check( - binding.get("launchArgs") is True, - f"{ref}: binding must admit launch arguments (saw " - f"{binding.get('launchArgs')!r})", - ) - check( - binding["ownerRef"] == BINDING_OWNER[owner], - f"{ref}: binding owner {binding['ownerRef']!r} " - f"!= {BINDING_OWNER[owner]!r}", - ) - expected_binary = { - "swtpm-socket": "swtpm", - "swtpm-init-flush": "swtpm-ioctl", - "gpu-worker": "crosvm", - "gpu-render-node": "crosvm", - "video-worker": "crosvm", - }[template] - check( - binding["binaryRef"] == expected_binary, - f"{ref}: binding binary {binding['binaryRef']!r} != {expected_binary!r}", - ) - print( - "[d2b] declared device worker bindings: " - + _json.dumps( - [ - { - "processRef": ref, - "template": binding["template"], - "binaryRef": binding["binaryRef"], - "launchArgs": binding.get("launchArgs", False), - } - for ref, binding in sorted(bindings.items()) - ], - sort_keys=True, - ) - ) - - # 2. The rows are ingested into the manager with their Device owner, so the - # Process controller is the component that launches them (KTD13). The - # three declared Process rows are the whole Process set this stage - # asserts (the fourth declared row is the flush EphemeralProcess below), - # each present with its Device owner, declared template, and a real v4 - # uid. - declared_process_triples = " or ".join( - f"(.metadata.name == \"{name}\" and .metadata.ownerRef == \"{owner}\" " - f"and .spec.template == \"{template}\")" - for name, (kind, template, owner) in DECLARED_ROWS.items() - if kind == "Process" - ) - stage("rows-ingested") - diag_wait( - "rows-ingested", - f"{list_json('Process', '/run/d2b-u17-ingest-process.json')} && " - f"{flush_get('/run/d2b-u17-ingest-flush.json')} && " - "jq -e --slurpfile flush /run/d2b-u17-ingest-flush.json '" - "([.resources[] | select(.type == \"Process\" and " - "(.metadata.name | test(\"^(swtpm-tpm0|gpu-gpu0|gpu-gpu1)$\")))] " - "| length) == 3 and " - "([.resources[] | select(.type == \"Process\" and " - "(.metadata.name | test(\"^(swtpm-tpm0|gpu-gpu0|gpu-gpu1)$\")) " - "and (.metadata.uid | test(\"" + UUID_V4 + "\")))] | length) == 3 and " - "([.resources[] | select(.type == \"Process\" and " - "(" + declared_process_triples + "))] | length) == 3 and " - "([$flush[0] | select(.type == \"EphemeralProcess\" and " - ".metadata.name == \"swtpm-flush-tpm0\" and " - ".metadata.ownerRef == \"Device/tpm0\" and " - ".spec.template == \"swtpm-init-flush\" and " - "(.metadata.uid | test(\"" + UUID_V4 + "\")))] | length) == 1' " - "/run/d2b-u17-ingest-process.json", - timeout=180, - rows=row_dumps, - explain=[("d2bd.service", "device-worker")], - ) - for name, (kind, template, owner) in DECLARED_ROWS.items(): - if kind == "Process": - source = "/run/d2b-u17-ingest-process.json" - row_source = ( - f".resources[] | select(.type == \"{kind}\" and " - f".metadata.name == \"{name}\")" - ) - else: - source = "/run/d2b-u17-ingest-flush.json" - row_source = "." - output = machine.succeed( - f"jq -c '[{row_source} | {{owner: .metadata.ownerRef, " - "template: .spec.template, phase: .status.phase}]' " + source - ) - rows = _json.loads(output) - check(len(rows) == 1, f"{kind}/{name}: expected one ingested row, got {output}") - check( - rows[0]["owner"] == owner and rows[0]["template"] == template, - f"{kind}/{name}: ingested shape {output}", - ) - - # Evidence for the read path above (not an assertion): a zone-wide - # `list EphemeralProcess` is exec-routed and never reaches the manager, so - # record its exit status and stderr once per run. The declared row itself - # is read by exact ref through `flush_get` instead. - list_probe = machine.execute( - d2b("list EphemeralProcess 2>&1", "/run/d2b-u17-ephemeralprocess-probe.json") - ) - print( - f"[d2b] zone-wide list EphemeralProcess probe: exit {list_probe[0]}; " - + list_probe[1].strip().replace("\n", " | ") - ) - - # 3. Launch-outcome evidence (diagnostic, not an assertion): every - # declared row reaches either Ready or a terminal classification within - # the bounded window, and the log carries the classification the row - # and the daemon publish. A row still Pending here is a launch that - # never resolved; the stages below assert the target behavior. - stage("worker-launch-outcome") - outcome_deadline = _time.monotonic() + 150 - while _time.monotonic() < outcome_deadline: - machine.succeed( - list_json("Process", "/run/d2b-u17-outcome-process.json") + "; echo OK" - ) - machine.succeed( - flush_get("/run/d2b-u17-outcome-flush.json") + "; echo OK" - ) - rows_now = _json.loads( - machine.succeed( - "jq -c '[.resources[] | select(.metadata.name | " - "test(\"^(swtpm-tpm0|gpu-gpu0|gpu-gpu1)$\")) | " - "{name: .metadata.name, phase: .status.phase, " - "resource: .status.resource}]' /run/d2b-u17-outcome-process.json" - " && echo '---' && jq -c '[select(.metadata.name == " - "\"swtpm-flush-tpm0\") | {name: .metadata.name, " - "phase: .status.phase, resource: .status.resource}]' " - "/run/d2b-u17-outcome-flush.json" - ).split("---")[0] - ) - flat_now = rows_now - settled = [ - row for row in flat_now - if row["phase"] in ("Ready", "Failed", "Quarantined") - ] - if len(settled) == len(flat_now) and flat_now: - break - _time.sleep(0.5) - for row in _json.loads( - machine.succeed( - "jq -c '[.resources[] | select(.metadata.name | " - "test(\"^(swtpm-tpm0|gpu-gpu0|gpu-gpu1)$\")) | " - "{name: .metadata.name, owner: .metadata.ownerRef, " - "phase: .status.phase, resource: .status.resource}]' " - "/run/d2b-u17-outcome-process.json" - ) - ): - print("[d2b] declared row outcome: " + _json.dumps(row, sort_keys=True)) - print( - "[d2b] flush row outcome: " - + machine.succeed( - "jq -c '[select(.metadata.name == \"swtpm-flush-tpm0\") | " - "{phase: .status.phase, resource: .status.resource}]' " - "/run/d2b-u17-outcome-flush.json" - ) - ) - print( - "[d2b] launch refusal lines:\n" - + machine.succeed( - "journalctl -u d2bd.service --no-pager -o cat -b -n 4000 " - "| grep -E 'device-worker|process-resolution-refused|" - "provider-ticket|swtpm|w1-gpu' | tail -n 40 || true" - ) - ) - - # 4. TPM end to end. The declared `Process/swtpm-tpm0` row is launched by - # the Process controller with the parameters the Device row, the - # declared template, and the daemon runtime paths supply: it reaches - # Ready, the real swtpm process is alive with that argv, and its - # sockets exist. - stage("tpm-worker") - diag_wait( - "tpm-worker-ready", - f"{list_json('Process', '/run/d2b-u17-tpm-process.json')} && " - "jq -e '([.resources[] | select(.type == \"Process\" and " - ".metadata.name == \"swtpm-tpm0\") | select(" - ".status.phase == \"Ready\" and " - ".status.observedGeneration == .metadata.generation and " - ".metadata.ownerRef == \"Device/tpm0\" and " - ".spec.template == \"swtpm-socket\")] | length) == 1' " - "/run/d2b-u17-tpm-process.json", - timeout=180, - rows=row_dumps, - explain=[("d2bd.service", "swtpm"), ("d2b-broker.service", "w1-swtpm")], - ) - diag_wait( - "tpm-worker-process", - "test \"$(ps -eo args= | awk '/[s]wtpm socket/ {c++} END {print c+0}')\" -ge 1", - timeout=60, - rows=row_dumps, - explain=[("d2bd.service", "swtpm")], - ) - argv = machine.succeed( - "for pid in $(pgrep -f '[s]wtpm socket'); do tr '\\0' ' ' < /proc/$pid/cmdline; " - "echo; done" - ).strip() - print("[d2b] live swtpm argv: " + argv) - check( - "--tpm2" in argv and "--ctrl" in argv and "--server" in argv - and "--tpmstate" in argv, - f"the live swtpm argv must be the composed swtpm shape: {argv!r}", - ) - check( - f"path=/run/d2b/vms/{GUEST}/tpm.sock" in argv, - f"the live swtpm argv must carry the per-VM server socket: {argv!r}", - ) - check( - "device-" in argv and "tpm-state/ctrl.sock" in argv, - f"the live swtpm argv must carry the controller-created state dir: {argv!r}", - ) - state_dir = machine.succeed( - "for pid in $(pgrep -f '[s]wtpm socket'); do " - "tr '\\0' '\\n' < /proc/$pid/cmdline | sed -n '/--pid/{n;p}'; done " - "| head -n1 | sed 's|^file=||' | xargs -r dirname" - ).strip() - check( - state_dir.startswith("/var/lib/d2b/") and state_dir.endswith("tpm-state"), - f"the swtpm state dir must be the controller-created Volume root: {state_dir!r}", - ) - diag_wait( - "tpm-sockets", - f"test -S /run/d2b/vms/{GUEST}/tpm.sock && test -S {state_dir}/ctrl.sock", - timeout=60, - rows=row_dumps, - explain=[("d2bd.service", "swtpm")], - ) - print( - "[d2b] swtpm state dir: " - + state_dir - + "; sockets: " - + machine.succeed( - f"stat -c '%F %a %U:%G %n' /run/d2b/vms/{GUEST}/tpm.sock {state_dir}/ctrl.sock" - ).replace("\n", " | ") - ) - - # 4. The one-shot flush publishes its outcome as the row's status - # projection, which is what the TPM port's flush gate reads. - stage("tpm-flush") - diag_wait( - "tpm-flush-outcome", - f"{flush_get('/run/d2b-u17-flush-process.json')} && " - "jq -e '([select(.type == \"EphemeralProcess\" and " - ".metadata.name == \"swtpm-flush-tpm0\" and " - ".metadata.ownerRef == \"Device/tpm0\" and " - ".spec.template == \"swtpm-init-flush\" and " - ".status.resource.ephemeral.state == \"succeeded\" and " - ".status.resource.ephemeral.code == \"process-exited\")] | length) == 1' " - "/run/d2b-u17-flush-process.json", - timeout=180, - rows=row_dumps, - explain=[("d2bd.service", "swtpm-flush"), ("d2bd.service", "ephemeral")], - ) - print( - "[d2b] flush outcome projection: " - + machine.succeed( - "jq -c '[.status.resource]' /run/d2b-u17-flush-process.json" - ) - ) - - # 5. GPU: the launch path, honestly. The VM has no GPU, so the fixture - # states what must happen instead of faking a device: the declared rows - # resolve, their launch is attempted through the Process controller and - # the broker, and every GPU row ends on a named refusal - never Ready, - # never a bare launch. (The GPU Device templates that need Provider - # settings - `video-worker`, `gpu-render-node` - would need the - # Provider's signed settings schema registered in the zone; this - # fixture declares the plain `gpu-worker` template on both Devices.) - stage("gpu-launch") - # The GPU rows must end on a named refusal, never Ready and never a fake - # device. The row status (and the daemon journal) name the stage. - observed: Any = None - deadline = _time.monotonic() + 180 - while _time.monotonic() < deadline: - machine.succeed( - list_json("Process", "/run/d2b-u17-gpu-process.json") + "; echo OK" - ) - rows = _json.loads( - machine.succeed( - "jq -c '{gpu0: [.resources[] | select(.metadata.name == \"gpu-gpu0\") " - "| {phase: .status.phase, resource: .status.resource}], " - "gpu1: [.resources[] | select(.metadata.name == \"gpu-gpu1\") " - "| {phase: .status.phase, resource: .status.resource}]}' " - "/run/d2b-u17-gpu-process.json" - ) - ) - flat = rows["gpu0"] + rows["gpu1"] - terminal = [ - row for row in flat if row["phase"] in ("Failed", "Quarantined") - ] - if len(terminal) == 2 or any(row["phase"] == "Ready" for row in flat): - observed = rows - break - _time.sleep(0.5) - check( - observed is not None, - "every GPU/video row must reach a terminal refusal without a GPU", - ) - flat = observed["gpu0"] + observed["gpu1"] - check( - len(flat) == 2 and all(row["phase"] == "Failed" for row in flat), - "every GPU/video row must end Failed without a GPU: " - + _json.dumps(observed, sort_keys=True), - ) - for row in flat: - # The refusal is the closed classification the plan documents: the - # restart ceiling makes a persistently refused launch terminal as - # `process-start-budget-exhausted` at the launch stage - # (`reconcile/launch`), never Ready and never a bare failure. - failure = (row["resource"] or {}).get("driverFailure") or {} - check( - failure.get("code") == "process-start-budget-exhausted" - and failure.get("stage") == "reconcile/launch", - "the GPU/video refusal must be the budget-exhausted launch refusal " - "(process-start-budget-exhausted at reconcile/launch): " - + _json.dumps(row, sort_keys=True), - ) - print( - "[d2b] GPU/video terminal refusals: " + _json.dumps(observed, sort_keys=True) - ) - codes = machine.succeed( - "journalctl -u d2bd.service --no-pager -o cat -b -n 4000 " - "| grep -E 'device-worker|process-resolution-refused|gpu-runner-shape|" - "render|w1-gpu|video' | tail -n 40 || true" - ) - print("[d2b] GPU/video refusal evidence:\n" + codes) - - # 6. Teardown: deleting the Device retires its declared rows through the - # Process controller - the process stops and the rows go away, children - # first, with nothing of the Device left behind. - stage("tpm-teardown") - machine.succeed(f"{list_json('Device', '/run/d2b-u17-device-pre-delete.json')}") - revision = machine.succeed( - "jq -er '.resources[] | select(.type == \"Device\" and " - ".metadata.name == \"tpm0\") | .metadata.revision' " - "/run/d2b-u17-device-pre-delete.json" - ).strip() - machine.succeed( - d2b("delete Device/tpm0 --revision " + revision, "/run/d2b-u17-device-delete.json") - ) - diag_wait( - "tpm-teardown", - f"{list_json('Process', '/run/d2b-u17-teardown-process.json')} && " - "jq -e 'all(.resources[]; " - "(.type == \"Process\" and .metadata.name == \"swtpm-tpm0\") | not)' " - "/run/d2b-u17-teardown-process.json", - timeout=180, - rows=row_dumps, - explain=[("d2bd.service", "swtpm"), ("d2bd.service", "delete")], - ) - flush_gone = machine.execute(flush_get("/run/d2b-u17-teardown-flush.json")) - check( - flush_gone[0] != 0, - "the Device delete must retire EphemeralProcess/swtpm-flush-tpm0 " - f"(Get must refuse, saw exit {flush_gone[0]})", - ) - print( - "[d2b] flush row after teardown: " + flush_gone[1].strip().replace("\n", " | ") - ) - check( - machine.execute("pgrep -f '[s]wtpm socket' >/dev/null")[0] != 0, - "no swtpm worker may outlive the Device that declared it", - ) - print("[d2b] Device/tpm0 teardown retired its declared rows and the worker") - - stage("done") - print("[d2b] U17 device-worker launch path holds on the live host") - ''; -} From 812b842e2d2736e54082f453cf263a66e0601bec Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sat, 26 Sep 2026 22:43:09 -0700 Subject: [PATCH 22/51] refactor(vm): assert virtiofsd-volume-runtime in Rust and retire its fixture The U11 midpoint Volume proof moves into packages/d2b-test-vm-harness/src/checks/virtiofsd_volume_runtime.rs in the fixture's own order: 2 stage ("boot" and the tag `dump_rows` stages), 2 wait_for_unit (nftables.service 180s, d2b-broker.socket 30s), 1 wait_for_file (public.sock, 30s), 1 diag_unit (d2bd.service, 180s), 7 diag_wait (volume-realized 180, binding-realized 120, worker-realized 120, endpoint-realized 120, serving-socket 60, virtiofsd-process 60, binding-deleting 60), 6 succeed (the pre-delete list, the revision read, the delete, the post-delete read, and the two calls inside the teardown sampling loop) and 1 diag (the tag's row dump). No fail call exists in the fixture and none exists here. The two identities the fixture derived - the binding name from the attachment tuple and the serving socket path from (zone, volume, guest) - are its own frozen-v1 derivations and appear here as the constants they resolved to; the row projection every wait carries, the fixture's `chain_row_dumps` and its `dump_rows`, come with it as functions of the module. The teardown window is sampled the way the fixture sampled it: the three lists are read parent -> worker -> endpoint so a violation is an invariant rather than a straddled read, and the samples are asserted over afterwards for endpoint-first ordering and for no owned row outliving its parent, in the fixture's own words. Its guest - the daemon shape plus nftables, the acceptance host runtime, the Volume acceptance artifact and its publisher key, the two zones and their rows with the operator role binding - is declared in nix/test-support/host-integration-node.nix as d2bVirtiofsdVolumeRuntimeNode. Counts the port must satisfy: 2 stage, 2 wait_for_unit, 1 wait_for_file, 1 diag_unit, 7 diag_wait, 1 diag, 6 succeed, 2 assertions plus the non-convergence raise, before and after. --- bazel/checks/vm/BUILD.bazel | 1 + nix/test-support/host-integration-node.nix | 252 +++++++- .../d2b-test-vm-harness/src/checks/mod.rs | 5 + .../src/checks/virtiofsd_volume_runtime.rs | 573 ++++++++++++++++++ .../virtiofsd-volume-runtime.nix | 533 ---------------- 5 files changed, 830 insertions(+), 534 deletions(-) create mode 100644 packages/d2b-test-vm-harness/src/checks/virtiofsd_volume_runtime.rs delete mode 100644 tests/host-integration/virtiofsd-volume-runtime.nix diff --git a/bazel/checks/vm/BUILD.bazel b/bazel/checks/vm/BUILD.bazel index 397c7a5a0..1343a314f 100644 --- a/bazel/checks/vm/BUILD.bazel +++ b/bazel/checks/vm/BUILD.bazel @@ -154,6 +154,7 @@ _PORTED_CHECKS = [ "privilege-oracle", "resource-operator-activation", "state-posture-contract", + "virtiofsd-volume-runtime", "wayland-proxy", ] diff --git a/nix/test-support/host-integration-node.nix b/nix/test-support/host-integration-node.nix index ae5efc750..30a57a256 100644 --- a/nix/test-support/host-integration-node.nix +++ b/nix/test-support/host-integration-node.nix @@ -993,6 +993,249 @@ rec { }; }; + # The guest `virtiofsd-volume-runtime` boots: the reusable daemon node + # plus the fixture's own contributions. + # + # The fixture declared no machine size, disk or device of its own - the shape + # carries those - so what moves here is its `let` bindings (the Volume + # acceptance artifact and the acceptance host runtime) and the extra module + # that turns nftables on, installs the host runtime, declares the two zones + # and their rows, and adds `jq` and `procps`. + d2bVirtiofsdVolumeRuntimeNode = + d2bDaemonNode { + extra = + { lib, pkgs, ... }: + let + d2bLib = import ../../tests/host-integration/lib.nix { + inherit self; + inherit lib; + hostToolBundle = + if self.lib ? d2bHostToolBundle then self.lib.d2bHostToolBundle else null; + }; + volumeProviderArtifact = d2bLib.mkVolumeProviderArtifact pkgs; + artifacts = { + volume-acceptance-provider = { + inherit (volumeProviderArtifact) package type catalog; + }; + }; + hostRuntime = pkgs.writeText "d2b-acceptance-host-runtime.json" (builtins.toJSON { + schemaVersion = "v1"; + bundleVersion = 1; + generatedAt = "1970-01-01T00:00:00.000Z"; + nftAppliedHash = null; + ifnames = [ ]; + }); + in + { + networking.nftables.enable = true; + networking.nftables.ruleset = lib.mkAfter '' + table inet d2b {} + ''; + systemd.tmpfiles.rules = [ + "d /etc/NetworkManager/conf.d 0755 root root -" + ]; + environment.etc."d2b/acceptance-host-runtime.json".source = hostRuntime; + d2b.site.adminUsers = [ "alice" ]; + systemd.services.d2bd.serviceConfig.ExecStartPre = lib.mkAfter [ + "+${pkgs.writeShellScript "d2b-acceptance-hosts-prep" '' + if [ -L /etc/hosts ]; then + ${pkgs.coreutils}/bin/cat /etc/hosts > /run/d2b-acceptance-hosts + ${pkgs.coreutils}/bin/rm -f /etc/hosts + ${pkgs.coreutils}/bin/install -o root -g root -m 0644 \ + /run/d2b-acceptance-hosts /etc/hosts + fi + ''}" + "+${pkgs.writeShellScript "d2b-acceptance-host-runtime-prep" '' + ${pkgs.coreutils}/bin/install -D -o root -g d2bd -m 0640 \ + /etc/d2b/acceptance-host-runtime.json \ + /var/lib/d2b/runtime/host-runtime.json + ''}" + ]; + d2b.artifacts = artifacts; + d2b.zones.local-root.trustedPublishers.d2b-volume-acceptance.signingKey = + volumeProviderArtifact.trustedPublisher.signingKey; + d2b.zones.work.parentZone = "local-root"; + d2b.zones.work.trustedPublishers.d2b-volume-acceptance.signingKey = + volumeProviderArtifact.trustedPublisher.signingKey; + d2b.zones.work.resources = { + alice = { + type = "User"; + spec = { + displayName = "Alice"; + groups = [ ]; + osUsername = "alice"; + }; + }; + d2bd = { + type = "User"; + spec = { + displayName = "d2bd"; + groups = [ ]; + osUsername = "d2bd"; + }; + }; + volume-operator = { + type = "Role"; + spec.rules = [ + { + resourceTypes = [ + "Endpoint" + "Host" + "Process" + "Provider" + "Volume" + "VolumeBinding" + ]; + verbs = [ "get" "list" ]; + subresources = [ ]; + resourceNames = [ ]; + zones = [ "work" ]; + executionRefs = [ ]; + sessionVerbs = [ "connect" "invoke" ]; + } + { + resourceTypes = [ "Volume" ]; + verbs = [ "delete" ]; + subresources = [ ]; + resourceNames = [ "state" ]; + zones = [ "work" ]; + executionRefs = [ ]; + sessionVerbs = [ "connect" "invoke" ]; + } + ]; + }; + volume-operator-binding = { + type = "RoleBinding"; + spec = { + roleRef = "Role/volume-operator"; + subjects = [ "User/alice" ]; + externalPrincipalSelector = null; + scopeNarrowing = null; + }; + }; + # The attachment execution target. The Nix bundle validation requires + # attachment refs to resolve to a same-Zone Host or Guest; the + # virtiofs serving path is host-side (the binding mints its socket + # under /run/d2b/vms//), so this fixture asserts the Volume + # chain only - the Guest row itself stays declared input (KTD1). + acceptance-guest = { + type = "Guest"; + spec = { + defaultDomain = "system"; + providerRef = "Provider/volume-virtiofs"; + budget = { }; + networkAttachments = [ ]; + deviceAttachments = [ ]; + volumeAttachmentDefaults = [ ]; + }; + }; + host-system = { + type = "Host"; + spec = { + providerRef = "Provider/system-core"; + defaultDomain = "system"; + allowedDomains = [ "system" ]; + budget = { }; + networkAttachments = [ ]; + deviceAttachments = [ ]; + volumeAttachmentDefaults = [ ]; + }; + }; + # The fixed daemon-owned Volume owner: volume-local is the only + # Provider a Volume may select (U7 driver contract). + volume-local = { + type = "Provider"; + spec = { + artifactId = "volume-acceptance-provider"; + config = { + controllerExecutionRef = "Host/host-system"; + sourcePolicies = [ + { + id = "daemon-state"; + class = "local-path"; + volumeKinds = [ "durable" "state" "cache" ]; + } + ]; + }; + }; + }; + # The serving Provider the derived VolumeBinding rows select and + # whose signed virtiofsd-worker template the worker launch resolves. + volume-virtiofs = { + type = "Provider"; + spec = { + artifactId = "volume-acceptance-provider"; + config.controllerExecutionRef = "Host/host-system"; + }; + }; + state = { + type = "Volume"; + spec = { + providerRef = "Provider/volume-local"; + kind = "state"; + source = { + executionRef = "Host/host-system"; + settings = { + kind = "local-path"; + sourcePolicyId = "daemon-state"; + }; + }; + layout = [{ + path = "state"; + type = "directory"; + # Daemon-owned so the unprivileged daemon can provision the + # local-path layout inline. + ownerRef = "User/d2bd"; + groupRef = "User/d2bd"; + mode = "0700"; + target = null; + accessAcl = [ ]; + defaultAcl = [ ]; + foreignChildPolicy = "preserve"; + noFollow = true; + recursive = false; + sensitivity = "private"; + createPolicy = "create-if-never-provisioned"; + repairPolicy = "exact-owner"; + cleanupPolicy = "owner-controlled"; + adoptionPolicy = "quarantine-on-ambiguity"; + restartPolicy = "preserve-across-controller-restart"; + leaseClass = "none"; + invariants = [ "no-symlink" ]; + }]; + views.controller = { + path = ""; + rights = [ "read" "write" "traverse" ]; + }; + # KTD1: the attachment stays declared input only. The Volume side + # mints the durable VolumeBinding at reconcile; the deterministic + # binding identity is derived from (volume, execution target, + # view, mount path) - the fixture asserts that exact identity. + attachments = [{ + executionRef = "Guest/acceptance-guest"; + transport = "virtiofs"; + view = "controller"; + access = "read-only"; + mountPath = "/state"; + settings = { + posixAcl = false; + xattr = false; + cache = "auto"; + inodeFileHandles = "never"; + threadPoolSize = null; + socketGroup = null; + }; + }]; + }; + }; + }; + environment.systemPackages = with pkgs; [ + jq + procps + ]; + }; + }; + # The guest `device-worker-launch` boots: the reusable daemon node # plus the fixture's own contributions. # @@ -1542,6 +1785,13 @@ rec { node = d2bStatePostureContractNode; testName = "d2b-state-posture-contract"; }; - + virtiofsd-volume-runtime = { + node = d2bVirtiofsdVolumeRuntimeNode; + testName = "d2b-virtiofsd-volume-runtime"; + }; + wayland-proxy = { + node = d2bWaylandProxyNode; + testName = "d2b-wayland-proxy"; + }; }; } diff --git a/packages/d2b-test-vm-harness/src/checks/mod.rs b/packages/d2b-test-vm-harness/src/checks/mod.rs index 84fb77785..7cb962fd9 100644 --- a/packages/d2b-test-vm-harness/src/checks/mod.rs +++ b/packages/d2b-test-vm-harness/src/checks/mod.rs @@ -31,6 +31,7 @@ pub mod guest_shell_service; pub mod privilege_oracle; pub mod state_posture_contract; pub mod resource_operator_activation; +pub mod virtiofsd_volume_runtime; pub mod wayland_proxy; use crate::legacy::{GuestControl, LegacyResult}; @@ -67,6 +68,10 @@ const PORTED: &[(&str, Assertions)] = &[ "state-posture-contract", state_posture_contract::assertions, ), + ( + "virtiofsd-volume-runtime", + virtiofsd_volume_runtime::assertions, + ), ("wayland-proxy", wayland_proxy::assertions), ]; diff --git a/packages/d2b-test-vm-harness/src/checks/virtiofsd_volume_runtime.rs b/packages/d2b-test-vm-harness/src/checks/virtiofsd_volume_runtime.rs new file mode 100644 index 000000000..39392d954 --- /dev/null +++ b/packages/d2b-test-vm-harness/src/checks/virtiofsd_volume_runtime.rs @@ -0,0 +1,573 @@ +//! The virtiofsd Volume chain check, ported from its fixture. +//! +//! It is the U11 midpoint Volume proof for the v3 resource plane, separate +//! from the Process-slice operator-activation check: the Nix-ingested Volume +//! derives one deterministic VolumeBinding child through the manager, that +//! binding owns the virtiofsd worker Process and its private Endpoint, the +//! worker is realized and binds its private serving socket, and deleting the +//! Volume tears the chain down. Every wait asserts on observable resource +//! identity, phase, generation, processes and sockets - read through the +//! installed d2b CLI and the host filesystem - and nothing here keys on +//! informational log text. +//! +//! The assertions are the fixture's, in the fixture's order and with the +//! fixture's own command text and bounds, and so are the row dumps its +//! failures print: the fixture's own `chain_row_dumps`, which every +//! volume-chain wait carries, and its `dump_rows`, which the teardown +//! window's failure path runs before it reports. The two identities the +//! fixture derives - the binding name from the attachment tuple and the +//! serving socket path from (zone, volume, guest) - are its own frozen-v1 +//! derivations, kept here as the constants they resolved to. +//! +//! The teardown window is sampled the way the fixture sampled it: the three +//! lists are read parent -> worker -> endpoint, so a parent observed gone +//! ahead of a child means the child was already gone when the parent retired +//! rather than that the two reads straddled the teardown, and the samples are +//! asserted over afterwards for endpoint-first ordering and for no owned row +//! outliving its parent. +//! +//! The guest is the reusable daemon node plus the fixture's own contributions +//! - nftables, the acceptance host runtime, the Volume acceptance artifact +//! and its publisher key, the two zones and their rows, the two declared +//! users and the operator role binding - declared in +//! `nix/test-support/host-integration-node.nix`. `start_all()` is not +//! restated here: it is the lane's own boot of the guest the check runs +//! against. + +use std::{thread, time::Duration}; + +use serde_json::Value; + +use crate::legacy::{DiagRow, GuestControl, LegacyError, LegacyResult}; + +/// The bound the nftables unit's boot wait gets, the fixture's own. +const BOOT: Duration = Duration::from_secs(180); + +/// The bound the socket-activated broker socket and the public socket's file +/// wait get, the fixture's own. +const SOCKET_ACTIVATION: Duration = Duration::from_secs(30); + +/// The bound the daemon's unit wait gets, the fixture's own. +const DAEMON_UP: Duration = Duration::from_secs(180); + +/// The bound the Volume realize wait gets, the fixture's own. +const VOLUME_REALIZED: Duration = Duration::from_secs(180); + +/// The bound each derived-child realize wait gets, the fixture's own. +const CHILD_REALIZED: Duration = Duration::from_secs(120); + +/// The bound the serving socket and the worker process get, the fixture's +/// own. +const SERVING: Duration = Duration::from_secs(60); + +/// The bound the binding's own deleting wait gets, the fixture's own. +const BINDING_DELETING: Duration = Duration::from_secs(60); + +/// How many times the teardown window is sampled, the fixture's own bound: +/// six hundred samples with the driver's interval between them. +const TEARDOWN_SAMPLES: usize = 600; + +/// The interval between two samples, the fixture's own `time.sleep(0.2)`. +const SAMPLE_INTERVAL: Duration = Duration::from_millis(200); + +/// The deterministic VolumeBinding identity the frozen v1 derivation pins, +/// the fixture's own `binding_name`: `vol-binding-` followed by the first +/// twenty-four hex digits of the sha256 of the NUL-joined +/// ("d2b/volume-local/binding/v1", "Volume/state", "Guest/acceptance-guest", +/// "controller", "/state"). +const BINDING_NAME: &str = "vol-binding-6a8ea4307a30f7ceae6533f2"; + +/// The private serving socket the worker binds, the fixture's own +/// derivation: `/run/d2b/vms/acceptance-guest/vol-.vfd.sock`, where the +/// tag is the first eight hex digits of the sha256 of the NUL-joined +/// ("work", "state", "acceptance-guest"). +const SOCKET_PATH: &str = "/run/d2b/vms/acceptance-guest/vol-a424ba7a.vfd.sock"; + +/// The uuid shape the read path must serve. A uid is not just non-null: the +/// daemon's `ResourceUid` Display is a redaction placeholder, and a +/// placeholder served through the read path would satisfy a null check while +/// breaking every consumer that parses the field. +const UUID_V4: &str = "^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$"; + +/// The row projection every volume-chain wait's diagnostics print, the +/// fixture's own `chain_projection`. +const CHAIN_PROJECTION: &str = concat!( + "[.resources[] | {type: .type, name: .metadata.name, ", + "owner: .metadata.ownerRef, phase: .status.phase, ", + "gen: .metadata.generation, obs: .status.observedGeneration, ", + "template: .spec.template, purpose: .spec.purpose, ", + "producer: .spec.producerRef}]", +); + +/// At least one live virtiofsd process, counted by the fixture's own `ps` +/// pipeline. +const VIRTIOFSD_PROCESS: &str = concat!( + "test \"$(ps -eo args= | awk '/virtiofsd/ && !/awk/ {c++} END {print c+0}')\" ", + "-ge 1", +); + +/// The revision the delete resolves its exact precondition from, read off the +/// pre-delete list the fixture saved. +const VOLUME_REVISION: &str = concat!( + "jq -er '.resources[] | select(.type == \"Volume\" and ", + ".metadata.name == \"state\") | .metadata.revision' ", + "/run/d2b-volume-pre-delete.json", +); + +/// The check's assertions, in the order its fixture made them. +pub fn assertions(control: &mut GuestControl) -> LegacyResult<()> { + let dumps = chain_row_dumps(); + let rows = rows(&dumps); + + control.stage("boot"); + control.wait_for_unit("nftables.service", None, BOOT)?; + control.wait_for_unit("d2b-broker.socket", None, SOCKET_ACTIVATION)?; + control.diag_unit("daemon-up", "d2bd.service", DAEMON_UP)?; + control.wait_for_file("/run/d2b/public.sock", SOCKET_ACTIVATION)?; + + // 1. Volume realize: the Nix-ingested Volume is Ready with a manager + // identity and an observed generation (U10 ingestion -> U7 driver). + control.diag_wait( + "volume-realized", + &volume_realized(), + VOLUME_REALIZED, + &rows, + &[("d2bd.service", "Volume/state")], + )?; + + // 2. The Volume side minted exactly one deterministic VolumeBinding + // child through the manager, and that child converged. + control.diag_wait( + "binding-realized", + &binding_realized(), + CHILD_REALIZED, + &rows, + &[("d2bd.service", BINDING_NAME)], + )?; + + // 3. The binding owns the virtiofsd worker Process and its private + // Endpoint as managed children; the worker is realized (a live + // virtiofsd process) and the serving socket is bound. + control.diag_wait( + "worker-realized", + &worker_realized(), + CHILD_REALIZED, + &rows, + &[("d2bd.service", "virtiofsd-worker")], + )?; + control.diag_wait( + "endpoint-realized", + &endpoint_realized(), + CHILD_REALIZED, + &rows, + &[("d2bd.service", BINDING_NAME)], + )?; + control.diag_wait( + "serving-socket", + &format!("test -S {SOCKET_PATH}"), + SERVING, + &rows, + &[("d2bd.service", "virtiofsd")], + )?; + control.diag_wait( + "virtiofsd-process", + VIRTIOFSD_PROCESS, + SERVING, + &rows, + &[("d2bd.service", "virtiofsd")], + )?; + + // 4. Deleting the owning Volume drives the whole chain through the + // preserved endpoint-first teardown: the binding is marked deleting + // first, its private Endpoint and socket are removed before the + // worker Process row, and nothing owned survives. + let pre_delete = d2b("list Volume", "/run/d2b-volume-pre-delete.json"); + control.succeed(&[&pre_delete], None)?; + let volume_revision = control.succeed(&[VOLUME_REVISION], None)?.trim().to_owned(); + let delete = d2b( + &format!("delete Volume/state --revision {volume_revision}"), + "/run/d2b-volume-delete.json", + ); + control.succeed(&[&delete], None)?; + + control.diag_wait( + "binding-deleting", + &binding_deleting(), + BINDING_DELETING, + &rows, + &[("d2bd.service", BINDING_NAME)], + )?; + + // Sample the teardown window: the Endpoint row and the worker Process + // row must both disappear, the Endpoint never after the worker, and no + // owned row may outlive its parent. The three lists are separate reads, + // so they run parent -> worker -> endpoint: a parent observed gone ahead + // of a child then really means the child was already gone when the + // parent retired (the invariant under test), while the reverse order + // could straddle the teardown and report a violation that never held. + let lists = teardown_lists(); + let sample_command = teardown_sample(); + let mut observed: Vec = Vec::new(); + let mut converged = false; + for _ in 0..TEARDOWN_SAMPLES { + control.succeed(&[&lists], None)?; + let sample = read_sample(&control.succeed(&[&sample_command], None)?)?; + converged = sample.is_torn_down(); + observed.push(sample); + if converged { + break; + } + sleep_between_samples(); + } + if !converged { + dump_rows(control, "teardown did not converge", &dumps); + let last = match observed.last() { + Some(sample) => sample.as_json(), + // The loop samples before it can leave, so this arm is a guard + // rather than a state; it is reported rather than panicked on. + None => "no sample was taken".to_owned(), + }; + return Err(LegacyError::Assertion(format!( + "volume teardown did not converge within its budget: {last}" + ))); + } + + for sample in &observed { + if sample.binding == 0 && (sample.endpoint != 0 || sample.worker != 0) { + return Err(LegacyError::Assertion(format!( + "owned child outlived its parent binding: {}", + sample.as_python_dict() + ))); + } + if sample.worker == 0 && sample.endpoint != 0 { + return Err(LegacyError::Assertion(format!( + concat!( + "worker Process row disappeared before the binding-owned ", + "Endpoint row (endpoint-first teardown violated): {}", + ), + sample.as_python_dict() + ))); + } + } + + control.succeed(&[&volume_after_delete()], None)?; + Ok(()) +} + +/// The CLI read the fixture's own `d2b(command, out)` helper built: one zone, +/// read as `alice` through the public socket, into the file the wait's jq +/// reads. +fn d2b(command: &str, out: &str) -> String { + format!( + concat!( + "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock ", + "d2b --zone work --json {command} >{out}", + ), + command = command, + out = out, + ) +} + +/// The Volume realize wait, the fixture's own command text. +fn volume_realized() -> String { + format!( + concat!( + "{list_volume} && ", + "jq -e '", + "([.resources[] | select(.type == \"Volume\" and ", + ".metadata.name == \"state\")] | length) == 1 and ", + "(.resources[] | select(.type == \"Volume\" and ", + ".metadata.name == \"state\") | ", + "((.metadata.uid | test(\"{uuid_v4}\")) and ", + ".metadata.generation > 0 and ", + ".metadata.ownerRef == null and ", + ".status.phase == \"Ready\" and ", + ".status.observedGeneration == .metadata.generation))' ", + "/run/d2b-volume-realized.json", + ), + list_volume = d2b("list Volume", "/run/d2b-volume-realized.json"), + uuid_v4 = UUID_V4, + ) +} + +/// The VolumeBinding realize wait, the fixture's own command text: exactly +/// one child of the Volume, at the deterministic identity, `Ready` and +/// settled, carrying the attachment tuple it was derived from. +fn binding_realized() -> String { + format!( + concat!( + "{list_binding} && ", + "jq -e '", + "([.resources[] | select(.type == \"VolumeBinding\" and ", + ".metadata.name == \"{binding_name}\" and ", + ".metadata.ownerRef == \"Volume/state\")] | length) == 1 and ", + "(.resources[] | select(.metadata.name == \"{binding_name}\") | ", + "((.metadata.uid | test(\"{uuid_v4}\")) and ", + ".metadata.generation > 0 and ", + ".status.phase == \"Ready\" and ", + ".status.observedGeneration == .metadata.generation and ", + ".spec.volumeRef == \"Volume/state\" and ", + ".spec.executionRef == \"Guest/acceptance-guest\" and ", + ".spec.view == \"controller\" and ", + ".spec.access == \"read-only\" and ", + ".spec.mountPath == \"/state\"))' ", + "/run/d2b-binding-realized.json", + ), + list_binding = d2b("list VolumeBinding", "/run/d2b-binding-realized.json"), + binding_name = BINDING_NAME, + uuid_v4 = UUID_V4, + ) +} + +/// The worker Process realize wait, the fixture's own command text: exactly +/// one worker owned by the binding, realized from the signed virtiofsd-worker +/// template, `Ready` and settled. +fn worker_realized() -> String { + format!( + concat!( + "{list_process} && ", + "jq -e '", + "([.resources[] | select(.type == \"Process\" and ", + ".metadata.ownerRef == \"VolumeBinding/{binding_name}\")] | length) ", + "== 1 and ", + "([.resources[] | select(.type == \"Process\" and ", + ".metadata.ownerRef == \"VolumeBinding/{binding_name}\") | ", + "(.spec.providerRef == \"Provider/system-minijail\" and ", + ".spec.executionRef == \"Host/host-system\" and ", + ".spec.processClass == \"worker\" and ", + ".spec.template == \"virtiofsd-worker\" and ", + ".status.phase == \"Ready\" and ", + ".status.observedGeneration == .metadata.generation)] | length) == 1' ", + "/run/d2b-worker-realized.json", + ), + list_process = d2b("list Process", "/run/d2b-worker-realized.json"), + binding_name = BINDING_NAME, + ) +} + +/// The Endpoint realize wait, the fixture's own command text: exactly one +/// private Endpoint owned by the binding, `Ready` and settled, whose producer +/// resolves back to the binding's own worker Process row. +fn endpoint_realized() -> String { + format!( + concat!( + "{list_endpoint} && ", + "{list_process} && ", + "jq -e --slurpfile proc /run/d2b-worker-producer.json '", + "([.resources[] | select(.type == \"Endpoint\" and ", + ".metadata.ownerRef == \"VolumeBinding/{binding_name}\")] | length) ", + "== 1 and ", + "([$proc[0].resources[] | select(.type == \"Process\" and ", + ".metadata.ownerRef == \"VolumeBinding/{binding_name}\" and ", + ".status.phase == \"Ready\" and ", + ".status.observedGeneration == .metadata.generation)] | length) ", + "== 1 and ", + "(.resources[] | select(.type == \"Endpoint\" and ", + ".metadata.ownerRef == \"VolumeBinding/{binding_name}\") | ", + "(.status.phase == \"Ready\" and ", + ".status.observedGeneration == .metadata.generation and ", + ".spec.transport == \"unix\" and ", + "(.spec.purpose == \"virtiofsd\" and ", + "(.spec.producerRef as $producer | ", + "any($proc[0].resources[]; ", + ".type == \"Process\" and ", + "\"\\(.type)/\\(.metadata.name)\" == $producer)))))' ", + "/run/d2b-endpoint-realized.json", + ), + list_endpoint = d2b("list Endpoint", "/run/d2b-endpoint-realized.json"), + list_process = d2b("list Process", "/run/d2b-worker-producer.json"), + binding_name = BINDING_NAME, + ) +} + +/// The binding-deleting wait, the fixture's own command text: the binding is +/// marked deleting before anything owned it is torn down. +fn binding_deleting() -> String { + format!( + concat!( + "{list_binding} && ", + "jq -e '", + "any(.resources[]; .type == \"VolumeBinding\" and ", + ".metadata.name == \"{binding_name}\" and ", + ".metadata.deletionRequestedAt != null)' ", + "/run/d2b-binding-deleting.json", + ), + list_binding = d2b("list VolumeBinding", "/run/d2b-binding-deleting.json"), + binding_name = BINDING_NAME, + ) +} + +/// The three teardown-window lists in one command, the fixture's own: read +/// parent -> worker -> endpoint and closed by the driver's own `echo OK`. +fn teardown_lists() -> String { + [ + d2b("list VolumeBinding", "/run/d2b-teardown-binding.json"), + d2b("list Process", "/run/d2b-teardown-process.json"), + d2b("list Endpoint", "/run/d2b-teardown-endpoint.json"), + "echo OK".to_owned(), + ] + .join("; ") +} + +/// The sample command, the fixture's own `sample_expr`: whether the serving +/// socket is still bound, and what each of the three lists it just read held +/// for the binding and its owned rows. The doubled braces are the JSON object +/// the fixture's jq program builds. +fn teardown_sample() -> String { + let binding_owner = format!("VolumeBinding/{BINDING_NAME}"); + format!( + concat!( + "socket=false; test -S {socket_path} && socket=true; ", + "jq -n --argjson socket \"$socket\" ", + "--slurpfile b /run/d2b-teardown-binding.json ", + "--slurpfile p /run/d2b-teardown-process.json ", + "--slurpfile e /run/d2b-teardown-endpoint.json ", + "'{{", + "endpoint: ([$e[0].resources[] | ", + "select(.metadata.ownerRef == \"{binding_owner}\")] | length), ", + "worker: ([$p[0].resources[] | select(.type == \"Process\" and ", + ".metadata.ownerRef == \"{binding_owner}\")] | length), ", + "binding: ([$b[0].resources[] | select(.type == \"VolumeBinding\" ", + "and .metadata.name == \"{binding_name}\")] | length), ", + "socket: $socket", + "}}'", + ), + socket_path = SOCKET_PATH, + binding_owner = binding_owner, + binding_name = BINDING_NAME, + ) +} + +/// The post-delete read, the fixture's own command text: no Volume row named +/// `state` is left. +fn volume_after_delete() -> String { + format!( + concat!( + "{list_volume} && ", + "jq -e 'all(.resources[]; ", + "(.type == \"Volume\" and .metadata.name == \"state\") | not)' ", + "/run/d2b-volume-after-delete.json", + ), + list_volume = d2b("list Volume", "/run/d2b-volume-after-delete.json"), + ) +} + +/// The row set every volume-chain wait asserts on, the fixture's own +/// `chain_row_dumps`: one labelled projection per resource type the chain +/// waits read, plus the directory the worker's serving socket is minted in. +fn chain_row_dumps() -> Vec<(String, String)> { + let mut dumps = ["Volume", "VolumeBinding", "Process", "Endpoint"] + .into_iter() + .map(|kind| { + let path = format!("/run/d2b-diag-{}.json", kind.to_lowercase()); + ( + format!("{kind} rows"), + format!( + "{} && jq -c '{CHAIN_PROJECTION}' {path} || true", + d2b(&format!("list {kind}"), &path) + ), + ) + }) + .collect::>(); + dumps.push(( + "vms dir".to_owned(), + "ls -la /run/d2b/vms/acceptance-guest/ 2>&1 || echo 'no vms dir'".to_owned(), + )); + dumps +} + +/// The labelled pair as the diagnostics rows are passed. +fn rows(dumps: &[(String, String)]) -> Vec> { + dumps + .iter() + .map(|(label, command)| (label.as_str(), command.as_str())) + .collect() +} + +/// The fixture's own `dump_rows`: announce the tag the failure is reported +/// under, and run every row dump under it. +fn dump_rows(control: &mut GuestControl, tag: &str, dumps: &[(String, String)]) { + control.stage(tag); + for (label, command) in dumps { + control.diag(command, &format!("{tag}: {label}")); + } +} + +/// One sample of the teardown window: what each of the three lists held, and +/// whether the serving socket was still bound when they were read. +struct Sample { + endpoint: u64, + worker: u64, + binding: u64, + socket: bool, +} + +impl Sample { + /// Whether the chain is fully torn down in this sample. + fn is_torn_down(&self) -> bool { + self.binding == 0 && self.endpoint == 0 && self.worker == 0 + } + + /// The sample as the fixture's own `assert` messages rendered it: a + /// Python dict, its keys in the order the jq program built them. + fn as_python_dict(&self) -> String { + format!( + "{{'endpoint': {}, 'worker': {}, 'binding': {}, 'socket': {}}}", + self.endpoint, + self.worker, + self.binding, + if self.socket { "True" } else { "False" }, + ) + } + + /// The sample as the fixture's own `json.dumps` rendered it for the + /// failure that reports a teardown which did not converge. + fn as_json(&self) -> String { + format!( + "{{\"endpoint\": {}, \"worker\": {}, \"binding\": {}, \"socket\": {}}}", + self.endpoint, + self.worker, + self.binding, + if self.socket { "true" } else { "false" }, + ) + } +} + +/// Read one sample out of what the sample command printed, the way the +/// fixture's own `json.loads` read it. +fn read_sample(output: &str) -> LegacyResult { + let value: Value = serde_json::from_str(output.trim()).map_err(|error| { + LegacyError::Assertion(format!( + "the teardown sample did not read as JSON: {error}: {}", + output.trim() + )) + })?; + let count = |name: &str| -> LegacyResult { + value.get(name).and_then(Value::as_u64).ok_or_else(|| { + LegacyError::Assertion(format!( + "the teardown sample carried no {name} count: {value}" + )) + }) + }; + let socket = value + .get("socket") + .and_then(Value::as_bool) + .ok_or_else(|| { + LegacyError::Assertion(format!("the teardown sample carried no socket flag: {value}")) + })?; + Ok(Sample { + endpoint: count("endpoint")?, + worker: count("worker")?, + binding: count("binding")?, + socket, + }) +} + +/// The wait between two samples of the teardown window, the fixture's own +/// `time.sleep(0.2)`: it ran on the driver, which is where a check's +/// assertions run, so it is a sleep on this thread rather than a command in +/// the guest. +#[allow(clippy::disallowed_methods, reason = "synchronous path")] +fn sleep_between_samples() { + thread::sleep(SAMPLE_INTERVAL); +} diff --git a/tests/host-integration/virtiofsd-volume-runtime.nix b/tests/host-integration/virtiofsd-volume-runtime.nix deleted file mode 100644 index c188d9da5..000000000 --- a/tests/host-integration/virtiofsd-volume-runtime.nix +++ /dev/null @@ -1,533 +0,0 @@ -# Type-G runNixOSTest: virtiofsd Volume slice with owned children on the v3 -# resource runtime. -# -# This is the U11 midpoint Volume proof the plan names for the new (v3) -# resource plane, separate from the Process-slice operator-activation fixture: -# the Nix-ingested Volume derives one deterministic VolumeBinding child -# through the manager, the binding owns the virtiofsd worker Process and its -# private Endpoint, the worker is realized and binds its private serving -# socket, and deleting the Volume tears the chain down endpoint-first -# (R9/F3). All assertions read observable resource identity, phase, -# generation, processes, and sockets through the installed d2b CLI and the -# host filesystem; nothing keys on informational log text. -{ pkgs, self }: - -let - inherit (pkgs) lib; - d2bLib = import ./lib.nix { - inherit self; - inherit lib; - hostToolBundle = - if self.lib ? d2bHostToolBundle then self.lib.d2bHostToolBundle else null; - }; - # The reusable guest configuration lives outside this directory so it - # survives the fixture (see `nix/test-support/host-integration-node.nix`). - d2bNode = import ../../nix/test-support/host-integration-node.nix { - inherit self; - inherit lib; - }; - volumeProviderArtifact = d2bLib.mkVolumeProviderArtifact pkgs; - artifacts = { - volume-acceptance-provider = { - inherit (volumeProviderArtifact) package type catalog; - }; - }; - hostRuntime = pkgs.writeText "d2b-acceptance-host-runtime.json" (builtins.toJSON { - schemaVersion = "v1"; - bundleVersion = 1; - generatedAt = "1970-01-01T00:00:00.000Z"; - nftAppliedHash = null; - ifnames = [ ]; - }); -in -pkgs.testers.runNixOSTest { - name = "d2b-virtiofsd-volume-runtime"; - - nodes.machine = d2bNode.d2bDaemonNode { - extra = { ... }: { - networking.nftables.enable = true; - networking.nftables.ruleset = lib.mkAfter '' - table inet d2b {} - ''; - systemd.tmpfiles.rules = [ - "d /etc/NetworkManager/conf.d 0755 root root -" - ]; - environment.etc."d2b/acceptance-host-runtime.json".source = hostRuntime; - d2b.site.adminUsers = [ "alice" ]; - systemd.services.d2bd.serviceConfig.ExecStartPre = lib.mkAfter [ - "+${pkgs.writeShellScript "d2b-acceptance-hosts-prep" '' - if [ -L /etc/hosts ]; then - ${pkgs.coreutils}/bin/cat /etc/hosts > /run/d2b-acceptance-hosts - ${pkgs.coreutils}/bin/rm -f /etc/hosts - ${pkgs.coreutils}/bin/install -o root -g root -m 0644 \ - /run/d2b-acceptance-hosts /etc/hosts - fi - ''}" - "+${pkgs.writeShellScript "d2b-acceptance-host-runtime-prep" '' - ${pkgs.coreutils}/bin/install -D -o root -g d2bd -m 0640 \ - /etc/d2b/acceptance-host-runtime.json \ - /var/lib/d2b/runtime/host-runtime.json - ''}" - ]; - d2b.artifacts = artifacts; - d2b.zones.local-root.trustedPublishers.d2b-volume-acceptance.signingKey = - volumeProviderArtifact.trustedPublisher.signingKey; - d2b.zones.work.parentZone = "local-root"; - d2b.zones.work.trustedPublishers.d2b-volume-acceptance.signingKey = - volumeProviderArtifact.trustedPublisher.signingKey; - d2b.zones.work.resources = { - alice = { - type = "User"; - spec = { - displayName = "Alice"; - groups = [ ]; - osUsername = "alice"; - }; - }; - d2bd = { - type = "User"; - spec = { - displayName = "d2bd"; - groups = [ ]; - osUsername = "d2bd"; - }; - }; - volume-operator = { - type = "Role"; - spec.rules = [ - { - resourceTypes = [ - "Endpoint" - "Host" - "Process" - "Provider" - "Volume" - "VolumeBinding" - ]; - verbs = [ "get" "list" ]; - subresources = [ ]; - resourceNames = [ ]; - zones = [ "work" ]; - executionRefs = [ ]; - sessionVerbs = [ "connect" "invoke" ]; - } - { - resourceTypes = [ "Volume" ]; - verbs = [ "delete" ]; - subresources = [ ]; - resourceNames = [ "state" ]; - zones = [ "work" ]; - executionRefs = [ ]; - sessionVerbs = [ "connect" "invoke" ]; - } - ]; - }; - volume-operator-binding = { - type = "RoleBinding"; - spec = { - roleRef = "Role/volume-operator"; - subjects = [ "User/alice" ]; - externalPrincipalSelector = null; - scopeNarrowing = null; - }; - }; - # The attachment execution target. The Nix bundle validation requires - # attachment refs to resolve to a same-Zone Host or Guest; the - # virtiofs serving path is host-side (the binding mints its socket - # under /run/d2b/vms//), so this fixture asserts the Volume - # chain only - the Guest row itself stays declared input (KTD1). - acceptance-guest = { - type = "Guest"; - spec = { - defaultDomain = "system"; - providerRef = "Provider/volume-virtiofs"; - budget = { }; - networkAttachments = [ ]; - deviceAttachments = [ ]; - volumeAttachmentDefaults = [ ]; - }; - }; - host-system = { - type = "Host"; - spec = { - providerRef = "Provider/system-core"; - defaultDomain = "system"; - allowedDomains = [ "system" ]; - budget = { }; - networkAttachments = [ ]; - deviceAttachments = [ ]; - volumeAttachmentDefaults = [ ]; - }; - }; - # The fixed daemon-owned Volume owner: volume-local is the only - # Provider a Volume may select (U7 driver contract). - volume-local = { - type = "Provider"; - spec = { - artifactId = "volume-acceptance-provider"; - config = { - controllerExecutionRef = "Host/host-system"; - sourcePolicies = [ - { - id = "daemon-state"; - class = "local-path"; - volumeKinds = [ "durable" "state" "cache" ]; - } - ]; - }; - }; - }; - # The serving Provider the derived VolumeBinding rows select and - # whose signed virtiofsd-worker template the worker launch resolves. - volume-virtiofs = { - type = "Provider"; - spec = { - artifactId = "volume-acceptance-provider"; - config.controllerExecutionRef = "Host/host-system"; - }; - }; - state = { - type = "Volume"; - spec = { - providerRef = "Provider/volume-local"; - kind = "state"; - source = { - executionRef = "Host/host-system"; - settings = { - kind = "local-path"; - sourcePolicyId = "daemon-state"; - }; - }; - layout = [{ - path = "state"; - type = "directory"; - # Daemon-owned so the unprivileged daemon can provision the - # local-path layout inline. - ownerRef = "User/d2bd"; - groupRef = "User/d2bd"; - mode = "0700"; - target = null; - accessAcl = [ ]; - defaultAcl = [ ]; - foreignChildPolicy = "preserve"; - noFollow = true; - recursive = false; - sensitivity = "private"; - createPolicy = "create-if-never-provisioned"; - repairPolicy = "exact-owner"; - cleanupPolicy = "owner-controlled"; - adoptionPolicy = "quarantine-on-ambiguity"; - restartPolicy = "preserve-across-controller-restart"; - leaseClass = "none"; - invariants = [ "no-symlink" ]; - }]; - views.controller = { - path = ""; - rights = [ "read" "write" "traverse" ]; - }; - # KTD1: the attachment stays declared input only. The Volume side - # mints the durable VolumeBinding at reconcile; the deterministic - # binding identity is derived from (volume, execution target, - # view, mount path) - the fixture asserts that exact identity. - attachments = [{ - executionRef = "Guest/acceptance-guest"; - transport = "virtiofs"; - view = "controller"; - access = "read-only"; - mountPath = "/state"; - settings = { - posixAcl = false; - xattr = false; - cache = "auto"; - inodeFileHandles = "never"; - threadPoolSize = null; - socketGroup = null; - }; - }]; - }; - }; - }; - environment.systemPackages = with pkgs; [ - jq - procps - ]; - }; - }; - - testScript = '' - ${d2bLib.fixtureDiagnostics} - - import hashlib - import json - import time - - # Deterministic identities the frozen v1 derivation pins (the fixture - # asserts them, it does not invent them): - # - the binding name derives from the attachment tuple; - # - the private socket path derives from (zone, volume, guest) exactly as - # the daemon's serving effects resolve it. - binding_name = "vol-binding-" + hashlib.sha256( - b"d2b/volume-local/binding/v1\x00Volume/state" - b"\x00Guest/acceptance-guest\x00controller\x00/state" - ).hexdigest()[:24] - socket_tag = hashlib.sha256(b"work\x00state\x00acceptance-guest").hexdigest()[:8] - socket_path = f"/run/d2b/vms/acceptance-guest/vol-{socket_tag}.vfd.sock" - - # The API serves real row identities. A uid is not just non-null: the - # daemon's `ResourceUid` Display is a redaction placeholder, and a - # placeholder served through the read path would satisfy a null check - # while breaking every consumer that parses the field (the operator - # delete resolves its exact precondition uid from it). - uuid_v4 = r"^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$" - - def d2b(command, out): - return ( - "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - f"d2b --zone work --json {command} >{out}" - ) - - # The row set every volume-chain wait asserts on. On timeout the shared - # `diag_wait` dumps print exactly these projections (plus the daemon lines - # mentioning the binding) into the driver log, so the lane log carries the - # evidence the wait last saw (issue #513). - chain_projection = ( - "[.resources[] | {type: .type, name: .metadata.name, " - "owner: .metadata.ownerRef, phase: .status.phase, " - "gen: .metadata.generation, obs: .status.observedGeneration, " - "template: .spec.template, purpose: .spec.purpose, " - "producer: .spec.producerRef}]" - ) - chain_row_dumps = [ - ( - f"{kind} rows", - d2b(f"list {kind}", f"/run/d2b-diag-{kind.lower()}.json") - + f" && jq -c '{chain_projection}' " - + f"/run/d2b-diag-{kind.lower()}.json || true", - ) - for kind in ["Volume", "VolumeBinding", "Process", "Endpoint"] - ] + [ - ( - "vms dir", - "ls -la /run/d2b/vms/acceptance-guest/ 2>&1 " - "|| echo 'no vms dir'", - ), - ] - - start_all() - stage("boot") - machine.wait_for_unit("nftables.service", timeout=180) - machine.wait_for_unit("d2b-broker.socket", timeout=30) - diag_unit("daemon-up", "d2bd.service", 180) - machine.wait_for_file("/run/d2b/public.sock", timeout=30) - - # 1. Volume realize: the Nix-ingested Volume is Ready with a manager - # identity and an observed generation (U10 ingestion -> U7 driver). - diag_wait( - "volume-realized", - f"{d2b('list Volume', '/run/d2b-volume-realized.json')} && " - "jq -e '" - "([.resources[] | select(.type == \"Volume\" and " - ".metadata.name == \"state\")] | length) == 1 and " - "(.resources[] | select(.type == \"Volume\" and " - ".metadata.name == \"state\") | " - f"((.metadata.uid | test(\"{uuid_v4}\")) and " - ".metadata.generation > 0 and " - ".metadata.ownerRef == null and " - ".status.phase == \"Ready\" and " - ".status.observedGeneration == .metadata.generation))' " - "/run/d2b-volume-realized.json", - timeout=180, - rows=chain_row_dumps, - explain=[("d2bd.service", "Volume/state")], - ) - - def dump_rows(tag): - stage(tag) - for label, command in chain_row_dumps: - diag(command, f"{tag}: {label}") - - # 2. The Volume side minted exactly one deterministic VolumeBinding - # child through the manager, and that child converged. - diag_wait( - "binding-realized", - f"{d2b('list VolumeBinding', '/run/d2b-binding-realized.json')} && " - "jq -e '" - f"([.resources[] | select(.type == \"VolumeBinding\" and " - f".metadata.name == \"{binding_name}\" and " - ".metadata.ownerRef == \"Volume/state\")] | length) == 1 and " - f"(.resources[] | select(.metadata.name == \"{binding_name}\") | " - f"((.metadata.uid | test(\"{uuid_v4}\")) and " - ".metadata.generation > 0 and " - ".status.phase == \"Ready\" and " - ".status.observedGeneration == .metadata.generation and " - ".spec.volumeRef == \"Volume/state\" and " - ".spec.executionRef == \"Guest/acceptance-guest\" and " - ".spec.view == \"controller\" and " - ".spec.access == \"read-only\" and " - ".spec.mountPath == \"/state\"))' " - "/run/d2b-binding-realized.json", - timeout=120, - rows=chain_row_dumps, - explain=[("d2bd.service", binding_name)], - ) - - # 3. The binding owns the virtiofsd worker Process and its private - # Endpoint as managed children; the worker is realized (a live - # virtiofsd process) and the serving socket is bound. - diag_wait( - "worker-realized", - f"{d2b('list Process', '/run/d2b-worker-realized.json')} && " - "jq -e '" - f"([.resources[] | select(.type == \"Process\" and " - f".metadata.ownerRef == \"VolumeBinding/{binding_name}\")] | length) " - "== 1 and " - f"([.resources[] | select(.type == \"Process\" and " - f".metadata.ownerRef == \"VolumeBinding/{binding_name}\") | " - "(.spec.providerRef == \"Provider/system-minijail\" and " - ".spec.executionRef == \"Host/host-system\" and " - ".spec.processClass == \"worker\" and " - ".spec.template == \"virtiofsd-worker\" and " - ".status.phase == \"Ready\" and " - ".status.observedGeneration == .metadata.generation)] | length) == 1' " - "/run/d2b-worker-realized.json", - timeout=120, - rows=chain_row_dumps, - explain=[("d2bd.service", "virtiofsd-worker")], - ) - diag_wait( - "endpoint-realized", - f"{d2b('list Endpoint', '/run/d2b-endpoint-realized.json')} && " - f"{d2b('list Process', '/run/d2b-worker-producer.json')} && " - "jq -e --slurpfile proc /run/d2b-worker-producer.json '" - f"([.resources[] | select(.type == \"Endpoint\" and " - f".metadata.ownerRef == \"VolumeBinding/{binding_name}\")] | length) " - "== 1 and " - f"([$proc[0].resources[] | select(.type == \"Process\" and " - f".metadata.ownerRef == \"VolumeBinding/{binding_name}\" and " - ".status.phase == \"Ready\" and " - ".status.observedGeneration == .metadata.generation)] | length) " - "== 1 and " - f"(.resources[] | select(.type == \"Endpoint\" and " - f".metadata.ownerRef == \"VolumeBinding/{binding_name}\") | " - "(.status.phase == \"Ready\" and " - ".status.observedGeneration == .metadata.generation and " - ".spec.transport == \"unix\" and " - "(.spec.purpose == \"virtiofsd\" and " - "(.spec.producerRef as $producer | " - "any($proc[0].resources[]; " - ".type == \"Process\" and " - "\"\\(.type)/\\(.metadata.name)\" == $producer)))))' " - "/run/d2b-endpoint-realized.json", - timeout=120, - rows=chain_row_dumps, - explain=[("d2bd.service", binding_name)], - ) - diag_wait( - "serving-socket", - f"test -S {socket_path}", - timeout=60, - rows=chain_row_dumps, - explain=[("d2bd.service", "virtiofsd")], - ) - diag_wait( - "virtiofsd-process", - "test \"$(ps -eo args= | awk '/virtiofsd/ && !/awk/ {c++} END {print c+0}')\" " - "-ge 1", - timeout=60, - rows=chain_row_dumps, - explain=[("d2bd.service", "virtiofsd")], - ) - - # 4. Deleting the owning Volume drives the whole chain through the - # preserved endpoint-first teardown: the binding is marked deleting - # first, its private Endpoint and socket are removed before the - # worker Process row, and nothing owned survives. - machine.succeed(f"{d2b('list Volume', '/run/d2b-volume-pre-delete.json')}") - volume_revision = machine.succeed( - "jq -er '.resources[] | select(.type == \"Volume\" and " - ".metadata.name == \"state\") | .metadata.revision' " - "/run/d2b-volume-pre-delete.json" - ).strip() - machine.succeed( - d2b("delete Volume/state --revision " + volume_revision, - "/run/d2b-volume-delete.json") - ) - - diag_wait( - "binding-deleting", - f"{d2b('list VolumeBinding', '/run/d2b-binding-deleting.json')} && " - "jq -e '" - f"any(.resources[]; .type == \"VolumeBinding\" and " - f".metadata.name == \"{binding_name}\" and " - ".metadata.deletionRequestedAt != null)' " - "/run/d2b-binding-deleting.json", - timeout=60, - rows=chain_row_dumps, - explain=[("d2bd.service", binding_name)], - ) - - # Sample the teardown window: the Endpoint row and the worker Process - # row must both disappear, the Endpoint never after the worker, and no - # owned row may outlive its parent. The three lists are separate reads, - # so they run parent -> worker -> endpoint: a parent observed gone ahead - # of a child then really means the child was already gone when the - # parent retired (the invariant under test), while the reverse order - # could straddle the teardown and report a violation that never held. - binding_owner = "VolumeBinding/" + binding_name - sample_expr = ( - "socket=false; test -S " + socket_path + " && socket=true; " - "jq -n --argjson socket \"$socket\" " - "--slurpfile b /run/d2b-teardown-binding.json " - "--slurpfile p /run/d2b-teardown-process.json " - "--slurpfile e /run/d2b-teardown-endpoint.json " - "'{" - "endpoint: ([$e[0].resources[] | " - "select(.metadata.ownerRef == \"" + binding_owner + "\")] | length), " - "worker: ([$p[0].resources[] | select(.type == \"Process\" and " - ".metadata.ownerRef == \"" + binding_owner + "\")] | length), " - "binding: ([$b[0].resources[] | select(.type == \"VolumeBinding\" " - "and .metadata.name == \"" + binding_name + "\")] | length), " - "socket: $socket" - "}'" - ) - observed = [] - for _ in range(600): - machine.succeed( - d2b("list VolumeBinding", "/run/d2b-teardown-binding.json") + "; " - + d2b("list Process", "/run/d2b-teardown-process.json") + "; " - + d2b("list Endpoint", "/run/d2b-teardown-endpoint.json") + "; " - + "echo OK" - ) - observed.append( - json.loads(machine.succeed(sample_expr)) - ) - last = observed[-1] - if last["binding"] == 0 and last["endpoint"] == 0 and last["worker"] == 0: - break - time.sleep(0.2) - else: - dump_rows("teardown did not converge") - raise AssertionError( - "volume teardown did not converge within its budget: " - + json.dumps(observed[-1]) - ) - - for sample in observed: - if sample["binding"] == 0: - assert sample["endpoint"] == 0 and sample["worker"] == 0, ( - f"owned child outlived its parent binding: {sample}" - ) - if sample["worker"] == 0: - assert sample["endpoint"] == 0, ( - f"worker Process row disappeared before the binding-owned " - f"Endpoint row (endpoint-first teardown violated): {sample}" - ) - - machine.succeed( - f"{d2b('list Volume', '/run/d2b-volume-after-delete.json')} && " - "jq -e 'all(.resources[]; " - "(.type == \"Volume\" and .metadata.name == \"state\") | not)' " - "/run/d2b-volume-after-delete.json" - ) - ''; -} From 520903f0b88f07723e406b7d5c2fcfc322067182 Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sat, 26 Sep 2026 22:56:19 -0700 Subject: [PATCH 23/51] fix(vm): repair two assertions found by differential verification An independent op-stream verification of the seven daemon-shape ports - the fixture's own evaluated testScript run against recording stubs, against the ported Rust run against a recording guest-control surface, comparing every command, bound, order, row label, explain pair and failure message - found two defects in the ports. guest-agent-cap-confinement: the host-namespace capability scan's shell text was transcribed with two quote pairs the fixture did not have. The fixture's command is `pid=${status#/proc/}; pid=${pid%/status};`, because the `''$` in its indented string is the escape for `$` rather than a quote. The ported text read `pid='${status#/proc/}'`, which the shell leaves as a literal rather than expanding, so every row was skipped, both capability tables came back empty, and the two aggregate assertions - no capability gained by a host-namespace process, none left behind by the agent's service - passed vacuously. The quotes are gone; the command is the fixture's. guest-shell-service: the listener wait's row carried the unit's name as its label. The prelude's `unit_dumps` labels that dump ` status`, which is what the port asserts now, so a failure report names the row the way the fixture's failure named it. Diagnostic-only, but the port's whole point is that a failure reads the way it read. Both checks' assertion counts are unchanged: 5 stage, 1 wait_for_unit, 1 diag_unit, 2 succeed, 12 assertions for the first, and 1 stage, 1 wait_for_unit, 1 diag_unit, 1 diag_wait, 1 succeed, 1 fail for the second, before and after. --- .../src/checks/guest_agent_cap_confinement.rs | 2 +- .../d2b-test-vm-harness/src/checks/guest_shell_service.rs | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/d2b-test-vm-harness/src/checks/guest_agent_cap_confinement.rs b/packages/d2b-test-vm-harness/src/checks/guest_agent_cap_confinement.rs index 922d79c47..cbc232c86 100644 --- a/packages/d2b-test-vm-harness/src/checks/guest_agent_cap_confinement.rs +++ b/packages/d2b-test-vm-harness/src/checks/guest_agent_cap_confinement.rs @@ -47,7 +47,7 @@ const AGENT_UP: Duration = Duration::from_secs(60); const HOST_NAMESPACE_CAPABILITIES: &str = concat!( "host_ns=$(readlink /proc/1/ns/net); ", "for status in /proc/[0-9]*/status; do ", - "pid='${status#/proc/}'; pid='${pid%/status}'; ", + "pid=${status#/proc/}; pid=${pid%/status}; ", "ns=$(readlink /proc/$pid/ns/net 2>/dev/null) || continue; ", "test \"$ns\" = \"$host_ns\" || continue; ", "cap=$(while IFS=: read -r key value; do ", diff --git a/packages/d2b-test-vm-harness/src/checks/guest_shell_service.rs b/packages/d2b-test-vm-harness/src/checks/guest_shell_service.rs index 13c89e3e3..43e1d0276 100644 --- a/packages/d2b-test-vm-harness/src/checks/guest_shell_service.rs +++ b/packages/d2b-test-vm-harness/src/checks/guest_shell_service.rs @@ -41,6 +41,10 @@ const LISTENER_BOUND: Duration = Duration::from_secs(60); /// The unit whose journal both greps read. const GUEST_DAEMON_UNIT: &str = "d2bd-guest.service"; +/// The label the row is reported under, the fixture's own: the prelude's +/// `unit_dumps` labels its dump `" status"`. +const GUEST_DAEMON_STATUS_LABEL: &str = "d2bd-guest.service status"; + /// The row the listener wait explains itself with, the fixture's own: the /// unit's status, dumped the way the prelude's `unit_dumps` dumped it. const GUEST_DAEMON_STATUS: &str = "systemctl status d2bd-guest.service --no-pager 2>&1 | tail -n 40 || true"; @@ -63,7 +67,7 @@ pub fn assertions(control: &mut GuestControl) -> LegacyResult<()> { control.diag_unit("guest-daemon", GUEST_DAEMON_UNIT, GUEST_DAEMON)?; control.succeed(&["systemctl is-active --quiet d2bd-guest.service"], None)?; - let rows: [DiagRow<'_>; 1] = [(GUEST_DAEMON_UNIT, GUEST_DAEMON_STATUS)]; + let rows: [DiagRow<'_>; 1] = [(GUEST_DAEMON_STATUS_LABEL, GUEST_DAEMON_STATUS)]; let explain = [("d2bd-guest.service", "")]; control.diag_wait( "guest-listener-bound", From 9a3ce03924ba320761164caa78d6246663904d43 Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sat, 26 Sep 2026 23:04:06 -0700 Subject: [PATCH 24/51] refactor(vm): make the host lane a single Bazel test and honour --test_filter Two defects in how the shipped lane is reached. `make test-host-integration` was still the nix recipe: it evaluated `getFlake(...).vmChecks` to discover check names, ran an Attic cache preflight and closure upload, staged a host-tool bundle for the flake to inject, and built each vmCheck with `nix build` under sudo - and only then built the host tools with Bazel. The recipe's own comment said it "retires with the nix lane", and it had not. It is now one `bazel test` invocation of the lane target under the committed `guest` profile, matching the shape the `perf` target already uses. The Attic preflight and closure upload go with it. The guest-image action declares its own substituters and preflights them itself, so the cache handling now lives with the build that needs it instead of in a second place that can drift out of step with it. `D2B_HOST_VM_CHECK` and `D2B_HOST_VM_JOBS` existed only inside the retired recipe and are gone; `D2B_VM_CHECK` still selects one named check. The x86_64-linux skip stays, reworded now that there are no vmChecks to skip, and a missing `/dev/kvm` now stops with a message instead of a note about a TCG fallback the lane has never had - R7 makes virtualization a precondition, and the recipe should say so rather than imply a slow path exists. The lane ignored Bazel's own `--test_filter`. `selection` read the target's `--check` argument and `D2B_VM_CHECK` and nothing else, so `bazel test --test_filter=` looked like it selected one check and quietly ran all eleven - a pool of guests, and a verdict for a check nobody asked about. It now reads `TESTBRIDGE_TEST_ONLY` as a third source, after the two existing ones, and the first that names anything wins so an explicit request is never widened by a default that happens to be set. The two contributor documents that described the handoff are rewritten to describe the shipped lane: the declared-input guest build, the restored pool, the virtualization precondition, and the loss of the emulation fallback. --- Makefile | 217 +++--------------- docs/contributing/gates-and-lints.md | 40 ++-- .../src/bin/d2b-test-vm-harness.rs | 28 ++- tests/README.md | 33 ++- 4 files changed, 105 insertions(+), 213 deletions(-) diff --git a/Makefile b/Makefile index da71d998f..55c67446d 100644 --- a/Makefile +++ b/Makefile @@ -104,7 +104,7 @@ SYSTEM ?= $(shell nix eval --extra-experimental-features 'nix-command flakes' \ # make check-ci check + test-integration for local/manual compatibility. # make test- focused Bazel suite. # make test-integration type-9 container integration; local host/manual pre-PR. -# make test-host-integration type-10 runNixOSTest; local NixOS/KVM pre-PR. +# make test-host-integration type-10 Bazel host lane; local NixOS/KVM pre-PR. # make heavy-check full Layer-1 check. # make heavy-flake-check full flake realization. # =========================================================================== @@ -175,196 +175,49 @@ generate: # Additional targets (helper utilities, legacy aliases, meta gates). # =========================================================================== -## test-host-integration - G-host: runNixOSTest VM integration tests (the -## `vmChecks` flake output, NOT swept by `nix flake check`). Each test boots a -## real NixOS VM with the d2b daemon surface and asserts live broker / -## daemon / host-posture behaviour (socket activation, bridge isolation, -## state-dir ACLs, broker privilege posture) - the hermetic, non-destructive -## successor to the `D2B_LIVE`-against-the-real-host scripts. Needs KVM (a local -## NixOS host; TCG software emulation is the slow fallback when /dev/kvm is -## absent). x86_64-linux only (a same-system VM builder is required). -## Set D2B_VM_CHECK= to build one named vmChecks entry. -## The host tools are built under the committed `guest` profile from -## .bazelrc, so an exported Bazel profile cannot change the guest closure. -## The Attic cache preflight and closure upload below belong to this nix -## recipe: the Bazel-owned lane's guest-image action declares its own -## substituters and preflights them itself, and the recipe retires with -## the nix lane. +## test-host-integration - G-host: the Bazel-owned host integration lane. +## +## Runs the eleven VM integration checks through the lane test target. Each +## check boots its own NixOS guest with the d2b daemon surface and asserts +## live broker / daemon / host-posture behaviour (socket activation, bridge +## isolation, state-dir ACLs, broker privilege posture) - the hermetic, +## non-destructive successor to the `D2B_LIVE`-against-the-real-host +## scripts. Every assertion is Rust. The guest images are graph outputs +## keyed on declared inputs, and the lane restores a pooled guest per check +## rather than booting one guest per check. +## +## This is a local, contributor-run lane and not a CI gate: it is a +## pre-PR surface, which is what lets the virtualization precondition be +## asserted rather than negotiated. +## +## Needs KVM. The lane declares virtualization as a precondition and has no +## silent emulation fallback, so on a host without it this target stops with +## a message rather than turning very slow. x86_64-linux only (it needs a +## same-system VM builder). +## +## Set D2B_VM_CHECK= to run one named check. `bazel test +## --test_filter=` works too: the lane reads Bazel's own filter as +## well as this variable, and the first of the two that names anything wins. +## +## The host tools and the guest are built under the committed `guest` profile +## from .bazelrc, so an exported Bazel profile cannot change the guest +## closure. The guest-image action declares its own substituters and +## preflights them itself, so the Attic preflight and closure upload that +## used to live here are gone with the nix recipe that needed them. test-host-integration: @set -eu; \ system="$$(nix eval --raw --impure --expr builtins.currentSystem)"; \ if [ "$$system" != "x86_64-linux" ]; then \ - echo "test-host-integration: vmChecks are x86_64-linux only (need a same-system VM builder); skipping on $$system"; \ + echo "test-host-integration: the lane is x86_64-linux only (it needs a same-system VM builder); skipping on $$system"; \ exit 0; \ fi; \ if [ ! -e /dev/kvm ]; then \ - echo "test-host-integration: /dev/kvm absent - runNixOSTest will fall back to slow TCG emulation"; \ - fi; \ - root="$$(pwd)"; \ - if [ -n "$${D2B_VM_CHECK:-}" ]; then \ - names="$$D2B_VM_CHECK"; \ - else \ - names="$$(nix eval --raw --impure --no-warn-dirty --expr "builtins.concatStringsSep \" \" (builtins.attrNames (builtins.getFlake \"git+file://$$root\").vmChecks.$$system)")"; \ - fi; \ - requested="$${D2B_HOST_VM_CHECK:-}"; \ - if [ -n "$$requested" ]; then \ - case "$$requested" in \ - *[!A-Za-z0-9._-]*) \ - echo "test-host-integration: invalid D2B_HOST_VM_CHECK (use one discovered vmCheck name): $$requested" >&2; \ - exit 1;; \ - esac; \ - fi; \ - if [ -z "$$names" ]; then \ - if [ -n "$$requested" ]; then \ - echo "test-host-integration: unknown vmCheck '$$requested' (available: none)" >&2; \ - exit 1; \ - fi; \ - echo "test-host-integration: no vmChecks present"; \ - exit 0; \ - fi; \ - if [ -n "$$requested" ]; then \ - case " $$names " in \ - *" $$requested "*) names="$$requested";; \ - *) \ - echo "test-host-integration: unknown vmCheck '$$requested' (available: $$names)" >&2; \ - exit 1;; \ - esac; \ - fi; \ - run_dir="$$(mktemp -d "$${TMPDIR:-/tmp}/d2b-host-integration.XXXXXX")"; \ - chmod 700 "$$run_dir"; \ - cleanup() { rm -rf -- "$$run_dir"; nix-store --gc --print-roots >/dev/null 2>&1 || true; }; \ - trap cleanup EXIT; \ - trap 'exit 129' HUP; \ - trap 'exit 130' INT; \ - trap 'exit 143' TERM; \ - trap 'exit 131' QUIT; \ - attic_cache=""; \ - attic_config=""; \ - if [ -n "$${XDG_CONFIG_HOME:-}" ]; then \ - attic_config="$$XDG_CONFIG_HOME/attic/config.toml"; \ - elif [ -n "$${HOME:-}" ]; then \ - attic_config="$$HOME/.config/attic/config.toml"; \ - fi; \ - if ! command -v attic >/dev/null 2>&1; then \ - echo "test-host-integration: Attic unavailable - skipping closure upload"; \ - elif [ -z "$$attic_config" ] || [ ! -e "$$attic_config" ]; then \ - echo "test-host-integration: Attic config absent - skipping closure upload"; \ - else \ - fail_attic_state() { echo "test-host-integration: configured Attic state is invalid or ambiguous" >&2; exit 1; }; \ - attic_meta="$$(ATTIC_CONFIG="$$attic_config" nix eval --impure --json --expr 'let config = builtins.fromTOML (builtins.readFile (builtins.getEnv "ATTIC_CONFIG")); names = builtins.attrNames (config.servers or {}); server = if config ? "default-server" then config."default-server" else if builtins.length names == 1 then builtins.head names else throw "ambiguous Attic servers"; endpoint = config.servers.$${server}.endpoint or (throw "missing Attic endpoint"); in { inherit server endpoint; }' 2>/dev/null)" || fail_attic_state; \ - attic_server="$$(printf '%s' "$$attic_meta" | jq -er '.server | select(test("^[A-Za-z0-9][A-Za-z0-9._+-]*$$"))')" || fail_attic_state; \ - attic_base="$$(printf '%s' "$$attic_meta" | jq -er '.endpoint | capture("^(?https?)://(?[^/@?#]+)(?:/[^?#]*)?$$") | ((.scheme | ascii_downcase) + "://" + (.authority | ascii_downcase))')" || fail_attic_state; \ - attic_name="$$(nix config show --json | jq -er --arg base "$$attic_base" '.substituters.value | if type == "string" then split(" ") else . end | map(try capture("^(?https?)://(?[^/@?#]+)(?/[^?#]*)?(?:\\?[^#]*)?$$") catch empty | select(((.scheme | ascii_downcase) + "://" + (.authority | ascii_downcase)) == $$base) | ((.path // "") | rtrimstr("/") | split("/") | last)) | map(select(test("^[A-Za-z0-9][A-Za-z0-9_+-]*$$"))) | unique | select(length == 1) | .[0]')" || fail_attic_state; \ - attic_cache="$$attic_server:$$attic_name"; \ - if ! attic cache info "$$attic_cache" >"$$run_dir/attic-info.log" 2>&1; then \ - echo "test-host-integration: configured Attic cache preflight failed" >&2; \ + echo "test-host-integration: /dev/kvm is absent, and the lane declares virtualization as a precondition with no emulation fallback" >&2; \ exit 1; \ fi; \ - echo "test-host-integration: Attic cache preflight passed"; \ - fi; \ - echo "test-host-integration: building host tools under the committed guest profile"; \ - '$(BAZEL_BIN)' build --config=guest \ - //packages/d2b:d2b \ - //packages/d2bd:d2bd \ - //packages/d2b-broker-composition:d2b-broker \ - //packages/d2b-host:d2b-activation-helper \ - //packages/d2b-host-activation-helper:d2b-host-activation-helper \ - //packages/d2b-unsafe-local-helper:d2b-unsafe-local-helper \ - //packages/d2b-resource-compiler:d2b-resource-compiler \ - //packages/d2b-provider-display-wayland:d2b-wayland-proxy \ - //packages/d2b-provider-test-controller:d2b-provider-test-controller \ - //packages/d2b-provider-guest-cloud-hypervisor:d2b-cloud-hypervisor-controller; \ - bazel_bin="$$(realpath -e "$$('$(BAZEL_BIN)' info --config=local bazel-bin)")"; \ - stage="$$run_dir/bundle"; \ - controller_stage="$$run_dir/cloud-hypervisor-controller"; \ - mkdir -m 700 "$$stage"; \ - mkdir -m 700 "$$controller_stage"; \ - stage_tool() { source="$$(realpath -e "$$bazel_bin/$$1")"; case "$$source" in "$$bazel_bin"/*) ;; *) echo "test-host-integration: Bazel output escaped bazel-bin" >&2; return 1;; esac; [ -f "$$source" ] && [ -x "$$source" ] || { echo "test-host-integration: invalid Bazel output $$1" >&2; return 1; }; install -m 755 "$$source" "$$stage/$$2"; }; \ - stage_tool packages/d2b/d2b d2b; \ - stage_tool packages/d2bd/d2bd d2bd; \ - stage_tool packages/d2b-broker-composition/d2b-broker d2b-broker; \ - stage_tool packages/d2b-host/d2b-activation-helper d2b-activation-helper; \ - stage_tool packages/d2b-host-activation-helper/d2b-host-activation-helper d2b-host-activation-helper; \ - stage_tool packages/d2b-unsafe-local-helper/d2b-unsafe-local-helper d2b-unsafe-local-helper; \ - stage_tool packages/d2b-resource-compiler/d2b-resource-compiler d2b-resource-compiler; \ - stage_tool packages/d2b-provider-display-wayland/d2b-wayland-proxy d2b-wayland-proxy; \ - stage_tool packages/d2b-provider-test-controller/d2b-provider-test-controller d2b-provider-test-controller; \ - source="$$(realpath -e "$$bazel_bin/packages/d2b-provider-guest-cloud-hypervisor/d2b-cloud-hypervisor-controller")"; \ - case "$$source" in "$$bazel_bin"/*) ;; *) echo "test-host-integration: Cloud Hypervisor controller escaped bazel-bin" >&2; exit 1;; esac; \ - [ -f "$$source" ] && [ -x "$$source" ] || { echo "test-host-integration: invalid Bazel Cloud Hypervisor controller" >&2; exit 1; }; \ - install -m 755 "$$source" "$$controller_stage/d2b-cloud-hypervisor-controller"; \ - echo "test-host-integration: staged Bazel host-tool bundle"; \ - echo "test-host-integration: building vmChecks serially: $$names"; \ - : >"$$run_dir/outputs"; \ - : >"$$run_dir/summary"; \ - lane_rc=0; \ - max_jobs="$${D2B_HOST_VM_JOBS:-1}"; \ - case "$$max_jobs" in ''|*[!0-9]*) echo "test-host-integration: invalid D2B_HOST_VM_JOBS (want a positive integer)" >&2; exit 1;; esac; \ - if [ "$$max_jobs" -lt 1 ]; then echo "test-host-integration: D2B_HOST_VM_JOBS must be at least 1" >&2; exit 1; fi; \ - echo "test-host-integration: building vmChecks (jobs=$$max_jobs): $$names"; \ - : >"$$run_dir/failed"; \ - run_vm_check() { \ - name="$$1"; \ - check_start="$$(date +%s)"; \ - rc=0; \ - D2B_HOST_TOOL_BUNDLE="$$stage" D2B_CH_CONTROLLER_BUNDLE="$$controller_stage" \ - D2B_HOST_RUNTIME_PATH="$$run_dir/absent-host-runtime.json" \ - sudo -A -E nix build --option build-users-group "" --option extra-sandbox-paths "/dev/vhost-vsock" --impure --out-link "$$run_dir/result-$$name" --print-build-logs --print-out-paths "git+file://$$root#vmChecks.$$system.$$name" >"$$run_dir/$$name.outputs" 2>"$$run_dir/$$name.log" || rc=$$?; \ - check_duration="$$(( $$(date +%s) - check_start ))"; \ - if [ "$$rc" -eq 0 ]; then \ - status=PASS; \ - cat "$$run_dir/$$name.outputs" >>"$$run_dir/outputs"; \ - else \ - status=FAIL; \ - printf '%s\n' "$$name" >>"$$run_dir/failed"; \ - fi; \ - printf 'test-host-integration: vmCheck %-42s %s %ss\n' "$$name" "$$status" "$$check_duration" | tee -a "$$run_dir/summary"; \ - if [ "$$rc" -ne 0 ]; then \ - printf 'test-host-integration: %s tail of %s:\n' "$$name" "$$run_dir/$$name.log" >&2; \ - tail -20 "$$run_dir/$$name.log" >&2 || true; \ - fi; \ - }; \ - running=0; \ - for name in $$names; do \ - if [ "$$running" -ge "$$max_jobs" ]; then wait || true; running=0; fi; \ - run_vm_check "$$name" & \ - running="$$((running + 1))"; \ - done; \ - wait || true; \ - if [ -s "$$run_dir/failed" ]; then lane_rc=1; fi; \ - echo "test-host-integration: vmCheck summary (name, status, wall time):"; \ - cat "$$run_dir/summary"; \ - if [ -n "$$attic_cache" ]; then \ - : >"$$run_dir/attic-closure-all"; \ - while IFS= read -r output; do \ - drv="$$(nix-store -qd "$$output")" || { \ - echo "test-host-integration: could not resolve a vmCheck derivation for Attic" >&2; \ - exit 1; \ - }; \ - if [ "$$drv" = "unknown-deriver" ]; then \ - continue; \ - fi; \ - if ! nix-store -qR --include-outputs "$$drv" >>"$$run_dir/attic-closure-all"; then \ - echo "test-host-integration: could not resolve a vmCheck dependency closure for Attic" >&2; \ - exit 1; \ - fi; \ - done <"$$run_dir/outputs"; \ - sort -u -o "$$run_dir/attic-closure-all" "$$run_dir/attic-closure-all"; \ - awk 'NR == FNR { skip[$$0] = 1; next } !skip[$$0]' \ - "$$run_dir/outputs" "$$run_dir/attic-closure-all" >"$$run_dir/attic-closure"; \ - if [ ! -s "$$run_dir/attic-closure" ]; then \ - echo "test-host-integration: no Attic closure paths to upload (vmChecks satisfied from substituters)"; \ - elif ! timeout 60s attic push --jobs 16 --no-closure --stdin "$$attic_cache" <"$$run_dir/attic-closure" >"$$run_dir/attic-push.log" 2>&1; then \ - echo "test-host-integration: warning: Attic closure upload failed" >&2; \ - cat "$$run_dir/attic-push.log" >&2; \ - else \ - echo "test-host-integration: Attic closure upload succeeded"; \ - fi; \ - fi; \ - if [ "$$lane_rc" -ne 0 ]; then \ - echo "test-host-integration: at least one vmCheck failed (see the summary above)" >&2; \ - exit "$$lane_rc"; \ - fi + $(D2B_BAZEL_TEST) --config=guest \ + --test_env=D2B_VM_CHECK="$${D2B_VM_CHECK:-}" \ + //bazel/checks/vm:host_integration_lane_run ## perf - run the advisory performance budget suite. perf: diff --git a/docs/contributing/gates-and-lints.md b/docs/contributing/gates-and-lints.md index 0b56df26a..e1539c435 100644 --- a/docs/contributing/gates-and-lints.md +++ b/docs/contributing/gates-and-lints.md @@ -242,23 +242,33 @@ make test-integration make test-host-integration ``` -`make test-host-integration` builds the fixed host-tool set with local Bazel, -stages them as one `D2B_HOST_TOOL_BUNDLE`, and injects that bundle into the -selected NixOS `vmChecks`. Nix realizes the VM check around those binaries; it -must not rebuild d2b binaries through Nix. After every selected check succeeds, the lane -uploads the built dependency closures to the configured Attic cache in one -operation. It excludes the `vmCheck` result paths so a capability `SKIP` or -`BLOCKED` result cannot be substituted as a passing test on another host. -The handoff implementation is in the `test-host-integration` Make recipe and +`make test-host-integration` runs the Bazel-owned host integration lane as one +`bazel test` invocation of `//bazel/checks/vm:host_integration_lane_run`. Each of +the eleven checks boots its own NixOS guest, built as a graph output keyed on +declared inputs - the flake and its lock, the guest module sources, and the d2b +host binaries all arrive as label inputs - and the lane restores a pooled guest +per check rather than booting one guest per check. Every assertion is Rust; the +fixtures' `runNixOSTest` `testScript` surface is gone. Nix realizes the guest +closure and does not rebuild the injected d2b binaries; the implementation is in +[`bazel/checks/vm/defs.bzl`](../../bazel/checks/vm/defs.bzl), +[`nix/test-support/guest-image.nix`](../../nix/test-support/guest-image.nix), and [`nix/test-support/bazel-host-tools.nix`](../../nix/test-support/bazel-host-tools.nix). -Attic is optional for this lane. When the Attic client or its configuration is -unavailable, the lane reports an explicit skip and continues with the Bazel -and VM work. A present configuration that is invalid, ambiguous, inaccessible, -or otherwise unusable fails closed before the expensive work; an upload failure -also fails the lane. `D2B_VM_CHECK=` selects one named `vmChecks` entry -to build; `D2B_HOST_VM_CHECK=` designates the validated selected check -for the run and fails closed on an unknown name. +The guest-image action declares its own substituters and preflights them itself, +so the Attic preflight and closure upload that the old nix recipe carried are +gone with it. Cache handling now lives with the build that needs it rather than +in a second place that can drift out of step. + +The lane is a local, contributor-run pre-PR surface, not a CI gate, and it +declares virtualization as a precondition: it needs `/dev/kvm` and has no silent +emulation fallback, so on a host without KVM it stops with a message rather than +turning very slow. It is x86_64-linux only. + +`D2B_VM_CHECK=` runs one named check, and `bazel test --test_filter=` +works too - the lane reads Bazel's own filter as well as that variable, and the +first of the two that names anything wins. Each check reports under its own name +in the lane's output with the stage it was in, the rows it asserted on, and the +guest's journal and zone dump. For cold and unchanged warm evidence, run the same command twice: diff --git a/packages/d2b-test-vm-harness/src/bin/d2b-test-vm-harness.rs b/packages/d2b-test-vm-harness/src/bin/d2b-test-vm-harness.rs index a3f5f5b0c..47469e884 100644 --- a/packages/d2b-test-vm-harness/src/bin/d2b-test-vm-harness.rs +++ b/packages/d2b-test-vm-harness/src/bin/d2b-test-vm-harness.rs @@ -107,6 +107,14 @@ const IMAGES: &str = "D2B_TEST_VM_HARNESS_IMAGES"; /// The checks a contributor selected, as the existing selection variables /// carry them: a whitespace- or comma-separated list of check names. const CHECKS: &str = "D2B_VM_CHECK"; +/// Bazel's own filter, which `--test_filter` sets for a test action. +/// +/// The flag is the one a contributor reaches for without being told this +/// lane has selection variables, and the lane ignoring it is the worst +/// available failure: `bazel test --test_filter=` looks like it +/// selected one check and quietly runs all of them, which costs a pool of +/// guests and reports a verdict for a check nobody asked about. +const TEST_FILTER: &str = "TESTBRIDGE_TEST_ONLY"; /// One check, its own guest, and what that guest costs. #[derive(Clone)] @@ -365,8 +373,11 @@ fn run_group( } } -/// The checks this run was asked for, from the target's own argument or from -/// the environment variable contributors already use. +/// The checks this run was asked for, in the order the three sources are +/// consulted: the target's own `--check` argument, then the selection +/// variable contributors already use, then Bazel's own `--test_filter`. +/// The first that names anything wins, so an explicit request is never +/// widened by a default that happens to be set. #[allow(clippy::disallowed_methods, reason = "synchronous path")] fn selection(arguments: &[String]) -> Vec { let from_arguments: Vec = arguments @@ -381,8 +392,17 @@ fn selection(arguments: &[String]) -> Vec { if !from_arguments.is_empty() { return from_arguments; } - env::var(CHECKS) - .unwrap_or_default() + let named = env::var(CHECKS).unwrap_or_default(); + if !named.trim().is_empty() { + return split_names(&named); + } + split_names(&env::var(TEST_FILTER).unwrap_or_default()) +} + +/// One selection, however it was written: a whitespace-, comma- or +/// newline-separated list of check names. +fn split_names(selection: &str) -> Vec { + selection .split([',', ' ', '\n', '\t']) .map(str::trim) .filter(|name| !name.is_empty()) diff --git a/tests/README.md b/tests/README.md index e88c14e96..f6eedcb8b 100644 --- a/tests/README.md +++ b/tests/README.md @@ -86,18 +86,27 @@ or required gate evidence. completions, protocol bindings, Nix outputs, and policy inputs in the checkout; it does not alter the repository-default remote profile used by `make check`. -`make test-host-integration` first builds the fixed eight host tools with local -Bazel, injects the staged bundle into the selected NixOS `vmChecks`, and then -uploads their built dependency closures to configured Attic in one operation. -The `vmCheck` result paths are excluded so capability skips are never cached as -passing test results. -If Attic or its configuration is unavailable, the lane reports an explicit -skip and continues. If present configuration is invalid or unusable, the lane -fails closed; an upload failure is also fatal. `D2B_VM_CHECK=` builds one -named `vmChecks` entry; `D2B_HOST_VM_CHECK=` designates the validated -selected check for the run and fails closed on an unknown name. Repeating the -same command without source changes is the warm run and should execute zero -Rust compilation actions. +`make test-host-integration` runs the Bazel-owned host integration lane as one +`bazel test` invocation. Each of the eleven checks boots its own NixOS guest, +built as a graph output keyed on declared inputs, and the lane restores a +pooled guest per check rather than booting one guest per check. Every +assertion is Rust; the guest images rebuild when a guest module or a d2b host +binary changes, and repeat runs execute no Rust compilation actions. + +The lane is a local, contributor-run pre-PR surface, not a CI gate, and it +declares virtualization as a precondition: it needs `/dev/kvm` and has no +silent emulation fallback, so on a host without KVM it stops with a message +rather than turning very slow. It is x86_64-linux only. + +`D2B_VM_CHECK=` runs one named check, and `bazel test --test_filter=` +works too — the lane reads Bazel's own filter as well as that variable, and the +first of the two that names anything wins. A failing check reports under its own +name in the lane's test output, with the stage it was in, the rows it was +asserting on, and the guest's journal and zone dump. + +There is no Attic preflight or closure upload here: the guest-image action +declares its own substituters and preflights them itself, so the cache handling +lives with the build that needs it rather than in a second place that can drift. Run these aliases directly from a normal Nix-enabled checkout. Make enters the pinned `.#bazel` shell automatically when the explicit d2b shell contract From 0648cb2b72cc40cbb81be116ee6370bc91475c48 Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sat, 26 Sep 2026 23:08:39 -0700 Subject: [PATCH 25/51] refactor(vm): assert runtime-cloud-hypervisor-guest-preflight in Rust and retire its fixture The last of the eleven, and the one whose fixture was the largest: its assertions move into packages/d2b-test-vm-harness/src/checks/runtime_cloud_hypervisor_guest_preflight.rs in the fixture's own order, with the fixture's own command text and bounds. Counts, fixture and port alike: 32 succeed, 1 fail, 2 wait_for_file, 1 wait_for_unit, 3 diag_unit, 15 diag_wait, 1 diag, 9 explicit stage (plus the 18 stage names the diag_unit and diag_wait calls carry), 1 guest sleep, no bare assertion, and no start_all (the lane boots the guest). The fixture's diag_projection and its three row builders (live_rows, saved_rows, summary_rows) come with it as functions of the module. Two tokens could not stay byte-identical, and they are the only two: the fixture interpolated two nix store paths into one enrollment command (${fixtureKeys}/host.key and ${fixtureKeys}/guest.pub), and a store path is not addressable from the lane's Rust. The node installs the same two files of the same fixtureKeys derivation at /etc/d2b/fixture-keys/host.key and /etc/d2b/fixture-keys/guest.pub, and the command's two source paths are those; every other byte of every command is the fixture's. The guest is the writable-store shape plus the fixture's own let bindings (the two provider artifacts, the ComponentSession key pair, the v3 guest bundle, the Cloud Hypervisor configuration, the nested guest system whose boot the check preflights, its store-view image and the artifacts), all declared in nix/test-support/host-integration-node.nix as d2bRuntimeCloudHypervisorPreflightNode. Counts the port must satisfy: 32 succeed, 1 fail, 2 wait_for_file, 1 wait_for_unit, 3 diag_unit, 15 diag_wait, 1 diag, 9 stage plus the 18 the diagnostics carry, 1 sleep, before and after. --- bazel/checks/vm/BUILD.bazel | 1 + nix/test-support/host-integration-node.nix | 419 ++++++ .../src/checks/device_worker_launch.rs | 10 +- .../d2b-test-vm-harness/src/checks/mod.rs | 5 + .../checks/resource_operator_activation.rs | 2 +- ...untime_cloud_hypervisor_guest_preflight.rs | 1205 +++++++++++++++ .../src/checks/state_posture_contract.rs | 8 +- .../src/checks/virtiofsd_volume_runtime.rs | 6 +- ...ntime-cloud-hypervisor-guest-preflight.nix | 1295 ----------------- 9 files changed, 1643 insertions(+), 1308 deletions(-) create mode 100644 packages/d2b-test-vm-harness/src/checks/runtime_cloud_hypervisor_guest_preflight.rs delete mode 100644 tests/host-integration/runtime-cloud-hypervisor-guest-preflight.nix diff --git a/bazel/checks/vm/BUILD.bazel b/bazel/checks/vm/BUILD.bazel index 1343a314f..b6a2d8f08 100644 --- a/bazel/checks/vm/BUILD.bazel +++ b/bazel/checks/vm/BUILD.bazel @@ -153,6 +153,7 @@ _PORTED_CHECKS = [ "guest-shell-service", "privilege-oracle", "resource-operator-activation", + "runtime-cloud-hypervisor-guest-preflight", "state-posture-contract", "virtiofsd-volume-runtime", "wayland-proxy", diff --git a/nix/test-support/host-integration-node.nix b/nix/test-support/host-integration-node.nix index 30a57a256..6cbba4d29 100644 --- a/nix/test-support/host-integration-node.nix +++ b/nix/test-support/host-integration-node.nix @@ -1725,6 +1725,421 @@ rec { }; }; + # The guest `runtime-cloud-hypervisor-guest-preflight` boots: the + # writable-store shape plus the fixture's own contributions - the two + # provider artifacts, the ComponentSession key pair and v3 guest bundle, + # the Cloud Hypervisor configuration, the nested guest system whose boot + # the check preflights, its store-view image, and the zones and rows that + # carry them. + # + # Two files the fixture's commands interpolated as nix store paths are + # installed here at fixed guest paths instead, because a store path is not + # addressable from the lane's Rust: /etc/d2b/fixture-keys/host.key and + # /etc/d2b/fixture-keys/guest.pub, which are the same two files of the + # same fixtureKeys derivation. + d2bRuntimeCloudHypervisorPreflightNode = + d2bCloudHypervisorNode { + extra = + { lib, pkgs, ... }: + let + d2bLib = import ../../tests/host-integration/lib.nix { + inherit self; + inherit lib; + hostToolBundle = + if self.lib ? d2bHostToolBundle then self.lib.d2bHostToolBundle else null; + }; + cloudHypervisorArtifact = + d2bLib.mkRuntimeCloudHypervisorArtifact pkgs; + volumeProviderArtifact = d2bLib.mkVolumeProviderArtifact pkgs; + fixtureKeys = pkgs.runCommand "acceptance-component-session-keys" { } '' + mkdir -p "$out" + printf '\001\002\003\004\005\006\007\010\011\012\013\014\015\016\017\020\021\022\023\024\025\026\027\030\031\032\033\034\035\036\037\040' > "$out/host.key" + printf '\007\243\174\274\024\040\223\310\267\125\334\033\020\350\154\264\046\067\112\321\152\250\123\355\013\337\300\262\270\155\034\174' > "$out/host.pub" + printf '\041\042\043\044\045\046\047\050\051\052\053\054\055\056\057\060\061\062\063\064\065\066\067\070\071\072\073\074\075\076\077\100' > "$out/guest.key" + printf '\130\151\257\364\120\124\227\062\313\252\355\136\135\371\263\012\155\243\034\260\345\164\053\255\132\324\241\247\150\361\246\173' > "$out/guest.pub" + ''; + guestBundle = pkgs.runCommand "acceptance-guest-bundle" { + nativeBuildInputs = [ pkgs.python3 ]; + } '' + mkdir -p "$out" + cat > "$out/host.json" <<'EOF' + {"schemaVersion":"v2","site":{"allowUnsafeEastWest":false},"environments":[],"nftables":{"family":"inet","table":"d2b","chains":[],"tableHashAfterApply":null,"ownershipId":"host-integration"},"networkManager":{"filePath":"/etc/NetworkManager/conf.d/00-d2b-unmanaged.conf","matchCriteria":[],"reloadBehavior":"atomic-reload","ownership":{"owner":"root","group":"root","mode":"0644","driftPolicy":"replace"}},"hostsFile":{"startMarker":"# d2b-managed begin","endMarker":"# d2b-managed end","rule":"replace-managed-block"},"kernelModules":[],"fdOwnership":[],"cloudHypervisorCapabilities":[],"ifNameMappings":[],"ch":null,"firewallCoexistencePolicy":null} + EOF + printf '%s\n' '{"schemaVersion":"v2","vms":[]}' > "$out/processes.json" + printf '%s\n' '{"schemaVersion":"v2","publicOperations":[],"brokerOperations":[]}' > "$out/privileges.json" + printf '%s\n' '{"_manifest":{"manifestVersion":6},"_observability":{"enabled":false,"signozUrl":"http://127.0.0.1:8080","signozOtlpGrpcPort":4317,"signozOtlpHttpPort":4318,"obsVsockCid":0,"obsVsockHostSocket":"","vmName":""}}' > "$out/vms.json" + python3 - "$out/bundle.json" <<'PY' + import hashlib + import json + import sys + + # Zone-native v3 bundle: the loader (BundleResolver) accepts only the + # v3 contract. The self-hash is computed over the serialization with + # bundleHash absent and artifactHashes nullified (verify_bundle_hash). + bundle = { + "artifactHashes": {}, + "bundleVersion": 1, + "schemaVersion": "v3", + "privilegesPath": "privileges.json", + "zones": [], + "generation": { + "generatedAt": None, + "generator": "host-integration", + "sourceRevision": None, + }, + } + preimage = dict(bundle) + preimage["artifactHashes"] = None + canonical = json.dumps(preimage, sort_keys=True, separators=(",", ":")).encode() + bundle["bundleHash"] = "sha256:" + hashlib.sha256(canonical).hexdigest() + with open(sys.argv[1], "w", encoding="utf-8") as output: + json.dump(bundle, output, sort_keys=True, separators=(",", ":")) + output.write("\n") + PY + ''; + + cloudHypervisorConfig = { + controllerExecutionRef = "Host/host-system"; + defaultVcpus = 2; + defaultMemoryMb = 512; + defaultMachineType = "microvm"; + watchdog = true; + adoptionWindowMs = 30000; + healthCheckIntervalMs = 5000; + healthCheckTimeoutMs = 1000; + healthCheckFailureThreshold = 3; + startupDeadlineMs = 120000; + }; + guestSystem = d2bLib.mkGuestSystem { + inherit pkgs; + name = "acceptance-guest"; + modules = [ + ({ lib, name, ... }: { + boot.kernelParams = [ "console=ttyS0" "loglevel=7" ]; + environment.etc."d2b/component-session/guest.key".source = + "${fixtureKeys}/guest.key"; + environment.etc."d2b/component-session/parent.pub".source = + "${fixtureKeys}/host.pub"; + systemd.services.d2bd-guest = { + environment = { + RUST_LOG = "d2bd=debug"; + }; + serviceConfig = { + ReadOnlyPaths = [ + "/etc/d2b/component-session/guest.key" + "/etc/d2b/component-session/parent.pub" + ]; + StandardOutput = lib.mkForce "journal+console"; + StandardError = lib.mkForce "journal+console"; + }; + }; + systemd.services.d2b-test-boot-identity = { + wantedBy = [ "basic.target" ]; + before = [ "d2bd-guest.service" ]; + serviceConfig.Type = "oneshot"; + script = '' + printf 'D2B_GUEST_BOOT_ID=%s\n' \ + "$(${pkgs.coreutils}/bin/cat /proc/sys/kernel/random/boot_id)" \ + > /dev/console + ''; + }; + d2b.componentSession.localPrivateKeyPath = + "/etc/d2b/component-session/guest.key"; + d2b.componentSession.parentPublicKeyPath = + "/etc/d2b/component-session/parent.pub"; + d2b.componentSession.bundlePath = + "/var/lib/d2b/guest-bundle/bundle.json"; + d2b.guestBroker.bundlePath = + "/var/lib/d2b/guest-bundle/bundle.json"; + systemd.services.d2b-install-guest-bundle = { + requiredBy = [ "d2b-broker-guest.service" "d2bd-guest.service" ]; + before = [ "d2b-broker-guest.service" "d2bd-guest.service" ]; + serviceConfig.Type = "oneshot"; + script = '' + install -d -o root -g d2bd -m 0750 /var/lib/d2b/guest-bundle + for file in bundle.json host.json processes.json privileges.json; do + install -o root -g d2bd -m 0640 \ + ${guestBundle}/"$file" /var/lib/d2b/guest-bundle/"$file" + done + install -o root -g d2bd -m 0644 \ + ${guestBundle}/vms.json /var/lib/d2b/guest-bundle/vms.json + ''; + }; + networking.useDHCP = lib.mkForce false; + networking.networkmanager.enable = lib.mkForce false; + systemd.network.enable = lib.mkForce false; + services.dbus.enable = lib.mkForce false; + services.resolved.enable = lib.mkForce false; + systemd.services.systemd-vconsole-setup.enable = false; + d2b.vms.${name}.runner = { + store.onDisk = true; + store.disk = guestStoreDisk; + shares = lib.mkForce [ ]; + }; + fileSystems."/nix/store" = { + device = "/dev/vda"; + fsType = "ext4"; + options = [ "ro" "x-initrd.mount" ]; + neededForBoot = true; + }; + }) + ]; + }; + guestClosure = pkgs.closureInfo { + rootPaths = [ guestSystem.config.system.build.toplevel ]; + }; + guestStoreDisk = pkgs.runCommand "acceptance-guest-store.img" { + nativeBuildInputs = [ pkgs.coreutils pkgs.e2fsprogs ]; + } '' + mkdir -p root + while IFS= read -r path; do + cp -r --no-preserve=ownership,xattr,context "$path" root/ + done < ${guestClosure}/store-paths + truncate -s 4096M "$out" + # Reproducible ext4 image: SOURCE_DATE_EPOCH pins the superblock times and + # a fixed UUID seed pins the htree hash seed (e2fsprogs ignores an all-zero + # seed and randomizes it), so every build is byte-identical. With a random + # seed each build differed, and the nixos-install closure spec (recorded + # from an earlier build) could never match the freshly built image. + SOURCE_DATE_EPOCH=0 mkfs.ext4 -q -F \ + -U 123e4567-e89b-12d3-a456-426614174000 \ + -E hash_seed=123e4567-e89b-12d3-a456-426614174000 \ + -d root "$out" + ''; + artifacts = { + runtime-cloud-hypervisor = { + inherit (cloudHypervisorArtifact) package type catalog; + }; + volume-acceptance-provider = { + inherit (volumeProviderArtifact) package type catalog; + }; + acceptance-system = { + package = guestSystem.config.system.build.toplevel; + type = "nixos-system"; + }; + }; + in + { + d2b.site.adminUsers = [ "alice" ]; + environment.systemPackages = with pkgs; [ + iproute2 + jq + iputils + procps + ]; + d2b.artifacts = artifacts; + d2b.guestSystems.work.acceptance-guest = guestSystem; + d2b.zones.local-root.trustedPublishers.d2b-cloud-hypervisor.signingKey = + cloudHypervisorArtifact.trustedPublisher.signingKey; + d2b.zones.local-root.trustedPublishers.d2b-volume-acceptance.signingKey = + volumeProviderArtifact.trustedPublisher.signingKey; + d2b.zones.work.trustedPublishers.d2b-cloud-hypervisor.signingKey = + cloudHypervisorArtifact.trustedPublisher.signingKey; + d2b.zones.work.trustedPublishers.d2b-volume-acceptance.signingKey = + volumeProviderArtifact.trustedPublisher.signingKey; + d2b.zones.local-root.resources.host-system = { + type = "Host"; + spec = { + providerRef = "Provider/system-core"; + defaultDomain = "system"; + allowedDomains = [ "system" ]; + budget = { }; + networkAttachments = [ ]; + deviceAttachments = [ ]; + volumeAttachmentDefaults = [ ]; + }; + }; + d2b.zones.work = { + parentZone = "local-root"; + resources = { + alice = { + type = "User"; + spec = { + displayName = "Alice"; + groups = [ ]; + osUsername = "alice"; + }; + }; + d2bd = { + type = "User"; + spec = { + displayName = "d2bd"; + groups = [ ]; + osUsername = "d2bd"; + }; + }; + lifecycle-operator = { + type = "Role"; + spec.rules = [ + { + resourceTypes = [ "Endpoint" "Guest" "Host" "Process" "Provider" "Volume" "VolumeBinding" ]; + verbs = [ "get" "list" ]; + subresources = [ ]; + resourceNames = [ ]; + zones = [ "work" ]; + executionRefs = [ ]; + sessionVerbs = [ "connect" "invoke" ]; + } + { + resourceTypes = [ "Guest" ]; + verbs = [ "delete" ]; + subresources = [ ]; + resourceNames = [ "acceptance-guest" ]; + zones = [ "work" ]; + executionRefs = [ ]; + sessionVerbs = [ "connect" "invoke" ]; + } + { + resourceTypes = [ "Volume" ]; + verbs = [ "delete" ]; + subresources = [ ]; + resourceNames = [ "state" ]; + zones = [ "work" ]; + executionRefs = [ ]; + sessionVerbs = [ "connect" "invoke" ]; + } + ]; + }; + lifecycle-operator-binding = { + type = "RoleBinding"; + spec = { + roleRef = "Role/lifecycle-operator"; + subjects = [ "User/alice" ]; + externalPrincipalSelector = null; + scopeNarrowing = null; + }; + }; + host-system = { + type = "Host"; + spec = { + providerRef = "Provider/system-core"; + defaultDomain = "system"; + allowedDomains = [ "system" ]; + budget = { }; + networkAttachments = [ ]; + deviceAttachments = [ ]; + volumeAttachmentDefaults = [ ]; + }; + }; + volume-local = { + type = "Provider"; + spec = { + artifactId = "volume-acceptance-provider"; + config = { + controllerExecutionRef = "Host/host-system"; + sourcePolicies = [ + { + id = "default-state"; + class = "local-path"; + volumeKinds = [ "durable" "state" "cache" ]; + } + # U7: daemon-owned root the unprivileged daemon can + # lock and provision inline (path:daemon-state). + { + id = "daemon-state"; + class = "local-path"; + volumeKinds = [ "durable" "state" "cache" ]; + } + ]; + }; + }; + }; + volume-virtiofs = { + type = "Provider"; + spec = { + artifactId = "volume-acceptance-provider"; + config.controllerExecutionRef = "Host/host-system"; + }; + }; + state = { + type = "Volume"; + spec = { + providerRef = "Provider/volume-local"; + kind = "state"; + source = { + executionRef = "Host/host-system"; + settings = { + kind = "local-path"; + sourcePolicyId = "daemon-state"; + }; + }; + layout = [{ + path = "state"; + type = "directory"; + # U7: daemon-owned so the unprivileged daemon can + # provision inline; the guest share stays read-only. + ownerRef = "User/d2bd"; + groupRef = "User/d2bd"; + mode = "0700"; + target = null; + accessAcl = [ ]; + defaultAcl = [ ]; + foreignChildPolicy = "preserve"; + noFollow = true; + recursive = false; + sensitivity = "private"; + createPolicy = "create-if-never-provisioned"; + repairPolicy = "exact-owner"; + cleanupPolicy = "owner-controlled"; + adoptionPolicy = "quarantine-on-ambiguity"; + restartPolicy = "preserve-across-controller-restart"; + leaseClass = "none"; + invariants = [ "no-symlink" ]; + }]; + views.controller = { + path = ""; + rights = [ "read" "write" "traverse" ]; + }; + # KTD1: the attachment stays declared input only. The Volume + # side mints the durable VolumeBinding at reconcile; the + # deterministic binding identity below is + # vol-binding-6a8ea4307a30f7ceae6533f2 (volume, execution + # target, view, mount path). + attachments = [{ + executionRef = "Guest/acceptance-guest"; + transport = "virtiofs"; + view = "controller"; + access = "read-only"; + mountPath = "/state"; + settings = { + posixAcl = false; + xattr = false; + cache = "auto"; + inodeFileHandles = "never"; + threadPoolSize = null; + socketGroup = null; + }; + }]; + }; + }; + runtime-cloud-hypervisor = { + type = "Provider"; + spec = { + artifactId = "runtime-cloud-hypervisor"; + config = cloudHypervisorConfig; + }; + }; + acceptance-guest = { + type = "Guest"; + spec = { + providerRef = "Provider/runtime-cloud-hypervisor"; + executionRef = "Host/host-system"; + systemArtifactId = "acceptance-system"; + defaultDomain = "system"; + allowedDomains = [ "system" ]; + budget = { }; + volumeAttachmentDefaults = [ ]; + networkAttachments = [ ]; + deviceAttachments = [ ]; + }; + }; + }; + }; + environment.etc."d2b/fixture-keys/host.key".source = "${fixtureKeys}/host.key"; + environment.etc."d2b/fixture-keys/guest.pub".source = "${fixtureKeys}/guest.pub"; + }; + }; + # The guest each fixture-less image evaluates, by the name the image action # asks for. A check's own guest is read out of the check's fixture; these are # the guests with no fixture to be read out of - the two reusable shapes the @@ -1781,6 +2196,10 @@ rec { node = d2bResourceOperatorActivationNode; testName = "d2b-resource-operator-activation"; }; + runtime-cloud-hypervisor-guest-preflight = { + node = d2bRuntimeCloudHypervisorPreflightNode; + testName = "d2b-runtime-cloud-hypervisor-guest-preflight"; + }; state-posture-contract = { node = d2bStatePostureContractNode; testName = "d2b-state-posture-contract"; diff --git a/packages/d2b-test-vm-harness/src/checks/device_worker_launch.rs b/packages/d2b-test-vm-harness/src/checks/device_worker_launch.rs index e7a6adb44..6febea9dc 100644 --- a/packages/d2b-test-vm-harness/src/checks/device_worker_launch.rs +++ b/packages/d2b-test-vm-harness/src/checks/device_worker_launch.rs @@ -817,10 +817,10 @@ fn declared_rows(bundle: &Value) -> BTreeMap<(&str, &str), &Value> { .get("metadata") .and_then(|metadata| metadata.get("name")) .and_then(Value::as_str); - if let (Some(kind), Some(name)) = (kind, name) { - if matches!(kind, "Process" | "EphemeralProcess") { - declared.insert((kind, name), row); - } + if let (Some(kind), Some(name)) = (kind, name) + && matches!(kind, "Process" | "EphemeralProcess") + { + declared.insert((kind, name), row); } } declared @@ -1085,7 +1085,7 @@ fn dumps_string(text: &str, out: &mut String) { '\t' => out.push_str("\\t"), '\u{08}' => out.push_str("\\b"), '\u{0c}' => out.push_str("\\f"), - character if character < ' ' || character > '~' => { + character if !(' '..='~').contains(&character) => { let mut buffer = [0_u16; 2]; for unit in character.encode_utf16(&mut buffer) { out.push_str(&format!("\\u{unit:04x}")); diff --git a/packages/d2b-test-vm-harness/src/checks/mod.rs b/packages/d2b-test-vm-harness/src/checks/mod.rs index 7cb962fd9..ab8fc37b0 100644 --- a/packages/d2b-test-vm-harness/src/checks/mod.rs +++ b/packages/d2b-test-vm-harness/src/checks/mod.rs @@ -31,6 +31,7 @@ pub mod guest_shell_service; pub mod privilege_oracle; pub mod state_posture_contract; pub mod resource_operator_activation; +pub mod runtime_cloud_hypervisor_guest_preflight; pub mod virtiofsd_volume_runtime; pub mod wayland_proxy; @@ -72,6 +73,10 @@ const PORTED: &[(&str, Assertions)] = &[ "virtiofsd-volume-runtime", virtiofsd_volume_runtime::assertions, ), + ( + "runtime-cloud-hypervisor-guest-preflight", + runtime_cloud_hypervisor_guest_preflight::assertions, + ), ("wayland-proxy", wayland_proxy::assertions), ]; diff --git a/packages/d2b-test-vm-harness/src/checks/resource_operator_activation.rs b/packages/d2b-test-vm-harness/src/checks/resource_operator_activation.rs index 66ecdc454..2cf19f8ae 100644 --- a/packages/d2b-test-vm-harness/src/checks/resource_operator_activation.rs +++ b/packages/d2b-test-vm-harness/src/checks/resource_operator_activation.rs @@ -22,7 +22,7 @@ //! row builders are the fixture's own `live_rows` and `saved_rows`. //! //! The guest is the reusable daemon node plus the fixture's own contributions -//! - nftables, the acceptance artifacts and zones, the two users, and `jq` - +//! (nftables, the acceptance artifacts and zones, the two users, and `jq`), //! declared in `nix/test-support/host-integration-node.nix`. //! //! `start_all()` is not restated here: it is the lane's own boot of the diff --git a/packages/d2b-test-vm-harness/src/checks/runtime_cloud_hypervisor_guest_preflight.rs b/packages/d2b-test-vm-harness/src/checks/runtime_cloud_hypervisor_guest_preflight.rs new file mode 100644 index 000000000..666de7bea --- /dev/null +++ b/packages/d2b-test-vm-harness/src/checks/runtime_cloud_hypervisor_guest_preflight.rs @@ -0,0 +1,1205 @@ +//! The nested Cloud Hypervisor Guest acceptance check, ported from its +//! fixture. +//! +//! It boots the writable-store host node and drives the controller-owned Guest +//! lifecycle end to end: the daemon, the broker socket and the broker service +//! come up; the preflight capture reads the public, redacted Resource +//! projection before anything waits on the nested VMM; the two storage +//! providers establish their live ResourceV3 controller sessions; the host's +//! KVM, vhost-net and cgroup-v2 postures are asserted; the artifact catalog, +//! the per-Guest closure spec and the zone's resource bundle are read back; +//! the fixture enrolls the Guest's ComponentSession key pair and record; the +//! VMM API socket, the runner process, the Guest's endpoints and system +//! Volume, the store view and the declared `state` Volume's binding all reach +//! their declared end state; the runner process survives a `d2bd` restart +//! while the Guest's session generation advances behind it; and the Guest and +//! the Volume are then deleted and drained - socket gone, processes gone, +//! binding gone - so no serving effect outlives its owner. +//! +//! The assertions are the fixture's, in the fixture's order, with the +//! fixture's own command text and bounds. What the fixture expressed as +//! `machine.*` calls is [`GuestControl`]'s own operations, and what it +//! expressed as `stage`, `diag`, `diag_unit` and `diag_wait` calls are the +//! same primitives the fixtures' diagnostics prelude provides, so a failure +//! reported here reads the way the fixture's failure read. The fixture's row +//! projection and its three row builders (`live_rows`, `saved_rows`, +//! `summary_rows`) ride with it unchanged, as does the runner search and the +//! pid and start time a later assertion checks it by. +//! +//! Two tokens the fixture interpolated are the one thing this module cannot +//! carry: `${fixtureKeys}/host.key` and `${fixtureKeys}/guest.pub` name files +//! in a nix store path, and a store path is not addressable from the lane's +//! Rust. The node installs the same four `printf`'d key files at +//! `/etc/d2b/fixture-keys/`, and the enrollment command reads the two it needs +//! from there; every other byte of that command, and of every other command +//! here, is the fixture's own. +//! +//! The guest is the writable-store shape plus the fixture's own +//! contributions - the acceptance artifacts and their publisher keys, the two +//! zones and their rows, the checked guest system with its store image, and +//! the four userspace tools its commands drive it with - declared in +//! `nix/test-support/host-integration-node.nix`. +//! +//! `start_all()` is not restated here: it is the lane's own boot of the guest +//! the check runs against. + +use std::time::Duration; + +use crate::legacy::{DiagRow, GuestControl, LegacyError, LegacyResult}; + +/// The bound the daemon's activation gets, the fixture's own, before and +/// after the restart. +const DAEMON_BOUND: Duration = Duration::from_secs(180); + +/// The bound the broker socket unit and the two public-socket file waits get, +/// the fixture's own. +const SOCKET_BOUND: Duration = Duration::from_secs(30); + +/// The bound the broker service's activation gets, the fixture's own. +const BROKER_BOUND: Duration = Duration::from_secs(30); + +/// The bound the two controller waits get, the fixture's own: cold artifact +/// extraction inside a fresh VM varies widely on shared hardware, and the +/// fixture waited an eventual state rather than a timing SLO. +const CONTROLLER_BOUND: Duration = Duration::from_secs(180); + +/// The bound the Guest console's boot identity gets, the fixture's own. +const CONSOLE_BOUND: Duration = Duration::from_secs(30); + +/// The bound the Guest's own row waits get, the fixture's own, plus the VMM +/// Process drain's. +const GUEST_BOUND: Duration = Duration::from_secs(30); + +/// The bound the Guest's system Volume gets, the fixture's own. +const GUEST_VOLUME_BOUND: Duration = Duration::from_secs(180); + +/// The bound the declared `state` Volume's binding gets, the fixture's own. +const BINDING_BOUND: Duration = Duration::from_secs(180); + +/// The bound the binding's worker and endpoint get, the fixture's own. +const BINDING_WORKER_BOUND: Duration = Duration::from_secs(60); + +/// The bound the two VMM API-socket waits get, the fixture's own. +const API_SOCKET_BOUND: Duration = Duration::from_secs(30); + +/// The bound the two drain-requested waits get, the fixture's own. +const DRAINING_BOUND: Duration = Duration::from_secs(30); + +/// The bound the Guest's own drain gets, the fixture's own. +const DRAINED_BOUND: Duration = Duration::from_secs(60); + +/// The bound the binding's own drain gets, the fixture's own. +const BINDING_DRAIN_BOUND: Duration = Duration::from_secs(120); + +/// The row projection the shared diagnostics print on a timed-out wait; it +/// mirrors the fields each wait asserts on (issue #513). +const DIAG_PROJECTION: &str = concat!( + "[.resources[] | {type: .type, name: .metadata.name, ", + "owner: .metadata.ownerRef, uid: .metadata.uid, ", + "gen: .metadata.generation, phase: .status.phase, ", + "obs: .status.observedGeneration, ", + "provider: .spec.providerRef, execution: .spec.executionRef, ", + "processClass: .spec.processClass, template: .spec.template, ", + "conditions: [.status.conditions[]? | ", + "{type: .type, status: .status, reason: .reason}], ", + "outcome: (.status.outcome | ", + "if . == null then null else ", + "{code: .code, retryable: .retryable} end), ", + "resource: .status.resource}]", +); + +/// The broker service's activation, the fixture's own command. +const BROKER_START: &str = "systemctl start d2b-broker.service"; + +/// The preflight capture: one public, redacted row projection per +/// resource type, then the daemon's ComponentSession terminal-error count, all +/// written to `/run/d2b-preflight-summary.log` and echoed. +const PREFLIGHT_CAPTURE: &str = concat!( + "set -o pipefail; ", + ": > /run/d2b-preflight-summary.log; ", + "for resource_type in Guest Process Endpoint Volume Provider VolumeBinding; do ", + "printf '%s: ' \"$resource_type\" >> /run/d2b-preflight-summary.log; ", + "timeout 5s runuser -u alice -- env ", + "D2B_PUBLIC_SOCKET=/run/d2b/public.sock ", + "d2b --zone work --json list \"$resource_type\" ", + "2>/dev/null | ", + "jq -c '[.resources[] | ", + "{type: .type, ", + "metadata: {name: .metadata.name, uid: .metadata.uid, ", + "generation: .metadata.generation, ownerRef: .metadata.ownerRef, ", + "zone: .metadata.zone}, ", + "spec: {providerRef: .spec.providerRef, ", + "executionRef: .spec.executionRef, ", + "processClass: .spec.processClass, template: .spec.template}, ", + "status: {phase: .status.phase, ", + "observedGeneration: .status.observedGeneration, ", + "conditions: [.status.conditions[]? | ", + "{type: .type, status: .status, reason: .reason}], ", + "outcome: (.status.outcome | ", + "if . == null then null else ", + "{code: .code, retryable: .retryable} end), ", + "resource: .status.resource}}]' ", + ">> /run/d2b-preflight-summary.log 2>/dev/null || ", + "printf 'unavailable\\n' >> /run/d2b-preflight-summary.log; ", + "done; ", + "session_errors=$(journalctl -u d2bd.service --no-pager -b ", + "2>/dev/null | grep -Ec ", + "'session-authentication-failed|session-generation-stale' || true); ", + "printf 'ComponentSession terminal error count: %s\\n' \"$session_errors\" ", + ">> /run/d2b-preflight-summary.log; ", + "cat /run/d2b-preflight-summary.log", +); + +/// The daemon's journal carries both external providers' live ResourceV3 +/// sessions, one per acceptance controller. +const CONTROLLER_SESSIONS: &str = concat!( + "test \"$(journalctl -u d2bd.service --no-pager -o cat -b ", + "| grep -Fc 'external Provider controller ResourceV3 session live')\" -ge 2", +); + +/// One `Ready`, generation-settled controller Process per volume provider. +const VOLUME_CONTROLLER_PROCESSES: &str = concat!( + "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock ", + "d2b --zone work --json list Process ", + ">/run/d2b-volume-controller-processes.json && ", + "jq -e '", + "([.resources[] | select(.type == \"Process\" and ", + ".metadata.ownerRef == \"Provider/volume-local\" and ", + ".spec.providerRef == \"Provider/system-minijail\" and ", + ".spec.processClass == \"controller\" and ", + ".spec.template == \"controller-volume-acceptance-provider-acceptance-controller\" and ", + "", + ".status.phase == \"Ready\" and ", + ".status.observedGeneration == .metadata.generation)] | length == 1) and ", + "([.resources[] | select(.type == \"Process\" and ", + ".metadata.ownerRef == \"Provider/volume-virtiofs\" and ", + ".spec.providerRef == \"Provider/system-minijail\" and ", + ".spec.processClass == \"controller\" and ", + ".spec.template == \"controller-volume-acceptance-provider-acceptance-controller\" and ", + "", + ".status.phase == \"Ready\" and ", + ".status.observedGeneration == .metadata.generation)] | length == 1)' ", + "/run/d2b-volume-controller-processes.json", +); + +/// Both controller Processes are live, and no `pause` process is: the +/// two are the fixture's own awk and ps pipeline. +const ACCEPTANCE_CONTROLLERS: &str = concat!( + "test \"$(ps -eo pid=,args= | awk '$NF ~ /acceptance-controller$/ {print $1}' ", + "| wc -l)\" -ge 2 && ", + "! ps -eo args= | grep -E '(^|/)pause([[:space:]]|$)' | grep -v grep", +); + +/// The nested VMM's API socket, waited for the fixture's own 180 attempts, +/// with its full failure report behind the wait. +const NESTED_VMM_API_SOCKET: &str = concat!( + "for attempt in $(seq 1 180); do ", + "test -S /var/lib/d2b/zones/work/guests/acceptance-guest/acceptance-guest.sock ", + "&& exit 0; ", + "sleep 1; ", + "done; ", + "echo 'Cloud Hypervisor API socket did not become ready within 180s'; ", + "cat /run/d2b-preflight-summary.log; ", + "for resource_type in Volume VolumeBinding; do ", + "echo \"=== $resource_type ===\"; ", + "timeout 10s runuser -u alice -- env ", + "D2B_PUBLIC_SOCKET=/run/d2b/public.sock ", + "d2b --zone work --json list \"$resource_type\" 2>/dev/null | ", + "jq -c '.resources[] | {name: .metadata.name, phase: .status.phase, ", + "observedGeneration: .status.observedGeneration, ", + "conditions: [.status.conditions[]? | {type: .type, reason: .reason}], ", + "ready: .status.resource.ready}' || true; ", + "done; ", + "echo '=== Process ==='; ", + "timeout 10s runuser -u alice -- env ", + "D2B_PUBLIC_SOCKET=/run/d2b/public.sock ", + "d2b --zone work --json list Process 2>/dev/null | ", + "jq -c '.resources[] | select(.name | startswith(\"vol-vfd\")) | ", + "{name: .metadata.name, phase: .status.phase, ", + "conditions: [.status.conditions[]? | {type: .type, reason: .reason}], ", + "outcome: .status.outcome, update: .status.update}' || true; ", + "exit 1", +); + +/// The Guest's boot identity reached the host journal through its console. +const GUEST_CONSOLE_BOOT_ID: &str = concat!( + "journalctl --no-pager -b ", + "| grep -q 'D2B_GUEST_BOOT_ID='", +); + +/// The journal tail the boot-id wait prints. +const HOST_JOURNAL_TAIL: &str = + "journalctl --no-pager -o cat -b -n 120 2>/dev/null || true"; + +/// The Guest's ComponentSession enrollment: the Guest's uid, the boot +/// digest of the boot id the console printed, the key pair and the +/// `guest.json` enrollment record. +/// +/// The two `install` lines read the fixture's own key material from +/// `/etc/d2b/fixture-keys`, where the node installs the same four `printf`'d +/// files the fixture's `let` built. Those two source tokens are the only +/// bytes of this command that are not the fixture's own: the fixture +/// interpolated a nix store path there, and a store path is not addressable +/// from the lane's Rust. The destination paths, the owners, the modes and +/// every other byte are the fixture's. +const GUEST_SESSION_ENROLLMENT: &str = concat!( + "guest_uid=$(runuser -u alice -- env ", + "D2B_PUBLIC_SOCKET=/run/d2b/public.sock ", + "d2b --zone work --json list Guest | ", + "jq -er '.resources[] | select(.metadata.name == \"acceptance-guest\") ", + "| .metadata.uid') && ", + "boot_id=$(journalctl --no-pager -b ", + "| sed -n 's/.*D2B_GUEST_BOOT_ID=\\([0-9a-f-]*\\).*/\\1/p' ", + "| tail -1) && ", + "boot_digest=$(printf 'd2b-kernel-boot-id-v1\\0%s' \"$boot_id\" ", + "| sha256sum | cut -d' ' -f1) && ", + "install -d -o d2bd -g d2bd -m 0700 ", + "/var/lib/d2b/zones/work/guests/acceptance-guest/component-session && ", + "install -o d2bd -g d2bd -m 0600 /etc/d2b/fixture-keys/host.key ", + "/var/lib/d2b/zones/work/guests/acceptance-guest/component-session/host.key && ", + "install -o d2bd -g d2bd -m 0600 /etc/d2b/fixture-keys/guest.pub ", + "/var/lib/d2b/zones/work/guests/acceptance-guest/component-session/guest.pub && ", + "cat > /var/lib/d2b/zones/work/guests/acceptance-guest/component-session/guest.json ", + "</dev/null ", + "| grep -F 'Bundle resolver could not load'", +); + +/// The installed artifact catalog declares this Guest's setup descriptor +/// and its store view, from the same file the daemon reads. +const GUEST_SETUP_DESCRIPTOR: &str = concat!( + "test -r /etc/d2b/artifact-catalog.json && ", + "jq -e '", + "(.guestSetupDescriptors | any(.[]; ", + ".zone == \"work\" and .guest == \"acceptance-guest\" and ", + ".providerArtifactId == \"runtime-cloud-hypervisor\" and ", + ".descriptor.providerRef == \"Provider/runtime-cloud-hypervisor\" and ", + ".descriptor.systemArtifactId == \"acceptance-system\" and ", + ".descriptor.childRoles == [\"vmm\", \"ch-api\", \"guest-control\", \"system\"])) and ", + "", + "(.guestClosures | any(.[]; ", + ".zone == \"work\" and .guest == \"acceptance-guest\" and ", + ".artifactId == \"acceptance-system\" and (.closurePaths | length > 0) and ", + "(. as $guest | ($guest.closurePaths | index($guest.toplevel)) != null) and ", + ".storeView.mountPoint == \"/nix/store\" and ", + "(.storeView.root | endswith(\"/zones/work/guests/acceptance-guest/store-view\")) and ", + "", + "(.vmm.binaryPath | endswith(\"/bin/cloud-hypervisor\"))))' ", + "/etc/d2b/artifact-catalog.json", +); + +/// The per-Guest closure spec the brokered store sync reads. +const GUEST_CLOSURE: &str = concat!( + "test -r /etc/d2b/closures/zones/work/acceptance-guest.json && ", + "jq -e '", + ".schemaVersion == \"v3\" and .artifactId == \"acceptance-system\" and ", + "(.closurePaths | length > 0) and ", + "(. as $guest | ($guest.closurePaths | index($guest.toplevel)) != null) and ", + ".storeView.mountPoint == \"/nix/store\" and ", + ".storeView.sync == \"broker-store-sync\" and ", + "(.vmm.argv | index(\"--api-socket\")) != null' ", + "/etc/d2b/closures/zones/work/acceptance-guest.json", +); + +/// The zone's resource bundle carries the Guest row and no store path or +/// VMM argv. +const RESOURCE_BUNDLE: &str = concat!( + "jq -e '", + ".resources | any(.[]; .type == \"Guest\" and ", + ".metadata.name == \"acceptance-guest\" and ", + ".spec.providerRef == \"Provider/runtime-cloud-hypervisor\" and ", + ".spec.systemArtifactId == \"acceptance-system\") and ", + "all(.[]; (tostring | contains(\"/nix/store/\") | not) and ", + "(tostring | contains(\"\\\"argv\\\"\") | not))' ", + "/etc/d2b/zones/work/resource-bundle.json", +); + +/// The Guest's own readiness loop, with its three terminal-failure exits: +/// a failed Guest, a failed VMM Process, and a terminal ComponentSession. +const GUEST_READY: &str = concat!( + "for attempt in $(seq 1 45); do ", + "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock ", + "d2b --zone work --json list Guest >/run/d2b-guest-ready.json && ", + "jq -e '", + "(.resources | map(select(.type == \"Guest\" and ", + ".metadata.name == \"acceptance-guest\"))) as $guests | ", + "($guests | length) == 1 and ", + "$guests[0].status.phase == \"Ready\" and ", + "$guests[0].status.observedGeneration == $guests[0].metadata.generation and ", + "$guests[0].status.resource.runtimeReady == true and ", + "$guests[0].status.resource.bootstrapReady == true and ", + "$guests[0].status.resource.activeProcessCount == 1' ", + "/run/d2b-guest-ready.json && exit 0; ", + "if jq -e 'any(.resources[]; ", + ".metadata.name == \"acceptance-guest\" and ", + ".status.phase == \"Failed\")' ", + "/run/d2b-guest-ready.json >/dev/null; then ", + "echo 'Guest reported a terminal failure'; exit 1; fi; ", + "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock ", + "d2b --zone work --json list Process >/run/d2b-vmm-fast-fail.json && ", + "if jq -e 'any(.resources[]; ", + ".metadata.name == \"acceptance-guest-vmm\" and ", + ".status.phase == \"Failed\" and ", + ".status.outcome.retryable != true)' ", + "/run/d2b-vmm-fast-fail.json >/dev/null; then ", + "echo 'VMM Process reported a terminal failure'; exit 1; fi; ", + "if journalctl -u d2bd.service --no-pager -b ", + "| grep -q 'session-authentication-failed\\|session-generation-stale'; then ", + "echo 'ComponentSession reported a terminal failure'; exit 1; fi; ", + "sleep 1; done; ", + "echo 'Guest readiness failed:'; ", + "jq -c '.resources[] | select(.type == \"Guest\" and ", + ".metadata.name == \"acceptance-guest\") | ", + "{name: .metadata.name, uid: .metadata.uid, ", + "owner: .metadata.ownerRef, phase: .status.phase, ", + "observedGeneration: .status.observedGeneration, ", + "conditions: [.status.conditions[]? | ", + "{type: .type, status: .status, reason: .reason}], ", + "outcome: (.status.outcome | ", + "if . == null then null else ", + "{code: .code, retryable: .retryable} end), ", + "resource: .status.resource}' ", + "/run/d2b-guest-ready.json; ", + "echo 'Dependent Process status:'; ", + "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock ", + "d2b --zone work --json list Process | ", + "jq -c '.resources[] | ", + "{name: .metadata.name, uid: .metadata.uid, ", + "owner: .metadata.ownerRef, provider: .spec.providerRef, ", + "execution: .spec.executionRef, processClass: .spec.processClass, ", + "template: .spec.template, phase: .status.phase, ", + "observedGeneration: .status.observedGeneration, ", + "conditions: [.status.conditions[]? | ", + "{type: .type, status: .status, reason: .reason}], ", + "outcome: (.status.outcome | ", + "if . == null then null else ", + "{code: .code, retryable: .retryable} end), ", + "resource: .status.resource}'; ", + "for resource_type in Endpoint Volume Provider; do ", + "echo \"$resource_type status:\"; ", + "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock ", + "d2b --zone work --json list \"$resource_type\" | ", + "jq -c '.resources[] | {name: .metadata.name, uid: .metadata.uid, ", + "owner: .metadata.ownerRef, provider: .spec.providerRef, ", + "execution: .spec.executionRef, phase: .status.phase, ", + "observedGeneration: .status.observedGeneration, ", + "conditions: [.status.conditions[]? | ", + "{type: .type, status: .status, reason: .reason}], ", + "outcome: (.status.outcome | ", + "if . == null then null else ", + "{code: .code, retryable: .retryable} end), ", + "resource: .status.resource}'; done; exit 1", +); + +/// The Guest's VMM runner Process and the runtime controller are both +/// `Ready`, and the Guest has exactly one child Process. +const GUEST_VMM_PROCESS_READY: &str = concat!( + "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock ", + "d2b --zone work --json list Process ", + ">/run/d2b-process-ready.json && ", + "jq -e '", + "([.resources[] | select(.type == \"Process\" and ", + ".metadata.ownerRef == \"Guest/acceptance-guest\")] | length == 1) and ", + "([.resources[] | select(.type == \"Process\" and ", + ".metadata.name == \"acceptance-guest-vmm\" and ", + ".metadata.ownerRef == \"Guest/acceptance-guest\" and ", + ".spec.providerRef == \"Provider/system-minijail\" and ", + ".spec.executionRef == \"Host/host-system\" and ", + ".spec.processClass == \"worker\" and ", + ".spec.template == \"cloud-hypervisor-runner\" and ", + ".status.phase == \"Ready\")] | length == 1) and ", + "([.resources[] | select(.type == \"Process\" and ", + ".metadata.ownerRef == \"Provider/runtime-cloud-hypervisor\" and ", + ".spec.providerRef == \"Provider/system-minijail\" and ", + ".spec.executionRef == \"Host/host-system\" and ", + ".spec.processClass == \"controller\" and ", + ".spec.template == \"controller-runtime-cloud-hypervisor-cloud-hypervisor-controller\" ", + "and ", + ".status.phase == \"Ready\")] | length == 1)' ", + "/run/d2b-process-ready.json", +); + +/// The Guest's two endpoints are `Ready`. +const GUEST_ENDPOINTS_READY: &str = concat!( + "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock ", + "d2b --zone work --json list Endpoint ", + ">/run/d2b-endpoint-ready.json && ", + "jq -e '", + "([.resources[] | select(.type == \"Endpoint\" and ", + ".metadata.ownerRef == \"Guest/acceptance-guest\" and ", + ".status.phase == \"Ready\")] | length == 2)' ", + "/run/d2b-endpoint-ready.json", +); + +/// The Guest's system Volume is `Ready` from its `nix-closure` source. +const GUEST_VOLUME_READY: &str = concat!( + "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock ", + "d2b --zone work --json list Volume ", + ">/run/d2b-volume-ready.json && ", + "jq -e '", + "([.resources[] | select(.type == \"Volume\" and ", + ".metadata.name == \"acceptance-guest-system\" and ", + ".metadata.ownerRef == \"Guest/acceptance-guest\" and ", + ".spec.source.settings.kind == \"nix-closure\" and ", + ".spec.source.settings.sourcePolicyId == null and ", + ".spec.source.settings.systemArtifactId == \"acceptance-system\" and ", + ".status.phase == \"Ready\" and ", + ".status.observedGeneration == .metadata.generation)] | length == 1)' ", + "/run/d2b-volume-ready.json", +); + +/// The store view is its own Volume, owned by nobody, `Ready` and settled. +const STORE_VIEW_VOLUME: &str = concat!( + "jq -e '", + "([.resources[] | select(.type == \"Volume\" and ", + ".metadata.name == \"store-view-acceptance-guest\" and ", + ".metadata.ownerRef == null and ", + ".spec.source.settings.kind == \"nix-closure\" and ", + ".spec.source.settings.sourcePolicyId == null and ", + ".spec.source.settings.systemArtifactId == \"acceptance-system\" and ", + ".status.phase == \"Ready\" and ", + ".status.observedGeneration == .metadata.generation)] | length == 1)' ", + "/run/d2b-volume-ready.json", +); + +/// The store view and the Guest's system Volume are two distinct rows. +const STORE_VIEW_UIDS: &str = concat!( + "jq -e '", + "([.resources[] | select(.type == \"Volume\" and ", + "(.metadata.name == \"store-view-acceptance-guest\" or ", + ".metadata.name == \"acceptance-guest-system\")) ", + "| .metadata.uid]) as $uids | ", + "$uids | length == 2 and (unique | length == 2)' ", + "/run/d2b-volume-ready.json", +); + +/// The declared `state` Volume's attachment is served end to end: exactly +/// one deterministically named binding, owned by the Volume, `Ready` +/// under a current fence. +const VOLUME_BINDING_READY: &str = concat!( + "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock ", + "d2b --zone work --json list VolumeBinding ", + ">/run/d2b-binding-ready.json && ", + "jq -e '", + "([.resources[] | select(.type == \"VolumeBinding\" and ", + ".metadata.ownerRef == \"Volume/state\")] | length) == 1 and ", + "([.resources[] | select(.type == \"VolumeBinding\" and ", + ".metadata.name == \"vol-binding-6a8ea4307a30f7ceae6533f2\" and ", + ".metadata.ownerRef == \"Volume/state\" and ", + ".status.phase == \"Ready\" and ", + ".status.observedGeneration == .metadata.generation and ", + ".status.resource.ready == true and ", + ".status.resource.fence.uid == .metadata.uid and ", + ".status.resource.fence.generation == .metadata.generation and ", + ".status.resource.fence.revision > 0)] | length) == 1' ", + "/run/d2b-binding-ready.json", +); + +/// The binding's virtiofs worker Process and its private endpoint are +/// `Ready`. +const BINDING_WORKER_READY: &str = concat!( + "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock ", + "d2b --zone work --json list Process ", + ">/run/d2b-binding-worker.json && ", + "jq -e '", + "([.resources[] | select(.type == \"Process\" and ", + ".metadata.ownerRef == ", + "\"VolumeBinding/vol-binding-6a8ea4307a30f7ceae6533f2\" and ", + ".spec.providerRef == \"Provider/system-minijail\" and ", + ".spec.executionRef == \"Host/host-system\" and ", + ".spec.processClass == \"worker\" and ", + ".spec.template == \"virtiofsd-worker\" and ", + ".status.phase == \"Ready\")] | length) == 1' ", + "/run/d2b-binding-worker.json && ", + "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock ", + "d2b --zone work --json list Endpoint ", + ">/run/d2b-binding-endpoint.json && ", + "jq -e '", + "([.resources[] | select(.type == \"Endpoint\" and ", + ".metadata.ownerRef == ", + "\"VolumeBinding/vol-binding-6a8ea4307a30f7ceae6533f2\" and ", + ".status.phase == \"Ready\")] | length) == 1' ", + "/run/d2b-binding-endpoint.json", +); + +/// The VMM's API socket, as a socket. +const GUEST_API_SOCKET: &str = concat!( + "test -S ", + "/var/lib/d2b/zones/work/guests/acceptance-guest/acceptance-guest.sock", +); + +/// The Guest's state directory listing, printed by both socket waits. +const GUEST_STATE_DIR: &str = + "ls -la /var/lib/d2b/zones/work/guests/acceptance-guest/ 2>&1 || true"; + +/// The VMM's API socket and the store view's two current links exist. +const GUEST_STATE_CHAIN: &str = concat!( + "test -S /var/lib/d2b/zones/work/guests/acceptance-guest/acceptance-guest.sock && ", + "test -L /var/lib/d2b/zones/work/guests/acceptance-guest/store-view/state/current && ", + "", + "test -L /var/lib/d2b/zones/work/guests/acceptance-guest/store-view/meta/current && ", + "", + "test -d /var/lib/d2b/zones/work/guests/acceptance-guest/store-view/live", +); + +/// The one runner serving this Guest: its pid and its start time, read from +/// `/proc` the fixture's own way. +const RUNNER_PROCESS: &str = concat!( + "set -- $(for proc in /proc/[0-9]*; do ", + "exe=$(readlink \"$proc/exe\" 2>/dev/null || true); ", + "case \"$exe\" in */bin/cloud-hypervisor) ", + "cmd=$(tr '\\0' ' ' < \"$proc/cmdline\"); ", + "case \"$cmd\" in *--api-socket*acceptance-guest*) ", + "pid=${proc#/proc/}; ", + "printf '%s %s ' \"$pid\" \"$(awk '{print $22}' \"$proc/stat\")\";; ", + "esac;; esac; done); ", + "test \"$#\" -eq 2; printf '%s %s' \"$1\" \"$2\"", +); + +/// The enrollment record read back before the restart: the Guest's uid, +/// the four generations, and the session generation the console logged. +const GUEST_SESSION_BEFORE: &str = concat!( + "guest_uid=$(runuser -u alice -- env ", + "D2B_PUBLIC_SOCKET=/run/d2b/public.sock ", + "d2b --zone work --json list Guest | ", + "jq -er '.resources[] | select(.type == \"Guest\" and ", + ".metadata.name == \"acceptance-guest\" and ", + ".spec.providerRef == \"Provider/runtime-cloud-hypervisor\" and ", + ".spec.executionRef == \"Host/host-system\") | .metadata.uid') && ", + "jq -c ", + "'{guestRef, guestUid, zone, reconnectGeneration, ", + "providerGeneration, controllerGeneration, assignmentEpoch}' ", + "/var/lib/d2b/zones/work/guests/acceptance-guest/component-session/guest.json ", + ">/run/d2b-guest-session-before.json && ", + "jq -e --arg guest_uid \"$guest_uid\" ", + "'.guestRef == \"Guest/acceptance-guest\" and .guestUid == $guest_uid ", + "and .zone == \"work\" and .reconnectGeneration > 0 and ", + ".providerGeneration > 0 and .controllerGeneration > 0 and ", + ".assignmentEpoch > 0' ", + "/run/d2b-guest-session-before.json >/dev/null && ", + "session_generation=$(journalctl --no-pager -b 2>/dev/null | ", + "grep -F 'Guest ComponentSession Resource API server starting' | ", + "grep -oE 'generation[[:space:]]*=[[:space:]]*[0-9]+' | ", + "grep -oE '[0-9]+' | tail -1) && ", + "test -n \"$session_generation\" && ", + "test \"$session_generation\" -ge 1 && ", + "printf '%s\\n' \"$session_generation\" ", + ">/run/d2b-guest-session-generation-before", +); + +/// The Guest target agent never reported a bundle-validation failure: +/// the console the Guest's read failures are forwarded into is clean. +const BUNDLE_VALIDATION_FAILED: &str = concat!( + "journalctl --no-pager -o cat -b ", + "| grep -F 'Guest process bundle validation failed'", +); + +/// The restart boundary the adoption is asserted across. +const DAEMON_RESTART: &str = "systemctl restart d2bd.service"; + +/// The VMM's API socket is still there after the daemon restarted. +const API_SOCKET_AFTER_RESTART: &str = concat!( + "test -S ", + "/var/lib/d2b/zones/work/guests/acceptance-guest/acceptance-guest.sock", +); + +/// The 60-attempt loop that waits for the Guest's session generation to +/// advance and the adopted Guest, VMM runner and controller to be `Ready` +/// again at the same uid. +const SESSION_GENERATION_ADVANCE: &str = concat!( + "rm -f /run/d2b-guest-adopted.json /run/d2b-process-adopted.json; ", + "for attempt in $(seq 1 60); do ", + "session_generation_before=$(cat ", + "/run/d2b-guest-session-generation-before) && ", + "session_generation_after=$(journalctl --no-pager -b 2>/dev/null | ", + "grep -F 'Guest ComponentSession Resource API server starting' | ", + "grep -oE 'generation[[:space:]]*=[[:space:]]*[0-9]+' | ", + "grep -oE '[0-9]+' | tail -1) && ", + "test -n \"$session_generation_after\" && ", + "test \"$session_generation_after\" -gt \"$session_generation_before\" && ", + "jq -c '{guestRef, guestUid, zone, reconnectGeneration, ", + "providerGeneration, controllerGeneration, assignmentEpoch}' ", + "/var/lib/d2b/zones/work/guests/acceptance-guest/component-session/guest.json ", + ">/run/d2b-guest-session-after.json && ", + "jq -e --slurpfile expected /run/d2b-guest-session-before.json ", + "'. == $expected[0]' /run/d2b-guest-session-after.json >/dev/null && ", + "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock ", + "d2b --zone work --json list Guest ", + ">/run/d2b-guest-adopted.json && ", + "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock ", + "d2b --zone work --json list Process ", + ">/run/d2b-process-adopted.json && ", + "jq -e '", + "([.resources[] | select(.type == \"Process\" and ", + ".metadata.ownerRef == \"Guest/acceptance-guest\")] | length == 1) and ", + "([.resources[] | select(.type == \"Process\" and ", + ".metadata.name == \"acceptance-guest-vmm\" and ", + ".metadata.ownerRef == \"Guest/acceptance-guest\" and ", + ".spec.providerRef == \"Provider/system-minijail\" and ", + ".spec.executionRef == \"Host/host-system\" and ", + ".spec.processClass == \"worker\" and ", + ".spec.template == \"cloud-hypervisor-runner\" and ", + ".status.phase == \"Ready\")] | length == 1) and ", + "([.resources[] | select(.type == \"Process\" and ", + ".metadata.ownerRef == \"Provider/runtime-cloud-hypervisor\" and ", + ".spec.providerRef == \"Provider/system-minijail\" and ", + ".spec.executionRef == \"Host/host-system\" and ", + ".spec.processClass == \"controller\" and ", + ".spec.template == \"controller-runtime-cloud-hypervisor-cloud-hypervisor-controller\" ", + "and ", + ".status.phase == \"Ready\")] | length == 1)' ", + "/run/d2b-process-adopted.json && ", + "jq -e --slurpfile session /run/d2b-guest-session-after.json ", + "'any(.resources[]; ", + ".type == \"Guest\" and ", + ".metadata.name == \"acceptance-guest\" and ", + ".metadata.zone == \"work\" and ", + ".metadata.uid == $session[0].guestUid and ", + ".spec.providerRef == \"Provider/runtime-cloud-hypervisor\" and ", + ".spec.executionRef == \"Host/host-system\" and ", + ".status.phase == \"Ready\" and ", + ".status.observedGeneration == .metadata.generation and ", + ".status.resource.runtimeReady == true and ", + ".status.resource.bootstrapReady == true and ", + ".status.resource.activeProcessCount == 1)' ", + "/run/d2b-guest-adopted.json && exit 0; ", + "sleep 1; done; ", + "echo 'Guest ComponentSession generation did not advance after restart:'; ", + "printf 'before=%s after=%s\\n' ", + "\"$(cat /run/d2b-guest-session-generation-before 2>/dev/null || true)\" ", + "\"$(journalctl --no-pager -b 2>/dev/null | ", + "grep -F 'Guest ComponentSession Resource API server starting' | ", + "grep -oE 'generation[[:space:]]*=[[:space:]]*[0-9]+' | ", + "grep -oE '[0-9]+' | tail -1)\"; ", + "jq -c '.' /run/d2b-guest-session-before.json || true; ", + "jq -c '.' /run/d2b-guest-session-after.json || true; ", + "echo 'Post-restart Guest readiness failed:'; ", + "jq -c '.resources[] | select(.type == \"Guest\" and ", + ".metadata.name == \"acceptance-guest\") | ", + "{name: .metadata.name, uid: .metadata.uid, ", + "owner: .metadata.ownerRef, phase: .status.phase, ", + "observedGeneration: .status.observedGeneration, ", + "conditions: [.status.conditions[]? | ", + "{type: .type, status: .status, reason: .reason}], ", + "outcome: (.status.outcome | ", + "if . == null then null else ", + "{code: .code, retryable: .retryable} end), ", + "resource: .status.resource}' ", + "/run/d2b-guest-adopted.json || true; ", + "echo 'Post-restart Process readiness failed:'; ", + "jq -c '.resources[] | ", + "{name: .metadata.name, uid: .metadata.uid, ", + "owner: .metadata.ownerRef, provider: .spec.providerRef, ", + "execution: .spec.executionRef, processClass: .spec.processClass, ", + "template: .spec.template, phase: .status.phase, ", + "observedGeneration: .status.observedGeneration, ", + "conditions: [.status.conditions[]? | ", + "{type: .type, status: .status, reason: .reason}], ", + "outcome: (.status.outcome | ", + "if . == null then null else ", + "{code: .code, retryable: .retryable} end), ", + "resource: .status.resource}' ", + "/run/d2b-process-adopted.json || true; ", + "exit 1", +); + +/// The Guest's deletion, retried the fixture's own 30 attempts. +const GUEST_DELETE: &str = concat!( + "for attempt in $(seq 1 30); do ", + "guest_revision=$(runuser -u alice -- env ", + "D2B_PUBLIC_SOCKET=/run/d2b/public.sock ", + "d2b --zone work --json list Guest ", + "| jq -er '.resources[] | select(.type == \"Guest\" and ", + ".metadata.name == \"acceptance-guest\") | .metadata.revision') && ", + "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock ", + "d2b --zone work --json delete Guest/acceptance-guest ", + "--revision \"$guest_revision\" ", + ">/run/d2b-guest-delete.json 2>/run/d2b-guest-delete.err && exit 0; ", + "sleep 1; done; ", + "echo 'Guest deletion did not complete within 30s:'; ", + "jq -c '{resourceRef: .resourceRef, revision: .revision}' ", + "/run/d2b-guest-delete.json || true; ", + "echo 'last delete stderr:'; ", + "cat /run/d2b-guest-delete.err 2>/dev/null || true; ", + "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock ", + "d2b --zone work --json list Guest | ", + "jq -c '.resources[] | select(.type == \"Guest\" and ", + ".metadata.name == \"acceptance-guest\") | ", + "{name: .metadata.name, uid: .metadata.uid, ", + "owner: .metadata.ownerRef, phase: .status.phase, ", + "observedGeneration: .status.observedGeneration, ", + "conditions: [.status.conditions[]? | ", + "{type: .type, status: .status, reason: .reason}], ", + "outcome: (.status.outcome | ", + "if . == null then null else ", + "{code: .code, retryable: .retryable} end), ", + "resource: .status.resource}' || true; ", + "exit 1", +); + +/// The deletion was accepted for the Guest the check deleted. +const GUEST_DELETE_REVISION: &str = concat!( + "jq -e '.resourceRef == \"Guest/acceptance-guest\" and ", + ".revision > 0' ", + "/run/d2b-guest-delete.json", +); + +/// The Guest's deletion was recorded, so its drain has begun. +const GUEST_DRAINING: &str = concat!( + "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock ", + "d2b --zone work --json reconcile Guest/acceptance-guest ", + ">/run/d2b-guest-finalize.json 2>/dev/null || true; ", + "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock ", + "d2b --zone work --json list Guest ", + ">/run/d2b-guest-draining.json && ", + "jq -e 'any(.resources[]; .type == \"Guest\" and ", + ".metadata.name == \"acceptance-guest\" and ", + ".metadata.deletionRequestedAt != null)' ", + "/run/d2b-guest-draining.json", +); + +/// The Guest row is gone. +const GUEST_DRAINED: &str = concat!( + "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock ", + "d2b --zone work --json reconcile Guest/acceptance-guest ", + ">/run/d2b-guest-finalize.json 2>/dev/null || true; ", + "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock ", + "d2b --zone work --json list Guest ", + "| jq -e 'all(.resources[]; .metadata.name != \"acceptance-guest\")'", +); + +/// The Guest's VMM runner Process is gone. +const GUEST_VMM_PROCESS_DRAINED: &str = concat!( + "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock ", + "d2b --zone work --json list Process ", + "| jq -e 'all(.resources[]; .metadata.name != \"acceptance-guest-vmm\")'", +); + +/// The VMM's API socket is gone once the Guest drained. +const GUEST_API_SOCKET_GONE: &str = + "test ! -S /var/lib/d2b/zones/work/guests/acceptance-guest/acceptance-guest.sock"; + +/// The owning Volume's deletion, which drives the binding's drain. +const VOLUME_STATE_DELETE: &str = concat!( + "volume_revision=$(runuser -u alice -- env ", + "D2B_PUBLIC_SOCKET=/run/d2b/public.sock ", + "d2b --zone work --json list Volume | ", + "jq -er '.resources[] | select(.type == \"Volume\" and ", + ".metadata.name == \"state\") | .metadata.revision') && ", + "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock ", + "d2b --zone work --json delete Volume/state ", + "--revision \"$volume_revision\" >/run/d2b-volume-state-delete.json", +); + +/// The binding's deletion was recorded, so its drain has begun. +const VOLUME_BINDING_DRAINING: &str = concat!( + "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock ", + "d2b --zone work --json list VolumeBinding ", + ">/run/d2b-binding-draining.json && ", + "jq -e 'any(.resources[]; .type == \"VolumeBinding\" and ", + ".metadata.name == \"vol-binding-6a8ea4307a30f7ceae6533f2\" and ", + ".metadata.deletionRequestedAt != null)' ", + "/run/d2b-binding-draining.json", +); + +/// The binding, its worker Process and its endpoint are all gone. +const VOLUME_BINDING_DRAINED: &str = concat!( + "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock ", + "d2b --zone work --json list VolumeBinding ", + ">/run/d2b-binding-drained.json && ", + "jq -e 'all(.resources[]; .metadata.ownerRef != \"Volume/state\")' ", + "/run/d2b-binding-drained.json && ", + "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock ", + "d2b --zone work --json list Process | ", + "jq -e 'all(.resources[]; .metadata.ownerRef != ", + "\"VolumeBinding/vol-binding-6a8ea4307a30f7ceae6533f2\")' && ", + "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock ", + "d2b --zone work --json list Endpoint | ", + "jq -e 'all(.resources[]; .metadata.ownerRef != ", + "\"VolumeBinding/vol-binding-6a8ea4307a30f7ceae6533f2\")'", +); + +/// The check's assertions, in the order its fixture made them. +pub fn assertions(control: &mut GuestControl) -> LegacyResult<()> { + control.stage("boot"); + control.diag_unit("daemon-up", "d2bd.service", DAEMON_BOUND)?; + control.wait_for_unit("d2b-broker.socket", None, SOCKET_BOUND)?; + control.wait_for_file("/run/d2b/public.sock", SOCKET_BOUND)?; + control.succeed(&[BROKER_START], None)?; + control.diag_unit("broker-service", "d2b-broker.service", BROKER_BOUND)?; + + // Capture only the public, redacted Resource projection before waiting on + // the nested VMM. This keeps a missing API socket diagnostic without + // waiting for unrelated fixture controller sessions. + control.stage("preflight-capture"); + control.succeed(&[PREFLIGHT_CAPTURE], None)?; + control.diag("cat /run/d2b-preflight-summary.log", "preflight summary"); + + // Both storage Providers use the authenticated host acceptance controller. + // Their separate owner identities must establish live ResourceV3 sessions; + // a stale pause fixture would leave these controller Processes pending. + let summary = summary_rows(); + let process_rows = live_rows("Process rows", "Process"); + let volume_rows = live_rows("Volume rows", "Volume"); + let guest_rows = live_rows("Guest rows", "Guest"); + let state_dir = ("guest state dir", GUEST_STATE_DIR); + let controller_rows = saved_rows( + "Controller Process rows", + "/run/d2b-volume-controller-processes.json", + ); + control.diag_wait( + "controller-sessions", + CONTROLLER_SESSIONS, + CONTROLLER_BOUND, + &[row(&summary), row(&process_rows)], + &[("d2bd.service", "ResourceV3 session")], + )?; + // The rows hold the phase and generation the status was published for, + // per controller row. + control.diag_wait( + "volume-controller-processes", + VOLUME_CONTROLLER_PROCESSES, + CONTROLLER_BOUND, + &[row(&controller_rows), row(&summary)], + &[("d2bd.service", "acceptance-controller")], + )?; + control.succeed(&[ACCEPTANCE_CONTROLLERS], None)?; + + // The VMM API socket is the first nested-VM proof. Volume convergence can + // legitimately precede the nested boot, so keep this bound aligned with + // Guest readiness rather than failing before the U7 Runner re-enters. + control.stage("nested-vmm-api-socket"); + control.succeed(&[NESTED_VMM_API_SOCKET], None)?; + control.diag_wait( + "guest-console-boot-id", + GUEST_CONSOLE_BOOT_ID, + CONSOLE_BOUND, + &[("host journal tail", HOST_JOURNAL_TAIL)], + &[("", "D2B_GUEST_BOOT_ID")], + )?; + control.sleep(5)?; + + // The Guest's enrollment, then the host posture the nested boot needs. + control.stage("guest-session-enrollment"); + control.succeed(&[GUEST_SESSION_ENROLLMENT], None)?; + control.succeed(&[KVM_CAPABILITY], None)?; + control.succeed(&[VHOST_NET_CAPABILITY], None)?; + control.succeed(&[CGROUP_V2_CAPABILITY], None)?; + control.succeed(&[CGROUP_CONTROLLERS], None)?; + control.succeed(&[DELEGATED_CGROUP_POSTURE], None)?; + control.succeed(&[BUNDLE_RESOLVER_LOADED], None)?; + control.succeed(&[GUEST_SETUP_DESCRIPTOR], None)?; + control.succeed(&[GUEST_CLOSURE], None)?; + control.succeed(&[RESOURCE_BUNDLE], None)?; + + // The Guest's readiness, then each of its rows in the order the fixture + // declared them: the VMM and controller processes, the endpoints, the + // system Volume, the store view, and the declared Volume's binding. + control.stage("guest-ready"); + control.succeed(&[GUEST_READY], None)?; + let process_ready_rows = saved_rows("Process rows", "/run/d2b-process-ready.json"); + control.diag_wait( + "guest-vmm-process-ready", + GUEST_VMM_PROCESS_READY, + GUEST_BOUND, + &[row(&process_ready_rows)], + &[ + ("d2bd.service", "acceptance-guest-vmm"), + ("d2bd.service", "cloud-hypervisor-runner"), + ], + )?; + let endpoint_ready_rows = saved_rows("Endpoint rows", "/run/d2b-endpoint-ready.json"); + control.diag_wait( + "guest-endpoints-ready", + GUEST_ENDPOINTS_READY, + GUEST_BOUND, + &[row(&endpoint_ready_rows)], + &[("d2bd.service", "Guest/acceptance-guest")], + )?; + let volume_ready_rows = saved_rows("Volume rows", "/run/d2b-volume-ready.json"); + control.diag_wait( + "guest-volume-ready", + GUEST_VOLUME_READY, + GUEST_VOLUME_BOUND, + &[row(&volume_ready_rows)], + &[("d2bd.service", "acceptance-guest-system")], + )?; + control.succeed(&[STORE_VIEW_VOLUME], None)?; + control.succeed(&[STORE_VIEW_UIDS], None)?; + + // U7: the declared Volume/state attachment is served end to end through + // the neutral binding chain. The Volume side mints exactly one + // deterministically named binding owned by the Volume, and only a + // current fence (binding UID and generation) can report it ready. + let binding_ready_rows = saved_rows("VolumeBinding rows", "/run/d2b-binding-ready.json"); + control.diag_wait( + "volume-binding-ready", + VOLUME_BINDING_READY, + BINDING_BOUND, + &[row(&binding_ready_rows), row(&volume_rows)], + &[("d2bd.service", "vol-binding-6a8ea4307a30f7ceae6533f2")], + )?; + // The virtiofs serving side owns only its worker Process and private + // Endpoint as binding-owned children; the worker adopts the per-Volume + // vfd principal synthesized from the declared attachment. + let binding_worker_rows = saved_rows("Process rows", "/run/d2b-binding-worker.json"); + let binding_endpoint_rows = saved_rows("Endpoint rows", "/run/d2b-binding-endpoint.json"); + control.diag_wait( + "binding-worker-ready", + BINDING_WORKER_READY, + BINDING_WORKER_BOUND, + &[row(&binding_worker_rows), row(&binding_endpoint_rows)], + &[ + ("d2bd.service", "vol-binding-6a8ea4307a30f7ceae6533f2"), + ("d2bd.service", "virtiofsd"), + ], + )?; + control.diag_wait( + "guest-api-socket", + GUEST_API_SOCKET, + API_SOCKET_BOUND, + &[row(&summary), (state_dir.0, state_dir.1)], + &[("d2bd.service", "acceptance-guest")], + )?; + control.succeed(&[GUEST_STATE_CHAIN], None)?; + + // The runner process the restart below has to find again: its pid and its + // start time, so the adoption is about the same process and not about a + // search that found a replacement. + let runner = control.succeed(&[RUNNER_PROCESS], None)?; + let runner_fields = runner_fields(&runner)?; + let runner_pid = runner_fields[0]; + let runner_start = runner_fields[1]; + control.succeed(&[&format!("test -d /proc/{runner_pid}")], None)?; + control.succeed( + &[&format!("test \"$(awk '{{print $22}}' /proc/{runner_pid}/stat)\" = {runner_start}")], + None, + )?; + control.succeed( + &[&format!("tr '\\0' ' ' < /proc/{runner_pid}/cmdline | grep -F -- '--api-socket' | \ + grep -F -- 'acceptance-guest'")], + None, + )?; + + // The enrollment record read back before the restart, and the failure the + // Guest's own console must not have reported. + control.succeed(&[GUEST_SESSION_BEFORE], None)?; + + // The Guest target agent boots from its enrolled bundle and key pair, and + // the Guest console is forwarded into the host journal: a read it cannot + // make fails closed inside the Guest, so the host journal must never + // carry that failure. This is the gate the shell-pool fixture cannot + // provide (no vsock device there). + control.fail(&[BUNDLE_VALIDATION_FAILED], None)?; + + // The restart boundary: the same runner process, and a Guest whose session + // generation advanced behind it. + control.stage("restart-adoption"); + control.succeed(&[DAEMON_RESTART], None)?; + control.diag_unit("daemon-restarted", "d2bd.service", DAEMON_BOUND)?; + control.wait_for_file("/run/d2b/public.sock", SOCKET_BOUND)?; + control.diag_wait( + "api-socket-after-restart", + API_SOCKET_AFTER_RESTART, + API_SOCKET_BOUND, + &[row(&guest_rows), (state_dir.0, state_dir.1)], + &[("d2bd.service", "acceptance-guest")], + )?; + control.succeed(&[&format!("test -d /proc/{runner_pid}")], None)?; + control.succeed( + &[&format!("test \"$(awk '{{print $22}}' /proc/{runner_pid}/stat)\" = {runner_start}")], + None, + )?; + control.stage("session-generation-advance"); + control.succeed(&[SESSION_GENERATION_ADVANCE], None)?; + control.succeed( + &[&format!("set -- $(for proc in /proc/[0-9]*; do exe=$(readlink \"$proc/exe\" \ + 2>/dev/null || true); case \"$exe\" in */bin/cloud-hypervisor) cmd=$(tr \ + '\\0' ' ' < \"$proc/cmdline\"); case \"$cmd\" in \ + *--api-socket*acceptance-guest*) pid=${{proc#/proc/}}; printf '%s %s ' \ + \"$pid\" \"$(awk '{{print $22}}' \"$proc/stat\")\";; esac;; esac; \ + done); test \"$#\" -eq 2 && test \"$1\" = {runner_pid} && test \"$2\" = \ + {runner_start}")], + None, + )?; + + // The Guest's teardown, then the Volume's: both drain, and neither leaves + // a socket, a process or a binding behind. + control.stage("guest-teardown"); + control.succeed(&[GUEST_DELETE], None)?; + control.succeed(&[GUEST_DELETE_REVISION], None)?; + let draining_rows = saved_rows("Guest rows", "/run/d2b-guest-draining.json"); + control.diag_wait( + "guest-draining", + GUEST_DRAINING, + DRAINING_BOUND, + &[row(&draining_rows), row(&summary)], + &[("d2bd.service", "acceptance-guest")], + )?; + control.diag_wait( + "guest-drained", + GUEST_DRAINED, + DRAINED_BOUND, + &[row(&guest_rows)], + &[("d2bd.service", "acceptance-guest")], + )?; + control.diag_wait( + "guest-vmm-process-drained", + GUEST_VMM_PROCESS_DRAINED, + GUEST_BOUND, + &[row(&process_rows)], + &[("d2bd.service", "acceptance-guest-vmm")], + )?; + control.succeed(&[GUEST_API_SOCKET_GONE], None)?; + + // U7 teardown (F2/AE6): deleting the owning Volume drives the binding + // through its drain: deletion is requested first, then the worker and + // the private endpoint are gone before the binding disappears, leaving + // no orphaned serving effects. + control.stage("volume-teardown"); + control.succeed(&[VOLUME_STATE_DELETE], None)?; + let binding_draining_rows = saved_rows( + "VolumeBinding rows", + "/run/d2b-binding-draining.json", + ); + control.diag_wait( + "volume-binding-draining", + VOLUME_BINDING_DRAINING, + DRAINING_BOUND, + &[row(&binding_draining_rows)], + &[("d2bd.service", "vol-binding-6a8ea4307a30f7ceae6533f2")], + )?; + let binding_drained_rows = saved_rows( + "VolumeBinding rows", + "/run/d2b-binding-drained.json", + ); + control.diag_wait( + "volume-binding-drained", + VOLUME_BINDING_DRAINED, + BINDING_DRAIN_BOUND, + &[ + row(&binding_drained_rows), + row(&process_rows), + row(&live_rows("Endpoint rows", "Endpoint")), + ], + &[("d2bd.service", "vol-binding-6a8ea4307a30f7ceae6533f2")], + )?; + + Ok(()) +} + +/// The live rows of one resource type, as the fixture's own `live_rows` built +/// them: a labelled jq projection of what the public surface answers. +fn live_rows(label: &str, resource_type: &str) -> (String, String) { + ( + label.to_owned(), + format!( + "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock \ + d2b --zone work --json list {resource_type} 2>/dev/null | \ + jq -c '{DIAG_PROJECTION}' 2>/dev/null || true" + ), + ) +} + +/// The rows of a file the check just saved, as the fixture's own `saved_rows` +/// built them: the same projection, falling back to the file itself. +fn saved_rows(label: &str, path: &str) -> (String, String) { + ( + label.to_owned(), + format!( + "jq -c '{DIAG_PROJECTION}' {path} 2>/dev/null \ + || cat {path} 2>/dev/null || true" + ), + ) +} + +/// The preflight capture's own rows, as the fixture's `summary_rows` built +/// them. +fn summary_rows() -> (String, String) { + ( + "preflight summary".to_owned(), + "cat /run/d2b-preflight-summary.log 2>/dev/null || true".to_owned(), + ) +} + +/// One row of a labelled pair, as the diagnostics rows are passed. +fn row<'a>(pair: &'a (String, String)) -> DiagRow<'a> { + (pair.0.as_str(), pair.1.as_str()) +} + +/// The pid and the start time the runner search answered with, unpacked the +/// way the fixture unpacked them - on the same terms, too: a search that +/// answered with any other number of fields is the failure the fixture's own +/// `runner.split()` raised. +fn runner_fields(runner: &str) -> LegacyResult> { + let fields = runner.split_whitespace().collect::>(); + if fields.len() == 2 { + return Ok(fields); + } + let complaint = if fields.len() < 2 { + "not enough values to unpack" + } else { + "too many values to unpack" + }; + Err(LegacyError::Assertion(format!( + "{complaint} (expected 2, got {})", + fields.len() + ))) +} diff --git a/packages/d2b-test-vm-harness/src/checks/state_posture_contract.rs b/packages/d2b-test-vm-harness/src/checks/state_posture_contract.rs index 4f86a3d7b..df08c7f06 100644 --- a/packages/d2b-test-vm-harness/src/checks/state_posture_contract.rs +++ b/packages/d2b-test-vm-harness/src/checks/state_posture_contract.rs @@ -656,10 +656,11 @@ fn check_level( // re-scopes the whole group class. let declared_mask = declared_acl .iter() - .any(|spec| spec.splitn(3, ':').next() == Some("m")); + .any(|spec| spec.split(':').next() == Some("m")); let mask_entry = entries.iter().find(|entry| entry.starts_with("m::")); - if !declared_mask { - if let Some(mask_entry) = mask_entry { + if !declared_mask + && let Some(mask_entry) = mask_entry + { let group_entry = entries.iter().find(|entry| entry.starts_with("g::")); let mut expected_mask = group_entry .map(|entry| permission_bits(entry.splitn(3, ':').nth(2).unwrap_or(""))) @@ -678,7 +679,6 @@ fn check_level( named grants, observed {mask_entry}" ))); } - } } // The declared rights: an expectation of `preserve` or `not-required` is diff --git a/packages/d2b-test-vm-harness/src/checks/virtiofsd_volume_runtime.rs b/packages/d2b-test-vm-harness/src/checks/virtiofsd_volume_runtime.rs index 39392d954..1de3c790a 100644 --- a/packages/d2b-test-vm-harness/src/checks/virtiofsd_volume_runtime.rs +++ b/packages/d2b-test-vm-harness/src/checks/virtiofsd_volume_runtime.rs @@ -27,9 +27,9 @@ //! outliving its parent. //! //! The guest is the reusable daemon node plus the fixture's own contributions -//! - nftables, the acceptance host runtime, the Volume acceptance artifact -//! and its publisher key, the two zones and their rows, the two declared -//! users and the operator role binding - declared in +//! (nftables, the acceptance host runtime, the Volume acceptance artifact and +//! its publisher key, the two zones and their rows, the two declared users and +//! the operator role binding), declared in //! `nix/test-support/host-integration-node.nix`. `start_all()` is not //! restated here: it is the lane's own boot of the guest the check runs //! against. diff --git a/tests/host-integration/runtime-cloud-hypervisor-guest-preflight.nix b/tests/host-integration/runtime-cloud-hypervisor-guest-preflight.nix deleted file mode 100644 index 72fa82726..000000000 --- a/tests/host-integration/runtime-cloud-hypervisor-guest-preflight.nix +++ /dev/null @@ -1,1295 +0,0 @@ -# Type-G runNixOSTest: Zone-native Cloud Hypervisor Guest acceptance. -# -# This is the public host selector for the controller-owned Guest lifecycle. -# It requires the nested KVM posture and fails closed when the host cannot -# provide it; an environment block is not acceptance evidence. -{ pkgs, self }: - -let - inherit (pkgs) lib; - d2bLib = import ./lib.nix { - inherit self; - inherit lib; - hostToolBundle = - if self.lib ? d2bHostToolBundle then self.lib.d2bHostToolBundle else null; - }; - # The reusable guest configuration lives outside this directory so it - # survives the fixture (see `nix/test-support/host-integration-node.nix`). - d2bNode = import ../../nix/test-support/host-integration-node.nix { - inherit self; - inherit lib; - }; - cloudHypervisorArtifact = - d2bLib.mkRuntimeCloudHypervisorArtifact pkgs; - volumeProviderArtifact = d2bLib.mkVolumeProviderArtifact pkgs; - fixtureKeys = pkgs.runCommand "acceptance-component-session-keys" { } '' - mkdir -p "$out" - printf '\001\002\003\004\005\006\007\010\011\012\013\014\015\016\017\020\021\022\023\024\025\026\027\030\031\032\033\034\035\036\037\040' > "$out/host.key" - printf '\007\243\174\274\024\040\223\310\267\125\334\033\020\350\154\264\046\067\112\321\152\250\123\355\013\337\300\262\270\155\034\174' > "$out/host.pub" - printf '\041\042\043\044\045\046\047\050\051\052\053\054\055\056\057\060\061\062\063\064\065\066\067\070\071\072\073\074\075\076\077\100' > "$out/guest.key" - printf '\130\151\257\364\120\124\227\062\313\252\355\136\135\371\263\012\155\243\034\260\345\164\053\255\132\324\241\247\150\361\246\173' > "$out/guest.pub" - ''; - guestBundle = pkgs.runCommand "acceptance-guest-bundle" { - nativeBuildInputs = [ pkgs.python3 ]; - } '' - mkdir -p "$out" - cat > "$out/host.json" <<'EOF' - {"schemaVersion":"v2","site":{"allowUnsafeEastWest":false},"environments":[],"nftables":{"family":"inet","table":"d2b","chains":[],"tableHashAfterApply":null,"ownershipId":"host-integration"},"networkManager":{"filePath":"/etc/NetworkManager/conf.d/00-d2b-unmanaged.conf","matchCriteria":[],"reloadBehavior":"atomic-reload","ownership":{"owner":"root","group":"root","mode":"0644","driftPolicy":"replace"}},"hostsFile":{"startMarker":"# d2b-managed begin","endMarker":"# d2b-managed end","rule":"replace-managed-block"},"kernelModules":[],"fdOwnership":[],"cloudHypervisorCapabilities":[],"ifNameMappings":[],"ch":null,"firewallCoexistencePolicy":null} - EOF - printf '%s\n' '{"schemaVersion":"v2","vms":[]}' > "$out/processes.json" - printf '%s\n' '{"schemaVersion":"v2","publicOperations":[],"brokerOperations":[]}' > "$out/privileges.json" - printf '%s\n' '{"_manifest":{"manifestVersion":6},"_observability":{"enabled":false,"signozUrl":"http://127.0.0.1:8080","signozOtlpGrpcPort":4317,"signozOtlpHttpPort":4318,"obsVsockCid":0,"obsVsockHostSocket":"","vmName":""}}' > "$out/vms.json" - python3 - "$out/bundle.json" <<'PY' - import hashlib - import json - import sys - - # Zone-native v3 bundle: the loader (BundleResolver) accepts only the - # v3 contract. The self-hash is computed over the serialization with - # bundleHash absent and artifactHashes nullified (verify_bundle_hash). - bundle = { - "artifactHashes": {}, - "bundleVersion": 1, - "schemaVersion": "v3", - "privilegesPath": "privileges.json", - "zones": [], - "generation": { - "generatedAt": None, - "generator": "host-integration", - "sourceRevision": None, - }, - } - preimage = dict(bundle) - preimage["artifactHashes"] = None - canonical = json.dumps(preimage, sort_keys=True, separators=(",", ":")).encode() - bundle["bundleHash"] = "sha256:" + hashlib.sha256(canonical).hexdigest() - with open(sys.argv[1], "w", encoding="utf-8") as output: - json.dump(bundle, output, sort_keys=True, separators=(",", ":")) - output.write("\n") - PY - ''; - - cloudHypervisorConfig = { - controllerExecutionRef = "Host/host-system"; - defaultVcpus = 2; - defaultMemoryMb = 512; - defaultMachineType = "microvm"; - watchdog = true; - adoptionWindowMs = 30000; - healthCheckIntervalMs = 5000; - healthCheckTimeoutMs = 1000; - healthCheckFailureThreshold = 3; - startupDeadlineMs = 120000; - }; - guestSystem = d2bLib.mkGuestSystem { - inherit pkgs; - name = "acceptance-guest"; - modules = [ - ({ lib, name, ... }: { - boot.kernelParams = [ "console=ttyS0" "loglevel=7" ]; - environment.etc."d2b/component-session/guest.key".source = - "${fixtureKeys}/guest.key"; - environment.etc."d2b/component-session/parent.pub".source = - "${fixtureKeys}/host.pub"; - systemd.services.d2bd-guest = { - environment = { - RUST_LOG = "d2bd=debug"; - }; - serviceConfig = { - ReadOnlyPaths = [ - "/etc/d2b/component-session/guest.key" - "/etc/d2b/component-session/parent.pub" - ]; - StandardOutput = lib.mkForce "journal+console"; - StandardError = lib.mkForce "journal+console"; - }; - }; - systemd.services.d2b-test-boot-identity = { - wantedBy = [ "basic.target" ]; - before = [ "d2bd-guest.service" ]; - serviceConfig.Type = "oneshot"; - script = '' - printf 'D2B_GUEST_BOOT_ID=%s\n' \ - "$(${pkgs.coreutils}/bin/cat /proc/sys/kernel/random/boot_id)" \ - > /dev/console - ''; - }; - d2b.componentSession.localPrivateKeyPath = - "/etc/d2b/component-session/guest.key"; - d2b.componentSession.parentPublicKeyPath = - "/etc/d2b/component-session/parent.pub"; - d2b.componentSession.bundlePath = - "/var/lib/d2b/guest-bundle/bundle.json"; - d2b.guestBroker.bundlePath = - "/var/lib/d2b/guest-bundle/bundle.json"; - systemd.services.d2b-install-guest-bundle = { - requiredBy = [ "d2b-broker-guest.service" "d2bd-guest.service" ]; - before = [ "d2b-broker-guest.service" "d2bd-guest.service" ]; - serviceConfig.Type = "oneshot"; - script = '' - install -d -o root -g d2bd -m 0750 /var/lib/d2b/guest-bundle - for file in bundle.json host.json processes.json privileges.json; do - install -o root -g d2bd -m 0640 \ - ${guestBundle}/"$file" /var/lib/d2b/guest-bundle/"$file" - done - install -o root -g d2bd -m 0644 \ - ${guestBundle}/vms.json /var/lib/d2b/guest-bundle/vms.json - ''; - }; - networking.useDHCP = lib.mkForce false; - networking.networkmanager.enable = lib.mkForce false; - systemd.network.enable = lib.mkForce false; - services.dbus.enable = lib.mkForce false; - services.resolved.enable = lib.mkForce false; - systemd.services.systemd-vconsole-setup.enable = false; - d2b.vms.${name}.runner = { - store.onDisk = true; - store.disk = guestStoreDisk; - shares = lib.mkForce [ ]; - }; - fileSystems."/nix/store" = { - device = "/dev/vda"; - fsType = "ext4"; - options = [ "ro" "x-initrd.mount" ]; - neededForBoot = true; - }; - }) - ]; - }; - guestClosure = pkgs.closureInfo { - rootPaths = [ guestSystem.config.system.build.toplevel ]; - }; - guestStoreDisk = pkgs.runCommand "acceptance-guest-store.img" { - nativeBuildInputs = [ pkgs.coreutils pkgs.e2fsprogs ]; - } '' - mkdir -p root - while IFS= read -r path; do - cp -r --no-preserve=ownership,xattr,context "$path" root/ - done < ${guestClosure}/store-paths - truncate -s 4096M "$out" - # Reproducible ext4 image: SOURCE_DATE_EPOCH pins the superblock times and - # a fixed UUID seed pins the htree hash seed (e2fsprogs ignores an all-zero - # seed and randomizes it), so every build is byte-identical. With a random - # seed each build differed, and the nixos-install closure spec (recorded - # from an earlier build) could never match the freshly built image. - SOURCE_DATE_EPOCH=0 mkfs.ext4 -q -F \ - -U 123e4567-e89b-12d3-a456-426614174000 \ - -E hash_seed=123e4567-e89b-12d3-a456-426614174000 \ - -d root "$out" - ''; - artifacts = { - runtime-cloud-hypervisor = { - inherit (cloudHypervisorArtifact) package type catalog; - }; - volume-acceptance-provider = { - inherit (volumeProviderArtifact) package type catalog; - }; - acceptance-system = { - package = guestSystem.config.system.build.toplevel; - type = "nixos-system"; - }; - }; -in -pkgs.testers.runNixOSTest { - name = "d2b-runtime-cloud-hypervisor-guest-preflight"; - - nodes.machine = d2bNode.d2bCloudHypervisorNode { - extra = { ... }: { - d2b.site.adminUsers = [ "alice" ]; - environment.systemPackages = with pkgs; [ - iproute2 - jq - iputils - procps - ]; - d2b.artifacts = artifacts; - d2b.guestSystems.work.acceptance-guest = guestSystem; - d2b.zones.local-root.trustedPublishers.d2b-cloud-hypervisor.signingKey = - cloudHypervisorArtifact.trustedPublisher.signingKey; - d2b.zones.local-root.trustedPublishers.d2b-volume-acceptance.signingKey = - volumeProviderArtifact.trustedPublisher.signingKey; - d2b.zones.work.trustedPublishers.d2b-cloud-hypervisor.signingKey = - cloudHypervisorArtifact.trustedPublisher.signingKey; - d2b.zones.work.trustedPublishers.d2b-volume-acceptance.signingKey = - volumeProviderArtifact.trustedPublisher.signingKey; - d2b.zones.local-root.resources.host-system = { - type = "Host"; - spec = { - providerRef = "Provider/system-core"; - defaultDomain = "system"; - allowedDomains = [ "system" ]; - budget = { }; - networkAttachments = [ ]; - deviceAttachments = [ ]; - volumeAttachmentDefaults = [ ]; - }; - }; - d2b.zones.work = { - parentZone = "local-root"; - resources = { - alice = { - type = "User"; - spec = { - displayName = "Alice"; - groups = [ ]; - osUsername = "alice"; - }; - }; - d2bd = { - type = "User"; - spec = { - displayName = "d2bd"; - groups = [ ]; - osUsername = "d2bd"; - }; - }; - lifecycle-operator = { - type = "Role"; - spec.rules = [ - { - resourceTypes = [ "Endpoint" "Guest" "Host" "Process" "Provider" "Volume" "VolumeBinding" ]; - verbs = [ "get" "list" ]; - subresources = [ ]; - resourceNames = [ ]; - zones = [ "work" ]; - executionRefs = [ ]; - sessionVerbs = [ "connect" "invoke" ]; - } - { - resourceTypes = [ "Guest" ]; - verbs = [ "delete" ]; - subresources = [ ]; - resourceNames = [ "acceptance-guest" ]; - zones = [ "work" ]; - executionRefs = [ ]; - sessionVerbs = [ "connect" "invoke" ]; - } - { - resourceTypes = [ "Volume" ]; - verbs = [ "delete" ]; - subresources = [ ]; - resourceNames = [ "state" ]; - zones = [ "work" ]; - executionRefs = [ ]; - sessionVerbs = [ "connect" "invoke" ]; - } - ]; - }; - lifecycle-operator-binding = { - type = "RoleBinding"; - spec = { - roleRef = "Role/lifecycle-operator"; - subjects = [ "User/alice" ]; - externalPrincipalSelector = null; - scopeNarrowing = null; - }; - }; - host-system = { - type = "Host"; - spec = { - providerRef = "Provider/system-core"; - defaultDomain = "system"; - allowedDomains = [ "system" ]; - budget = { }; - networkAttachments = [ ]; - deviceAttachments = [ ]; - volumeAttachmentDefaults = [ ]; - }; - }; - volume-local = { - type = "Provider"; - spec = { - artifactId = "volume-acceptance-provider"; - config = { - controllerExecutionRef = "Host/host-system"; - sourcePolicies = [ - { - id = "default-state"; - class = "local-path"; - volumeKinds = [ "durable" "state" "cache" ]; - } - # U7: daemon-owned root the unprivileged daemon can - # lock and provision inline (path:daemon-state). - { - id = "daemon-state"; - class = "local-path"; - volumeKinds = [ "durable" "state" "cache" ]; - } - ]; - }; - }; - }; - volume-virtiofs = { - type = "Provider"; - spec = { - artifactId = "volume-acceptance-provider"; - config.controllerExecutionRef = "Host/host-system"; - }; - }; - state = { - type = "Volume"; - spec = { - providerRef = "Provider/volume-local"; - kind = "state"; - source = { - executionRef = "Host/host-system"; - settings = { - kind = "local-path"; - sourcePolicyId = "daemon-state"; - }; - }; - layout = [{ - path = "state"; - type = "directory"; - # U7: daemon-owned so the unprivileged daemon can - # provision inline; the guest share stays read-only. - ownerRef = "User/d2bd"; - groupRef = "User/d2bd"; - mode = "0700"; - target = null; - accessAcl = [ ]; - defaultAcl = [ ]; - foreignChildPolicy = "preserve"; - noFollow = true; - recursive = false; - sensitivity = "private"; - createPolicy = "create-if-never-provisioned"; - repairPolicy = "exact-owner"; - cleanupPolicy = "owner-controlled"; - adoptionPolicy = "quarantine-on-ambiguity"; - restartPolicy = "preserve-across-controller-restart"; - leaseClass = "none"; - invariants = [ "no-symlink" ]; - }]; - views.controller = { - path = ""; - rights = [ "read" "write" "traverse" ]; - }; - # KTD1: the attachment stays declared input only. The Volume - # side mints the durable VolumeBinding at reconcile; the - # deterministic binding identity below is - # vol-binding-6a8ea4307a30f7ceae6533f2 (volume, execution - # target, view, mount path). - attachments = [{ - executionRef = "Guest/acceptance-guest"; - transport = "virtiofs"; - view = "controller"; - access = "read-only"; - mountPath = "/state"; - settings = { - posixAcl = false; - xattr = false; - cache = "auto"; - inodeFileHandles = "never"; - threadPoolSize = null; - socketGroup = null; - }; - }]; - }; - }; - runtime-cloud-hypervisor = { - type = "Provider"; - spec = { - artifactId = "runtime-cloud-hypervisor"; - config = cloudHypervisorConfig; - }; - }; - acceptance-guest = { - type = "Guest"; - spec = { - providerRef = "Provider/runtime-cloud-hypervisor"; - executionRef = "Host/host-system"; - systemArtifactId = "acceptance-system"; - defaultDomain = "system"; - allowedDomains = [ "system" ]; - budget = { }; - volumeAttachmentDefaults = [ ]; - networkAttachments = [ ]; - deviceAttachments = [ ]; - }; - }; - }; - }; - }; - }; - - testScript = '' - ${d2bLib.fixtureDiagnostics} - - # Row projections the shared diagnostics print on a timed-out wait; they - # mirror the fields the waits assert on (issue #513). - diag_projection = ( - "[.resources[] | {type: .type, name: .metadata.name, " - "owner: .metadata.ownerRef, uid: .metadata.uid, " - "gen: .metadata.generation, phase: .status.phase, " - "obs: .status.observedGeneration, " - "provider: .spec.providerRef, execution: .spec.executionRef, " - "processClass: .spec.processClass, template: .spec.template, " - "conditions: [.status.conditions[]? | " - "{type: .type, status: .status, reason: .reason}], " - "outcome: (.status.outcome | " - "if . == null then null else " - "{code: .code, retryable: .retryable} end), " - "resource: .status.resource}]" - ) - - def live_rows(label, resource_type): - return ( - label, - "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - f"d2b --zone work --json list {resource_type} 2>/dev/null | " - f"jq -c '{diag_projection}' 2>/dev/null || true", - ) - - def saved_rows(label, path): - return ( - label, - f"jq -c '{diag_projection}' {path} 2>/dev/null " - f"|| cat {path} 2>/dev/null || true", - ) - - def summary_rows(): - return ( - "preflight summary", - "cat /run/d2b-preflight-summary.log 2>/dev/null || true", - ) - - start_all() - stage("boot") - diag_unit("daemon-up", "d2bd.service", 180) - machine.wait_for_unit("d2b-broker.socket", timeout=30) - machine.wait_for_file("/run/d2b/public.sock", timeout=30) - machine.succeed("systemctl start d2b-broker.service") - diag_unit("broker-service", "d2b-broker.service", 30) - - # Capture only the public, redacted Resource projection before waiting on - # the nested VMM. This keeps a missing API socket diagnostic without - # waiting for unrelated fixture controller sessions. - stage("preflight-capture") - machine.succeed( - "set -o pipefail; " - ": > /run/d2b-preflight-summary.log; " - "for resource_type in Guest Process Endpoint Volume Provider VolumeBinding; do " - "printf '%s: ' \"$resource_type\" >> /run/d2b-preflight-summary.log; " - "timeout 5s runuser -u alice -- env " - "D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - "d2b --zone work --json list \"$resource_type\" " - "2>/dev/null | " - "jq -c '[.resources[] | " - "{type: .type, " - "metadata: {name: .metadata.name, uid: .metadata.uid, " - "generation: .metadata.generation, ownerRef: .metadata.ownerRef, " - "zone: .metadata.zone}, " - "spec: {providerRef: .spec.providerRef, " - "executionRef: .spec.executionRef, " - "processClass: .spec.processClass, template: .spec.template}, " - "status: {phase: .status.phase, " - "observedGeneration: .status.observedGeneration, " - "conditions: [.status.conditions[]? | " - "{type: .type, status: .status, reason: .reason}], " - "outcome: (.status.outcome | " - "if . == null then null else " - "{code: .code, retryable: .retryable} end), " - "resource: .status.resource}}]' " - ">> /run/d2b-preflight-summary.log 2>/dev/null || " - "printf 'unavailable\\n' >> /run/d2b-preflight-summary.log; " - "done; " - "session_errors=$(journalctl -u d2bd.service --no-pager -b " - "2>/dev/null | grep -Ec " - "'session-authentication-failed|session-generation-stale' || true); " - "printf 'ComponentSession terminal error count: %s\\n' \"$session_errors\" " - ">> /run/d2b-preflight-summary.log; " - "cat /run/d2b-preflight-summary.log" - ) - diag( - "cat /run/d2b-preflight-summary.log", - "preflight summary", - ) - - # Both storage Providers use the authenticated host acceptance controller. - # Their separate owner identities must establish live ResourceV3 sessions; - # a stale pause fixture would leave these controller Processes pending. - diag_wait( - "controller-sessions", - "test \"$(journalctl -u d2bd.service --no-pager -o cat -b " - "| grep -Fc 'external Provider controller ResourceV3 session live')\" -ge 2", - # Cold artifact extraction inside a fresh VM varies widely on shared - # hardware; this waits an eventual state, not a timing SLO. - timeout=180, - rows=[summary_rows(), live_rows("Process rows", "Process")], - explain=[("d2bd.service", "ResourceV3 session")], - ) - diag_wait( - "volume-controller-processes", - "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - "d2b --zone work --json list Process " - ">/run/d2b-volume-controller-processes.json && " - "jq -e '" - "([.resources[] | select(.type == \"Process\" and " - ".metadata.ownerRef == \"Provider/volume-local\" and " - ".spec.providerRef == \"Provider/system-minijail\" and " - ".spec.processClass == \"controller\" and " - ".spec.template == \"controller-volume-acceptance-provider-acceptance-controller\" and " - ".status.phase == \"Ready\" and " - ".status.observedGeneration == .metadata.generation)] | length == 1) and " - "([.resources[] | select(.type == \"Process\" and " - ".metadata.ownerRef == \"Provider/volume-virtiofs\" and " - ".spec.providerRef == \"Provider/system-minijail\" and " - ".spec.processClass == \"controller\" and " - ".spec.template == \"controller-volume-acceptance-provider-acceptance-controller\" and " - ".status.phase == \"Ready\" and " - ".status.observedGeneration == .metadata.generation)] | length == 1)' " - "/run/d2b-volume-controller-processes.json", - # Same cold-start variance as the session wait above. - timeout=180, - # The rows hold the phase and generation the status was published for, - # per controller row. - rows=[ - saved_rows( - "Controller Process rows", - "/run/d2b-volume-controller-processes.json", - ), - summary_rows(), - ], - explain=[("d2bd.service", "acceptance-controller")], - ) - machine.succeed( - "test \"$(ps -eo pid=,args= | awk '$NF ~ /acceptance-controller$/ {print $1}' " - "| wc -l)\" -ge 2 && " - "! ps -eo args= | grep -E '(^|/)pause([[:space:]]|$)' | grep -v grep" - ) - - # The VMM API socket is the first nested-VM proof. Volume convergence can - # legitimately precede the nested boot, so keep this bound aligned with - # Guest readiness rather than failing before the U7 Runner re-enters. - stage("nested-vmm-api-socket") - machine.succeed( - "for attempt in $(seq 1 180); do " - "test -S /var/lib/d2b/zones/work/guests/acceptance-guest/acceptance-guest.sock " - "&& exit 0; " - "sleep 1; " - "done; " - "echo 'Cloud Hypervisor API socket did not become ready within 180s'; " - "cat /run/d2b-preflight-summary.log; " - "for resource_type in Volume VolumeBinding; do " - "echo \"=== $resource_type ===\"; " - "timeout 10s runuser -u alice -- env " - "D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - "d2b --zone work --json list \"$resource_type\" 2>/dev/null | " - "jq -c '.resources[] | {name: .metadata.name, phase: .status.phase, " - "observedGeneration: .status.observedGeneration, " - "conditions: [.status.conditions[]? | {type: .type, reason: .reason}], " - "ready: .status.resource.ready}' || true; " - "done; " - "echo '=== Process ==='; " - "timeout 10s runuser -u alice -- env " - "D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - "d2b --zone work --json list Process 2>/dev/null | " - "jq -c '.resources[] | select(.name | startswith(\"vol-vfd\")) | " - "{name: .metadata.name, phase: .status.phase, " - "conditions: [.status.conditions[]? | {type: .type, reason: .reason}], " - "outcome: .status.outcome, update: .status.update}' || true; " - "exit 1" - ) - diag_wait( - "guest-console-boot-id", - "journalctl --no-pager -b " - "| grep -q 'D2B_GUEST_BOOT_ID='", - timeout=30, - rows=[ - ( - "host journal tail", - "journalctl --no-pager -o cat -b -n 120 2>/dev/null || true", - ), - ], - explain=[(None, "D2B_GUEST_BOOT_ID")], - ) - machine.sleep(5) - stage("guest-session-enrollment") - machine.succeed( - "guest_uid=$(runuser -u alice -- env " - "D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - "d2b --zone work --json list Guest | " - "jq -er '.resources[] | select(.metadata.name == \"acceptance-guest\") " - "| .metadata.uid') && " - "boot_id=$(journalctl --no-pager -b " - "| sed -n 's/.*D2B_GUEST_BOOT_ID=\\([0-9a-f-]*\\).*/\\1/p' " - "| tail -1) && " - "boot_digest=$(printf 'd2b-kernel-boot-id-v1\\0%s' \"$boot_id\" " - "| sha256sum | cut -d' ' -f1) && " - "install -d -o d2bd -g d2bd -m 0700 " - "/var/lib/d2b/zones/work/guests/acceptance-guest/component-session && " - "install -o d2bd -g d2bd -m 0600 ${fixtureKeys}/host.key " - "/var/lib/d2b/zones/work/guests/acceptance-guest/component-session/host.key && " - "install -o d2bd -g d2bd -m 0600 ${fixtureKeys}/guest.pub " - "/var/lib/d2b/zones/work/guests/acceptance-guest/component-session/guest.pub && " - "cat > /var/lib/d2b/zones/work/guests/acceptance-guest/component-session/guest.json </dev/null " - "| grep -F 'Bundle resolver could not load'" - ) - - machine.succeed( - "test -r /etc/d2b/artifact-catalog.json && " - "jq -e '" - "(.guestSetupDescriptors | any(.[]; " - ".zone == \"work\" and .guest == \"acceptance-guest\" and " - ".providerArtifactId == \"runtime-cloud-hypervisor\" and " - ".descriptor.providerRef == \"Provider/runtime-cloud-hypervisor\" and " - ".descriptor.systemArtifactId == \"acceptance-system\" and " - ".descriptor.childRoles == [\"vmm\", \"ch-api\", \"guest-control\", \"system\"])) and " - "(.guestClosures | any(.[]; " - ".zone == \"work\" and .guest == \"acceptance-guest\" and " - ".artifactId == \"acceptance-system\" and (.closurePaths | length > 0) and " - "(. as $guest | ($guest.closurePaths | index($guest.toplevel)) != null) and " - ".storeView.mountPoint == \"/nix/store\" and " - "(.storeView.root | endswith(\"/zones/work/guests/acceptance-guest/store-view\")) and " - "(.vmm.binaryPath | endswith(\"/bin/cloud-hypervisor\"))))' " - "/etc/d2b/artifact-catalog.json" - ) - machine.succeed( - "test -r /etc/d2b/closures/zones/work/acceptance-guest.json && " - "jq -e '" - ".schemaVersion == \"v3\" and .artifactId == \"acceptance-system\" and " - "(.closurePaths | length > 0) and " - "(. as $guest | ($guest.closurePaths | index($guest.toplevel)) != null) and " - ".storeView.mountPoint == \"/nix/store\" and " - ".storeView.sync == \"broker-store-sync\" and " - "(.vmm.argv | index(\"--api-socket\")) != null' " - "/etc/d2b/closures/zones/work/acceptance-guest.json" - ) - machine.succeed( - "jq -e '" - ".resources | any(.[]; .type == \"Guest\" and " - ".metadata.name == \"acceptance-guest\" and " - ".spec.providerRef == \"Provider/runtime-cloud-hypervisor\" and " - ".spec.systemArtifactId == \"acceptance-system\") and " - "all(.[]; (tostring | contains(\"/nix/store/\") | not) and " - "(tostring | contains(\"\\\"argv\\\"\") | not))' " - "/etc/d2b/zones/work/resource-bundle.json" - ) - - stage("guest-ready") - machine.succeed( - "for attempt in $(seq 1 45); do " - "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - "d2b --zone work --json list Guest >/run/d2b-guest-ready.json && " - "jq -e '" - "(.resources | map(select(.type == \"Guest\" and " - ".metadata.name == \"acceptance-guest\"))) as $guests | " - "($guests | length) == 1 and " - "$guests[0].status.phase == \"Ready\" and " - "$guests[0].status.observedGeneration == $guests[0].metadata.generation and " - "$guests[0].status.resource.runtimeReady == true and " - "$guests[0].status.resource.bootstrapReady == true and " - "$guests[0].status.resource.activeProcessCount == 1' " - "/run/d2b-guest-ready.json && exit 0; " - "if jq -e 'any(.resources[]; " - ".metadata.name == \"acceptance-guest\" and " - ".status.phase == \"Failed\")' " - "/run/d2b-guest-ready.json >/dev/null; then " - "echo 'Guest reported a terminal failure'; exit 1; fi; " - "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - "d2b --zone work --json list Process >/run/d2b-vmm-fast-fail.json && " - "if jq -e 'any(.resources[]; " - ".metadata.name == \"acceptance-guest-vmm\" and " - ".status.phase == \"Failed\" and " - ".status.outcome.retryable != true)' " - "/run/d2b-vmm-fast-fail.json >/dev/null; then " - "echo 'VMM Process reported a terminal failure'; exit 1; fi; " - "if journalctl -u d2bd.service --no-pager -b " - "| grep -q 'session-authentication-failed\\|session-generation-stale'; then " - "echo 'ComponentSession reported a terminal failure'; exit 1; fi; " - "sleep 1; done; " - "echo 'Guest readiness failed:'; " - "jq -c '.resources[] | select(.type == \"Guest\" and " - ".metadata.name == \"acceptance-guest\") | " - "{name: .metadata.name, uid: .metadata.uid, " - "owner: .metadata.ownerRef, phase: .status.phase, " - "observedGeneration: .status.observedGeneration, " - "conditions: [.status.conditions[]? | " - "{type: .type, status: .status, reason: .reason}], " - "outcome: (.status.outcome | " - "if . == null then null else " - "{code: .code, retryable: .retryable} end), " - "resource: .status.resource}' " - "/run/d2b-guest-ready.json; " - "echo 'Dependent Process status:'; " - "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - "d2b --zone work --json list Process | " - "jq -c '.resources[] | " - "{name: .metadata.name, uid: .metadata.uid, " - "owner: .metadata.ownerRef, provider: .spec.providerRef, " - "execution: .spec.executionRef, processClass: .spec.processClass, " - "template: .spec.template, phase: .status.phase, " - "observedGeneration: .status.observedGeneration, " - "conditions: [.status.conditions[]? | " - "{type: .type, status: .status, reason: .reason}], " - "outcome: (.status.outcome | " - "if . == null then null else " - "{code: .code, retryable: .retryable} end), " - "resource: .status.resource}'; " - "for resource_type in Endpoint Volume Provider; do " - "echo \"$resource_type status:\"; " - "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - "d2b --zone work --json list \"$resource_type\" | " - "jq -c '.resources[] | {name: .metadata.name, uid: .metadata.uid, " - "owner: .metadata.ownerRef, provider: .spec.providerRef, " - "execution: .spec.executionRef, phase: .status.phase, " - "observedGeneration: .status.observedGeneration, " - "conditions: [.status.conditions[]? | " - "{type: .type, status: .status, reason: .reason}], " - "outcome: (.status.outcome | " - "if . == null then null else " - "{code: .code, retryable: .retryable} end), " - "resource: .status.resource}'; done; exit 1" - ) - diag_wait( - "guest-vmm-process-ready", - "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - "d2b --zone work --json list Process " - ">/run/d2b-process-ready.json && " - "jq -e '" - "([.resources[] | select(.type == \"Process\" and " - ".metadata.ownerRef == \"Guest/acceptance-guest\")] | length == 1) and " - "([.resources[] | select(.type == \"Process\" and " - ".metadata.name == \"acceptance-guest-vmm\" and " - ".metadata.ownerRef == \"Guest/acceptance-guest\" and " - ".spec.providerRef == \"Provider/system-minijail\" and " - ".spec.executionRef == \"Host/host-system\" and " - ".spec.processClass == \"worker\" and " - ".spec.template == \"cloud-hypervisor-runner\" and " - ".status.phase == \"Ready\")] | length == 1) and " - "([.resources[] | select(.type == \"Process\" and " - ".metadata.ownerRef == \"Provider/runtime-cloud-hypervisor\" and " - ".spec.providerRef == \"Provider/system-minijail\" and " - ".spec.executionRef == \"Host/host-system\" and " - ".spec.processClass == \"controller\" and " - ".spec.template == \"controller-runtime-cloud-hypervisor-cloud-hypervisor-controller\" and " - ".status.phase == \"Ready\")] | length == 1)' " - "/run/d2b-process-ready.json", - timeout=30, - rows=[saved_rows("Process rows", "/run/d2b-process-ready.json")], - explain=[ - ("d2bd.service", "acceptance-guest-vmm"), - ("d2bd.service", "cloud-hypervisor-runner"), - ], - ) - diag_wait( - "guest-endpoints-ready", - "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - "d2b --zone work --json list Endpoint " - ">/run/d2b-endpoint-ready.json && " - "jq -e '" - "([.resources[] | select(.type == \"Endpoint\" and " - ".metadata.ownerRef == \"Guest/acceptance-guest\" and " - ".status.phase == \"Ready\")] | length == 2)' " - "/run/d2b-endpoint-ready.json", - timeout=30, - rows=[saved_rows("Endpoint rows", "/run/d2b-endpoint-ready.json")], - explain=[("d2bd.service", "Guest/acceptance-guest")], - ) - diag_wait( - "guest-volume-ready", - "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - "d2b --zone work --json list Volume " - ">/run/d2b-volume-ready.json && " - "jq -e '" - "([.resources[] | select(.type == \"Volume\" and " - ".metadata.name == \"acceptance-guest-system\" and " - ".metadata.ownerRef == \"Guest/acceptance-guest\" and " - ".spec.source.settings.kind == \"nix-closure\" and " - ".spec.source.settings.sourcePolicyId == null and " - ".spec.source.settings.systemArtifactId == \"acceptance-system\" and " - ".status.phase == \"Ready\" and " - ".status.observedGeneration == .metadata.generation)] | length == 1)' " - "/run/d2b-volume-ready.json", - timeout=180, - rows=[saved_rows("Volume rows", "/run/d2b-volume-ready.json")], - explain=[("d2bd.service", "acceptance-guest-system")], - ) - machine.succeed( - "jq -e '" - "([.resources[] | select(.type == \"Volume\" and " - ".metadata.name == \"store-view-acceptance-guest\" and " - ".metadata.ownerRef == null and " - ".spec.source.settings.kind == \"nix-closure\" and " - ".spec.source.settings.sourcePolicyId == null and " - ".spec.source.settings.systemArtifactId == \"acceptance-system\" and " - ".status.phase == \"Ready\" and " - ".status.observedGeneration == .metadata.generation)] | length == 1)' " - "/run/d2b-volume-ready.json" - ) - machine.succeed( - "jq -e '" - "([.resources[] | select(.type == \"Volume\" and " - "(.metadata.name == \"store-view-acceptance-guest\" or " - ".metadata.name == \"acceptance-guest-system\")) " - "| .metadata.uid]) as $uids | " - "$uids | length == 2 and (unique | length == 2)' " - "/run/d2b-volume-ready.json" - ) - - # U7: the declared Volume/state attachment is served end to end through - # the neutral binding chain. The Volume side mints exactly one - # deterministically named binding owned by the Volume, and only a - # current fence (binding UID and generation) can report it ready. - diag_wait( - "volume-binding-ready", - "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - "d2b --zone work --json list VolumeBinding " - ">/run/d2b-binding-ready.json && " - "jq -e '" - "([.resources[] | select(.type == \"VolumeBinding\" and " - ".metadata.ownerRef == \"Volume/state\")] | length) == 1 and " - "([.resources[] | select(.type == \"VolumeBinding\" and " - ".metadata.name == \"vol-binding-6a8ea4307a30f7ceae6533f2\" and " - ".metadata.ownerRef == \"Volume/state\" and " - ".status.phase == \"Ready\" and " - ".status.observedGeneration == .metadata.generation and " - ".status.resource.ready == true and " - ".status.resource.fence.uid == .metadata.uid and " - ".status.resource.fence.generation == .metadata.generation and " - ".status.resource.fence.revision > 0)] | length) == 1' " - "/run/d2b-binding-ready.json", - timeout=180, - rows=[ - saved_rows("VolumeBinding rows", "/run/d2b-binding-ready.json"), - live_rows("Volume rows", "Volume"), - ], - explain=[ - ("d2bd.service", "vol-binding-6a8ea4307a30f7ceae6533f2"), - ], - ) - # The virtiofs serving side owns only its worker Process and private - # Endpoint as binding-owned children; the worker adopts the per-Volume - # vfd principal synthesized from the declared attachment. - diag_wait( - "binding-worker-ready", - "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - "d2b --zone work --json list Process " - ">/run/d2b-binding-worker.json && " - "jq -e '" - "([.resources[] | select(.type == \"Process\" and " - ".metadata.ownerRef == " - "\"VolumeBinding/vol-binding-6a8ea4307a30f7ceae6533f2\" and " - ".spec.providerRef == \"Provider/system-minijail\" and " - ".spec.executionRef == \"Host/host-system\" and " - ".spec.processClass == \"worker\" and " - ".spec.template == \"virtiofsd-worker\" and " - ".status.phase == \"Ready\")] | length) == 1' " - "/run/d2b-binding-worker.json && " - "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - "d2b --zone work --json list Endpoint " - ">/run/d2b-binding-endpoint.json && " - "jq -e '" - "([.resources[] | select(.type == \"Endpoint\" and " - ".metadata.ownerRef == " - "\"VolumeBinding/vol-binding-6a8ea4307a30f7ceae6533f2\" and " - ".status.phase == \"Ready\")] | length) == 1' " - "/run/d2b-binding-endpoint.json", - timeout=60, - rows=[ - saved_rows("Process rows", "/run/d2b-binding-worker.json"), - saved_rows("Endpoint rows", "/run/d2b-binding-endpoint.json"), - ], - explain=[ - ("d2bd.service", "vol-binding-6a8ea4307a30f7ceae6533f2"), - ("d2bd.service", "virtiofsd"), - ], - ) - diag_wait( - "guest-api-socket", - "test -S " - "/var/lib/d2b/zones/work/guests/acceptance-guest/acceptance-guest.sock", - timeout=30, - rows=[ - summary_rows(), - ( - "guest state dir", - "ls -la /var/lib/d2b/zones/work/guests/acceptance-guest/ " - "2>&1 || true", - ), - ], - explain=[("d2bd.service", "acceptance-guest")], - ) - machine.succeed( - "test -S /var/lib/d2b/zones/work/guests/acceptance-guest/acceptance-guest.sock && " - "test -L /var/lib/d2b/zones/work/guests/acceptance-guest/store-view/state/current && " - "test -L /var/lib/d2b/zones/work/guests/acceptance-guest/store-view/meta/current && " - "test -d /var/lib/d2b/zones/work/guests/acceptance-guest/store-view/live" - ) - - runner = machine.succeed( - "set -- $(for proc in /proc/[0-9]*; do " - "exe=$(readlink \"$proc/exe\" 2>/dev/null || true); " - "case \"$exe\" in */bin/cloud-hypervisor) " - "cmd=$(tr '\\0' ' ' < \"$proc/cmdline\"); " - "case \"$cmd\" in *--api-socket*acceptance-guest*) " - "pid=''${proc#/proc/}; " - "printf '%s %s ' \"$pid\" \"$(awk '{print $22}' \"$proc/stat\")\";; " - "esac;; esac; done); " - "test \"$#\" -eq 2; printf '%s %s' \"$1\" \"$2\"" - ).strip() - runner_pid, runner_start = runner.split() - machine.succeed(f"test -d /proc/{runner_pid}") - machine.succeed( - f"test \"$(awk '{{print $22}}' /proc/{runner_pid}/stat)\" = {runner_start}" - ) - machine.succeed( - f"tr '\\0' ' ' < /proc/{runner_pid}/cmdline | " - "grep -F -- '--api-socket' | grep -F -- 'acceptance-guest'" - ) - - machine.succeed( - "guest_uid=$(runuser -u alice -- env " - "D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - "d2b --zone work --json list Guest | " - "jq -er '.resources[] | select(.type == \"Guest\" and " - ".metadata.name == \"acceptance-guest\" and " - ".spec.providerRef == \"Provider/runtime-cloud-hypervisor\" and " - ".spec.executionRef == \"Host/host-system\") | .metadata.uid') && " - "jq -c " - "'{guestRef, guestUid, zone, reconnectGeneration, " - "providerGeneration, controllerGeneration, assignmentEpoch}' " - "/var/lib/d2b/zones/work/guests/acceptance-guest/component-session/guest.json " - ">/run/d2b-guest-session-before.json && " - "jq -e --arg guest_uid \"$guest_uid\" " - "'.guestRef == \"Guest/acceptance-guest\" and .guestUid == $guest_uid " - "and .zone == \"work\" and .reconnectGeneration > 0 and " - ".providerGeneration > 0 and .controllerGeneration > 0 and " - ".assignmentEpoch > 0' " - "/run/d2b-guest-session-before.json >/dev/null && " - "session_generation=$(journalctl --no-pager -b 2>/dev/null | " - "grep -F 'Guest ComponentSession Resource API server starting' | " - "grep -oE 'generation[[:space:]]*=[[:space:]]*[0-9]+' | " - "grep -oE '[0-9]+' | tail -1) && " - "test -n \"$session_generation\" && " - "test \"$session_generation\" -ge 1 && " - "printf '%s\\n' \"$session_generation\" " - ">/run/d2b-guest-session-generation-before" - ) - - # The Guest target agent boots from its enrolled bundle and key pair, and - # the Guest console is forwarded into the host journal: a read it cannot - # make fails closed inside the Guest, so the host journal must never - # carry that failure. This is the gate the shell-pool fixture cannot - # provide (no vsock device there). - machine.fail( - "journalctl --no-pager -o cat -b " - "| grep -F 'Guest process bundle validation failed'" - ) - - stage("restart-adoption") - machine.succeed("systemctl restart d2bd.service") - diag_unit("daemon-restarted", "d2bd.service", 180) - machine.wait_for_file("/run/d2b/public.sock", timeout=30) - diag_wait( - "api-socket-after-restart", - "test -S " - "/var/lib/d2b/zones/work/guests/acceptance-guest/acceptance-guest.sock", - timeout=30, - rows=[ - live_rows("Guest rows", "Guest"), - ( - "guest state dir", - "ls -la /var/lib/d2b/zones/work/guests/acceptance-guest/ " - "2>&1 || true", - ), - ], - explain=[("d2bd.service", "acceptance-guest")], - ) - machine.succeed(f"test -d /proc/{runner_pid}") - machine.succeed( - f"test \"$(awk '{{print $22}}' /proc/{runner_pid}/stat)\" = {runner_start}" - ) - stage("session-generation-advance") - machine.succeed( - "rm -f /run/d2b-guest-adopted.json /run/d2b-process-adopted.json; " - "for attempt in $(seq 1 60); do " - "session_generation_before=$(cat " - "/run/d2b-guest-session-generation-before) && " - "session_generation_after=$(journalctl --no-pager -b 2>/dev/null | " - "grep -F 'Guest ComponentSession Resource API server starting' | " - "grep -oE 'generation[[:space:]]*=[[:space:]]*[0-9]+' | " - "grep -oE '[0-9]+' | tail -1) && " - "test -n \"$session_generation_after\" && " - "test \"$session_generation_after\" -gt \"$session_generation_before\" && " - "jq -c '{guestRef, guestUid, zone, reconnectGeneration, " - "providerGeneration, controllerGeneration, assignmentEpoch}' " - "/var/lib/d2b/zones/work/guests/acceptance-guest/component-session/guest.json " - ">/run/d2b-guest-session-after.json && " - "jq -e --slurpfile expected /run/d2b-guest-session-before.json " - "'. == $expected[0]' /run/d2b-guest-session-after.json >/dev/null && " - "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - "d2b --zone work --json list Guest " - ">/run/d2b-guest-adopted.json && " - "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - "d2b --zone work --json list Process " - ">/run/d2b-process-adopted.json && " - "jq -e '" - "([.resources[] | select(.type == \"Process\" and " - ".metadata.ownerRef == \"Guest/acceptance-guest\")] | length == 1) and " - "([.resources[] | select(.type == \"Process\" and " - ".metadata.name == \"acceptance-guest-vmm\" and " - ".metadata.ownerRef == \"Guest/acceptance-guest\" and " - ".spec.providerRef == \"Provider/system-minijail\" and " - ".spec.executionRef == \"Host/host-system\" and " - ".spec.processClass == \"worker\" and " - ".spec.template == \"cloud-hypervisor-runner\" and " - ".status.phase == \"Ready\")] | length == 1) and " - "([.resources[] | select(.type == \"Process\" and " - ".metadata.ownerRef == \"Provider/runtime-cloud-hypervisor\" and " - ".spec.providerRef == \"Provider/system-minijail\" and " - ".spec.executionRef == \"Host/host-system\" and " - ".spec.processClass == \"controller\" and " - ".spec.template == \"controller-runtime-cloud-hypervisor-cloud-hypervisor-controller\" and " - ".status.phase == \"Ready\")] | length == 1)' " - "/run/d2b-process-adopted.json && " - "jq -e --slurpfile session /run/d2b-guest-session-after.json " - "'any(.resources[]; " - ".type == \"Guest\" and " - ".metadata.name == \"acceptance-guest\" and " - ".metadata.zone == \"work\" and " - ".metadata.uid == $session[0].guestUid and " - ".spec.providerRef == \"Provider/runtime-cloud-hypervisor\" and " - ".spec.executionRef == \"Host/host-system\" and " - ".status.phase == \"Ready\" and " - ".status.observedGeneration == .metadata.generation and " - ".status.resource.runtimeReady == true and " - ".status.resource.bootstrapReady == true and " - ".status.resource.activeProcessCount == 1)' " - "/run/d2b-guest-adopted.json && exit 0; " - "sleep 1; done; " - "echo 'Guest ComponentSession generation did not advance after restart:'; " - "printf 'before=%s after=%s\\n' " - "\"$(cat /run/d2b-guest-session-generation-before 2>/dev/null || true)\" " - "\"$(journalctl --no-pager -b 2>/dev/null | " - "grep -F 'Guest ComponentSession Resource API server starting' | " - "grep -oE 'generation[[:space:]]*=[[:space:]]*[0-9]+' | " - "grep -oE '[0-9]+' | tail -1)\"; " - "jq -c '.' /run/d2b-guest-session-before.json || true; " - "jq -c '.' /run/d2b-guest-session-after.json || true; " - "echo 'Post-restart Guest readiness failed:'; " - "jq -c '.resources[] | select(.type == \"Guest\" and " - ".metadata.name == \"acceptance-guest\") | " - "{name: .metadata.name, uid: .metadata.uid, " - "owner: .metadata.ownerRef, phase: .status.phase, " - "observedGeneration: .status.observedGeneration, " - "conditions: [.status.conditions[]? | " - "{type: .type, status: .status, reason: .reason}], " - "outcome: (.status.outcome | " - "if . == null then null else " - "{code: .code, retryable: .retryable} end), " - "resource: .status.resource}' " - "/run/d2b-guest-adopted.json || true; " - "echo 'Post-restart Process readiness failed:'; " - "jq -c '.resources[] | " - "{name: .metadata.name, uid: .metadata.uid, " - "owner: .metadata.ownerRef, provider: .spec.providerRef, " - "execution: .spec.executionRef, processClass: .spec.processClass, " - "template: .spec.template, phase: .status.phase, " - "observedGeneration: .status.observedGeneration, " - "conditions: [.status.conditions[]? | " - "{type: .type, status: .status, reason: .reason}], " - "outcome: (.status.outcome | " - "if . == null then null else " - "{code: .code, retryable: .retryable} end), " - "resource: .status.resource}' " - "/run/d2b-process-adopted.json || true; " - "exit 1" - ) - machine.succeed( - "set -- $(for proc in /proc/[0-9]*; do " - "exe=$(readlink \"$proc/exe\" 2>/dev/null || true); " - "case \"$exe\" in */bin/cloud-hypervisor) " - "cmd=$(tr '\\0' ' ' < \"$proc/cmdline\"); " - "case \"$cmd\" in *--api-socket*acceptance-guest*) " - "pid=''${proc#/proc/}; " - "printf '%s %s ' \"$pid\" \"$(awk '{print $22}' \"$proc/stat\")\";; " - "esac;; esac; done); " - f"test \"$#\" -eq 2 && test \"$1\" = {runner_pid} && " - f"test \"$2\" = {runner_start}" - ) - stage("guest-teardown") - machine.succeed( - "for attempt in $(seq 1 30); do " - "guest_revision=$(runuser -u alice -- env " - "D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - "d2b --zone work --json list Guest " - "| jq -er '.resources[] | select(.type == \"Guest\" and " - ".metadata.name == \"acceptance-guest\") | .metadata.revision') && " - "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - "d2b --zone work --json delete Guest/acceptance-guest " - "--revision \"$guest_revision\" " - ">/run/d2b-guest-delete.json 2>/run/d2b-guest-delete.err && exit 0; " - "sleep 1; done; " - "echo 'Guest deletion did not complete within 30s:'; " - "jq -c '{resourceRef: .resourceRef, revision: .revision}' " - "/run/d2b-guest-delete.json || true; " - "echo 'last delete stderr:'; " - "cat /run/d2b-guest-delete.err 2>/dev/null || true; " - "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - "d2b --zone work --json list Guest | " - "jq -c '.resources[] | select(.type == \"Guest\" and " - ".metadata.name == \"acceptance-guest\") | " - "{name: .metadata.name, uid: .metadata.uid, " - "owner: .metadata.ownerRef, phase: .status.phase, " - "observedGeneration: .status.observedGeneration, " - "conditions: [.status.conditions[]? | " - "{type: .type, status: .status, reason: .reason}], " - "outcome: (.status.outcome | " - "if . == null then null else " - "{code: .code, retryable: .retryable} end), " - "resource: .status.resource}' || true; " - "exit 1" - ) - machine.succeed( - "jq -e '.resourceRef == \"Guest/acceptance-guest\" and " - ".revision > 0' " - "/run/d2b-guest-delete.json" - ) - diag_wait( - "guest-draining", - "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - "d2b --zone work --json reconcile Guest/acceptance-guest " - ">/run/d2b-guest-finalize.json 2>/dev/null || true; " - "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - "d2b --zone work --json list Guest " - ">/run/d2b-guest-draining.json && " - "jq -e 'any(.resources[]; .type == \"Guest\" and " - ".metadata.name == \"acceptance-guest\" and " - ".metadata.deletionRequestedAt != null)' " - "/run/d2b-guest-draining.json", - timeout=30, - rows=[ - saved_rows("Guest rows", "/run/d2b-guest-draining.json"), - summary_rows(), - ], - explain=[("d2bd.service", "acceptance-guest")], - ) - diag_wait( - "guest-drained", - "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - "d2b --zone work --json reconcile Guest/acceptance-guest " - ">/run/d2b-guest-finalize.json 2>/dev/null || true; " - "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - "d2b --zone work --json list Guest " - "| jq -e 'all(.resources[]; .metadata.name != \"acceptance-guest\")'", - timeout=60, - rows=[live_rows("Guest rows", "Guest")], - explain=[("d2bd.service", "acceptance-guest")], - ) - diag_wait( - "guest-vmm-process-drained", - "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - "d2b --zone work --json list Process " - "| jq -e 'all(.resources[]; .metadata.name != \"acceptance-guest-vmm\")'", - timeout=30, - rows=[live_rows("Process rows", "Process")], - explain=[("d2bd.service", "acceptance-guest-vmm")], - ) - machine.succeed( - "test ! -S /var/lib/d2b/zones/work/guests/acceptance-guest/acceptance-guest.sock" - ) - - # U7 teardown (F2/AE6): deleting the owning Volume drives the binding - # through its drain: deletion is requested first, then the worker and - # the private endpoint are gone before the binding disappears, leaving - # no orphaned serving effects. - stage("volume-teardown") - machine.succeed( - "volume_revision=$(runuser -u alice -- env " - "D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - "d2b --zone work --json list Volume | " - "jq -er '.resources[] | select(.type == \"Volume\" and " - ".metadata.name == \"state\") | .metadata.revision') && " - "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - "d2b --zone work --json delete Volume/state " - "--revision \"$volume_revision\" >/run/d2b-volume-state-delete.json" - ) - diag_wait( - "volume-binding-draining", - "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - "d2b --zone work --json list VolumeBinding " - ">/run/d2b-binding-draining.json && " - "jq -e 'any(.resources[]; .type == \"VolumeBinding\" and " - ".metadata.name == \"vol-binding-6a8ea4307a30f7ceae6533f2\" and " - ".metadata.deletionRequestedAt != null)' " - "/run/d2b-binding-draining.json", - timeout=30, - rows=[ - saved_rows("VolumeBinding rows", "/run/d2b-binding-draining.json"), - ], - explain=[ - ("d2bd.service", "vol-binding-6a8ea4307a30f7ceae6533f2"), - ], - ) - diag_wait( - "volume-binding-drained", - "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - "d2b --zone work --json list VolumeBinding " - ">/run/d2b-binding-drained.json && " - "jq -e 'all(.resources[]; .metadata.ownerRef != \"Volume/state\")' " - "/run/d2b-binding-drained.json && " - "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - "d2b --zone work --json list Process | " - "jq -e 'all(.resources[]; .metadata.ownerRef != " - "\"VolumeBinding/vol-binding-6a8ea4307a30f7ceae6533f2\")' && " - "runuser -u alice -- env D2B_PUBLIC_SOCKET=/run/d2b/public.sock " - "d2b --zone work --json list Endpoint | " - "jq -e 'all(.resources[]; .metadata.ownerRef != " - "\"VolumeBinding/vol-binding-6a8ea4307a30f7ceae6533f2\")'", - timeout=120, - rows=[ - saved_rows("VolumeBinding rows", "/run/d2b-binding-drained.json"), - live_rows("Process rows", "Process"), - live_rows("Endpoint rows", "Endpoint"), - ], - explain=[ - ("d2bd.service", "vol-binding-6a8ea4307a30f7ceae6533f2"), - ], - ) - ''; -} From b3248017ed32cf79611672878389549445443cc9 Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sat, 26 Sep 2026 23:10:42 -0700 Subject: [PATCH 26/51] refactor(vm): retire the vmChecks flake output and the last fixture All eleven checks assert in Rust, so the nix VM orchestration has nothing left to run. The `vmChecks` output is removed, and with it the `D2B_HOST_TOOL_BUNDLE` / `D2B_CH_CONTROLLER_BUNDLE` environment handoff that only that output read: the guest-image action declares the same bundles as Bazel label inputs, which is what R2 and R3 asked for. The comment that stood over the output now says where the lane lives. The one file left under tests/host-integration/ is lib.nix, and it stays on purpose: the ported checks' guest declarations in nix/test-support/host-integration-node.nix still read it for their provider-artifact builders (mkAcceptanceProviderArtifact, mkRuntimeCloudHypervisorArtifact, mkVolumeProviderArtifact, mkDeviceWorkerProviderArtifact), so the directory survives for its builders rather than for fixtures. Also removed here: the deferred Gateway-isolation fixture, tests/host-integration/deferred/host-zone-gateway-isolation.nix. That check asserted that a Gateway relay credential never materializes on the host - not in /etc/d2b, /var/lib/d2b, /run/d2b, /var/log, the audit tree, the journal, coredumps, nor in any d2bd or broker process's environ, cmdline or fd table, with no relay socket established. **No retained check carries that assertion**, and the eleven checks were reviewed for it, so this is a removal rather than a consolidation: the tokens gateway, relay, canary and SharedAccessKey appear in none of the retained fixtures and no ported module sweeps host paths or process tables for a guest-held secret. The changelog fragment records the loss in those terms. Checked before removing the handoff: the two variables appeared only in the retired output's own evaluation, in the Makefile recipe that is being retired with it, and as a fallback read in tests/host-integration/lib.nix (read it back with getEnv: when the variable is absent, the builder resolves the same package through self.packages, which is the path every image build already takes). Nothing else in the tree reads them. --- flake.nix | 70 +-- .../deferred/host-zone-gateway-isolation.nix | 471 ------------------ 2 files changed, 10 insertions(+), 531 deletions(-) delete mode 100644 tests/host-integration/deferred/host-zone-gateway-isolation.nix diff --git a/flake.nix b/flake.nix index a5c0c12bd..d77186986 100644 --- a/flake.nix +++ b/flake.nix @@ -641,66 +641,16 @@ in builtins.listToAttrs (map mkImage imageFiles) else { }); - # Type-G runNixOSTest integration tests (the additive real-kernel - # coverage layer). Each test boots a real NixOS VM with the d2b - # daemon surface and asserts live broker/daemon/host-posture behaviour - # (socket activation, SO_PEERCRED, bridge isolation, state-dir ACLs, - # broker privilege posture) that the fake-backed native Rust canaries and - # pure-eval gates cannot exercise. This is the hermetic, non-destructive - # successor to the `D2B_LIVE`-against-the-real-host bash scripts. - # - # Exposed under `vmChecks`, NOT `checks`, so the Layer-1 `nix flake check - # --no-build --all-systems` never realizes a VM. Selected explicitly by - # `make test-host-integration` (`nix build .#vmChecks..`), - # which needs KVM (a local NixOS host; TCG fallback otherwise). - # - # Auto-discovered from tests/host-integration/*.nix (excluding lib.nix): each test is - # `{ pkgs, self }: pkgs.testers.runNixOSTest { ... }`, so adding a VM test - # is one new file - no edit here. x86_64-linux only: a runNixOSTest VM is - # built + booted for the builder's own system, and the hosted CI runners - # are x86_64 - aarch64 VM coverage needs an aarch64 builder. - vmChecks = forAllSystems (system: - if system == "x86_64-linux" then - let - pkgs = nixpkgsFor.${system}; - # The two environment reads are the legacy handoff. The - # Bazel-owned lane reaches the same package from the guest - # image action's declared label inputs; these reads stay until - # that lane replaces the recipe. - hostToolBundleEnv = builtins.getEnv "D2B_HOST_TOOL_BUNDLE"; - cloudHypervisorControllerBundleEnv = - builtins.getEnv "D2B_CH_CONTROLLER_BUNDLE"; - handoff = - if hostToolBundleEnv == "" then - null - else - mkBazelHostTools system hostToolBundleEnv - (if cloudHypervisorControllerBundleEnv == "" then - null - else - cloudHypervisorControllerBundleEnv); - testSelf = - if handoff == null then self else handoff.hostSelf; - bazelHostTools = - if handoff == null then null else handoff.tools; - testDir = ./tests/host-integration; - testFiles = if builtins.pathExists testDir - then builtins.attrNames (nixpkgs.lib.filterAttrs - (name: type: - type == "regular" - && nixpkgs.lib.hasSuffix ".nix" name - && name != "lib.nix") - (builtins.readDir testDir)) - else [ ]; - mkTest = file: { - name = nixpkgs.lib.removeSuffix ".nix" file; - value = import (testDir + "/${file}") { - inherit pkgs; - self = testSelf; - }; - }; - in builtins.listToAttrs (map mkTest testFiles) - else { }); + # The type-10 host-integration lane moved to Bazel: the lane test + # target (`//bazel/checks/vm:host_integration_lane_run`) builds one guest + # image per check as a declared-input action, boots a pool from them, + # and runs each check's assertions against a snapshot-restored copy of + # its guest. `make test-host-integration` invokes that target. The + # `vmChecks` flake output, its `D2B_HOST_TOOL_BUNDLE` / + # `D2B_CH_CONTROLLER_BUNDLE` environment handoff, and the runNixOSTest + # fixtures are gone with it; the one file kept under + # `tests/host-integration/` is `lib.nix`, which the ported checks' guest + # declarations still read for their provider-artifact builders. # The guest image for the Bazel-owned host-integration lane. It is a # function, not a package: the lane's guest-image action calls it diff --git a/tests/host-integration/deferred/host-zone-gateway-isolation.nix b/tests/host-integration/deferred/host-zone-gateway-isolation.nix deleted file mode 100644 index 6a83381d5..000000000 --- a/tests/host-integration/deferred/host-zone-gateway-isolation.nix +++ /dev/null @@ -1,471 +0,0 @@ -# Deferred Type-G runNixOSTest: host remains isolated from Gateway Guest relay credentials. -{ pkgs, self }: - -let - inherit (pkgs) lib; - d2bLib = import ./lib.nix { - inherit self; - inherit (pkgs) lib; - hostToolBundle = - if self.lib ? d2bHostToolBundle then self.lib.d2bHostToolBundle else null; - }; - d2bNode = import ../../../nix/test-support/host-integration-node.nix { - inherit self; - inherit (pkgs) lib; - }; - cloudHypervisorArtifact = - d2bLib.mkRuntimeCloudHypervisorArtifact pkgs; - volumeProviderArtifact = d2bLib.mkVolumeProviderArtifact pkgs; - providerArtifact = d2bLib.mkAcceptanceProviderArtifact pkgs; - acceptancePublisherKey = providerArtifact.trustedPublisher.signingKey; - gatewayCanary = "d2b-u5-gateway-canary-7f4e9c2a"; - gatewayStateDir = "/var/lib/d2b/zones/work/guests/gateway"; - gatewayObservationDir = "${gatewayStateDir}/canary-observation"; - gatewayCredentialDir = "/var/lib/d2b/guest-state/gateway-credentials"; - gatewaySealKeyB64 = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8="; - gatewaySealKey = pkgs.writeText - "d2b-u5-gateway-seal-key.b64" - "${gatewaySealKeyB64}\n"; - gatewayCredentialPython = pkgs.python3.withPackages - (pythonPackages: [ pythonPackages.cryptography ]); - gatewayCanaryDigest = builtins.hashString "sha256" gatewayCanary; - gatewayCredential = pkgs.runCommand "d2b-u5-gateway-sealed-credential" { - nativeBuildInputs = [ gatewayCredentialPython ]; - } '' - ${gatewayCredentialPython}/bin/python3 - <<'PY' > "$out" - import base64 - import json - from cryptography.hazmat.primitives.ciphers.aead import ChaCha20Poly1305 - - key = bytes(range(32)) - nonce = bytes(range(12)) - material = { - "relayListen": { - "keyName": "d2b-u5-listen", - "key": "${gatewayCanary}", - }, - "relaySend": { - "keyName": "d2b-u5-send", - "key": "${gatewayCanary}", - }, - } - plaintext = json.dumps(material, separators=(",", ":")).encode() - aad = ( - b"d2b-gateway-credential-v1" - + (1).to_bytes(8, "big") - + b"\x00" - + (0).to_bytes(8, "big") - ) - ciphertext = ChaCha20Poly1305(key).encrypt(nonce, plaintext, aad) - envelope = { - "schemaVersion": 1, - "generation": 1, - "notAfter": None, - "nonce": base64.b64encode(nonce).decode(), - "ciphertext": base64.b64encode(ciphertext).decode(), - } - print(json.dumps(envelope, separators=(",", ":"))) - PY - ''; - gatewayGuest = self.lib.evalGuest { - system = pkgs.system; - name = "gateway"; - zone = "work"; - stateDir = gatewayStateDir; - modules = [ - ({ lib, pkgs, name, ... }: { - environment.etc."d2b/gateway.json".text = builtins.toJSON { - credentialPath = "${gatewayCredentialDir}/relay.sealed.json"; - sealKeyPath = "${gatewayCredentialDir}/seal.key"; - observationPath = "/run/d2b-gateway-observation/opened"; - relay = { - namespace = "relns-d2b-prod"; - entity = "hc-d2b-work"; - }; - }; - d2b.vms.${name}.runner.shares = lib.mkAfter [ - { - source = gatewayObservationDir; - mountPoint = "/run/d2b-gateway-observation"; - tag = "d2b-canary"; - proto = "virtiofs"; - readOnly = false; - } - ]; - system.activationScripts.d2bGatewayCredential = { - deps = [ "users" ]; - text = '' - install -d -o d2bd -g d2bd -m 0700 ${gatewayCredentialDir} - ${pkgs.coreutils}/bin/base64 -d ${gatewaySealKey} \ - > ${gatewayCredentialDir}/seal.key - ${pkgs.coreutils}/bin/chown d2bd:d2bd \ - ${gatewayCredentialDir}/seal.key - ${pkgs.coreutils}/bin/chmod 0600 \ - ${gatewayCredentialDir}/seal.key - ${pkgs.coreutils}/bin/install -o d2bd -g d2bd -m 0600 ${gatewayCredential} \ - ${gatewayCredentialDir}/relay.sealed.json - ''; - }; - }) - ]; - }; - gatewaySystem = gatewayGuest.config.system.build.toplevel; - gatewayNetSystem = d2bLib.mkGuestSystem { - inherit pkgs; - name = "gateway-net-vm"; - }; - cloudHypervisorConfig = { - controllerExecutionRef = "Host/host-system"; - defaultVcpus = 2; - defaultMemoryMb = 512; - defaultMachineType = "microvm"; - watchdog = true; - adoptionWindowMs = 30000; - healthCheckIntervalMs = 5000; - healthCheckTimeoutMs = 1000; - healthCheckFailureThreshold = 3; - startupDeadlineMs = 120000; - }; -in -pkgs.testers.runNixOSTest { - name = "d2b-host-zone-gateway-isolation"; - - nodes.machine = d2bNode.d2bCloudHypervisorNode { - extra = { ... }: { - environment.systemPackages = [ - pkgs.iproute2 - pkgs.jq - ]; - - d2b.site.usePrebuiltHostTools = false; - system.activationScripts.d2bGatewayCanaryObservation = { - deps = [ "users" ]; - text = '' - install -d -m 0700 -o d2bd -g d2bd \ - ${gatewayObservationDir} - ${pkgs.coreutils}/bin/rm -f \ - ${gatewayObservationDir}/opened - ''; - }; - d2b.artifacts = { - gateway-system = { - package = gatewaySystem; - type = "nixos-system"; - }; - net-vm-base = { - package = gatewayNetSystem.config.system.build.toplevel; - type = "nixos-system"; - }; - acceptance-provider = { - inherit (providerArtifact) package type catalog; - }; - runtime-cloud-hypervisor = { - inherit (cloudHypervisorArtifact) package type catalog; - }; - volume-acceptance-provider = { - inherit (volumeProviderArtifact) package type catalog; - }; - }; - # Acceptance-only packages remain artifact-only fixtures; their - # identities are not rows in the closed Provider matrix. - d2b.zones.local-root.trustedPublishers.d2b-u20-acceptance.signingKey = - acceptancePublisherKey; - d2b.zones.local-root.trustedPublishers.d2b-cloud-hypervisor.signingKey = - cloudHypervisorArtifact.trustedPublisher.signingKey; - d2b.zones.local-root.trustedPublishers.d2b-volume-acceptance.signingKey = - volumeProviderArtifact.trustedPublisher.signingKey; - d2b.zones.work.trustedPublishers.d2b-u20-acceptance.signingKey = - acceptancePublisherKey; - d2b.zones.work.trustedPublishers.d2b-cloud-hypervisor.signingKey = - cloudHypervisorArtifact.trustedPublisher.signingKey; - d2b.zones.work.trustedPublishers.d2b-volume-acceptance.signingKey = - volumeProviderArtifact.trustedPublisher.signingKey; - d2b.guestSystems.work.gateway = gatewayGuest; - d2b.zones.local-root.resources.host-system = { - type = "Host"; - spec.providerRef = "Provider/system-core"; - }; - d2b.zones.work = { - parentZone = "local-root"; - resources = { - alice = { - type = "User"; - spec = { - displayName = "Alice"; - groups = [ ]; - osUsername = "alice"; - }; - }; - d2bd = { - type = "User"; - spec = { - displayName = "d2bd"; - groups = [ ]; - osUsername = "d2bd"; - }; - }; - host-system = { - type = "Host"; - spec.providerRef = "Provider/system-core"; - }; - gateway = { - type = "Guest"; - spec = { - providerRef = "Provider/runtime-cloud-hypervisor"; - executionRef = "Host/host-system"; - systemArtifactId = "gateway-system"; - networkAttachments = [ - { - default = true; - networkRef = "Network/relay-egress"; - } - ]; - }; - }; - network-local = { - type = "Provider"; - spec = { - artifactId = "acceptance-provider"; - config.controllerExecutionRef = "Host/host-system"; - }; - }; - volume-local = { - type = "Provider"; - spec = { - artifactId = "volume-acceptance-provider"; - config = { - controllerExecutionRef = "Host/host-system"; - sourcePolicies = [ - { - id = "default-state"; - class = "local-path"; - volumeKinds = [ "durable" "state" "cache" ]; - } - ]; - }; - }; - }; - volume-virtiofs = { - type = "Provider"; - spec = { - artifactId = "volume-acceptance-provider"; - config.controllerExecutionRef = "Host/host-system"; - }; - }; - relay-egress = { - type = "Network"; - spec = { - lanCidr = "10.70.0.0/24"; - providerRef = "Provider/network-local"; - netVmSystemArtifactId = "net-vm-base"; - uplinkCidr = "192.0.2.4/30"; - }; - }; - runtime-cloud-hypervisor = { - type = "Provider"; - spec = { - artifactId = "runtime-cloud-hypervisor"; - config = cloudHypervisorConfig; - }; - }; - transport-azure-relay = { - type = "Provider"; - spec = { - artifactId = "acceptance-provider"; - config = { - controllerExecutionRef = "Host/host-system"; - executionRef = "Guest/gateway"; - networkRef = "Network/relay-egress"; - }; - }; - }; - credential-managed-identity = { - type = "Provider"; - spec = { - artifactId = "acceptance-provider"; - config = { - controllerExecutionRef = "Host/host-system"; - credentialDomains = [ "system" ]; - supportedOperations = [ "acquire-token" ]; - }; - }; - }; - relay-listen = { - type = "Credential"; - spec = { - providerRef = "Provider/credential-managed-identity"; - audience = "azure-relay-listen"; - allowedOperations = [ "acquire-token" ]; - consumerRef = "Provider/transport-azure-relay"; - expiry.hardDeadlineMs = 0; - revocation = { - onOwnerDelete = "immediate"; - onProviderGeneration = "immediate"; - }; - rotation = { - maxLeaseLifetimeMs = 0; - policy = "on-expiry"; - proactiveWindowMs = null; - }; - scope.executionRef = "Guest/gateway"; - }; - }; - relay-send = { - type = "Credential"; - spec = { - providerRef = "Provider/credential-managed-identity"; - audience = "azure-relay-send"; - allowedOperations = [ "acquire-token" ]; - consumerRef = "Provider/transport-azure-relay"; - expiry.hardDeadlineMs = 0; - revocation = { - onOwnerDelete = "immediate"; - onProviderGeneration = "immediate"; - }; - rotation = { - maxLeaseLifetimeMs = 0; - policy = "on-expiry"; - proactiveWindowMs = null; - }; - scope.executionRef = "Guest/gateway"; - }; - }; - uplink = { - type = "ZoneLink"; - spec = { - childZoneName = "work"; - disabled = false; - limits = { - maxActiveStreams = 32; - maxPendingIntents = 256; - reconnectMaxAttempts = 10; - reconnectWindowSecs = 300; - }; - transportCredentials = [ - "Credential/relay-listen" - "Credential/relay-send" - ]; - transportProviderRef = "Provider/transport-azure-relay"; - transportSettings = { - relayEntityId = "hc-d2b-work"; - relayNamespaceId = "relns-d2b-prod"; - }; - }; - }; - }; - }; - }; - }; - - testScript = '' - start_all() - machine.wait_for_unit("d2bd.service", timeout=120) - machine.wait_for_unit("d2b-broker.socket", timeout=30) - machine.wait_for_file("/run/d2b/public.sock", timeout=30) - - # The committed Process/cloud-hypervisor-gateway row is desired-running; - # d2bd's startup process reconciliation is the production Guest launcher. - canary = ${builtins.toJSON gatewayCanary} - gateway_vsock = "/var/lib/d2b/zones/work/guests/gateway/vsock.sock" - machine.wait_for_file(gateway_vsock, timeout=120) - machine.succeed(f"test -S {gateway_vsock}") - observation_path = ( - "/var/lib/d2b/zones/work/guests/gateway/" - "canary-observation/opened" - ) - machine.wait_for_file(observation_path, timeout=60) - observation = machine.succeed(f"cat {observation_path}").strip() - assert observation == ( - "schemaVersion=1\n" - "generation=1\n" - "digest=sha256:${gatewayCanaryDigest}\n" - ) - - policy = "/etc/d2b/host-zone-relay-egress-policy.json" - machine.succeed(f"test -r {policy}") - machine.succeed( - f"jq -e '.mode == \"host-zone-relay-deny\" " - f"and (.gatewayInterfaces == []) " - f"and (.diagnostics.redacted == true) " - f"and (.diagnostics.rateLimited == true)' {policy}" - ) - policy_forbidden = [ - "relns-example.servicebus.windows.net", - "hc-d2b-work", - "Credential/relay-listen", - "Credential/relay-send", - "/var/lib/d2b/gateways/work/credential.sealed.json", - "/var/lib/d2b/gateways/work/seal.key", - "SharedAccessKey", - ] - for token in policy_forbidden: - machine.fail(f"grep -F {repr(token)} {policy}") - - runtime_forbidden = policy_forbidden + ["D2B_RELAY_", canary] - - machine.fail("test -e /etc/d2b/gateway.json") - machine.fail("systemd-tmpfiles --cat-config | grep -F '/var/lib/d2b/gateways/work'") - machine.succeed("test -r /etc/d2b/zones/work/resource-bundle.json") - for host_path in [ - "/etc/d2b/zones", - "/etc/d2b/bundle.json", - "/etc/d2b/allocator.json", - "/var/lib/d2b", - "/run/d2b", - ]: - machine.succeed( - f"! grep -R -F -- {canary!r} {host_path} 2>/dev/null" - ) - machine.fail( - "grep -R -F 'SharedAccessKey' /etc/d2b/zones /var/lib/d2b 2>/dev/null" - ) - machine.succeed( - f"! journalctl --no-pager -b 2>/dev/null | grep -F -- {canary!r}" - ) - machine.succeed( - f"! grep -R -F -- {canary!r} /var/log /var/lib/d2b/audit " - "2>/dev/null" - ) - machine.succeed( - f"! (coredumpctl --no-pager 2>/dev/null || true) " - f"| grep -F -- {canary!r}" - ) - - pids = machine.succeed("pgrep -x d2bd").strip().split() - assert pids, "d2bd pid missing" - machine.succeed("systemctl start d2b-broker.service") - broker_pid = machine.succeed( - "for i in $(seq 1 50); do " - "pid=$(systemctl show -p MainPID --value d2b-broker.service); " - "if [ -n \"$pid\" ] && [ \"$pid\" != 0 ]; then echo \"$pid\"; exit 0; fi; " - "sleep 0.2; done; exit 1" - ).strip() - pids.append(broker_pid) - - for pid in pids: - env = machine.succeed(f"tr '\\0' '\\n' < /proc/{pid}/environ || true") - cmd = machine.succeed(f"tr '\\0' ' ' < /proc/{pid}/cmdline || true") - fds = machine.succeed(f"ls -l /proc/{pid}/fd || true") - for token in runtime_forbidden: - assert token not in env, f"forbidden token leaked in environ for pid {pid}" - assert token not in cmd, f"forbidden token leaked in cmdline for pid {pid}" - assert token not in fds, f"forbidden token leaked in fd table for pid {pid}" - - machine.succeed( - f"! journalctl --no-pager -b 2>/dev/null | grep -F -- {canary!r}" - ) - machine.succeed( - f"! grep -R -F -- {canary!r} /etc/d2b /var/lib/d2b /run/d2b " - "2>/dev/null" - ) - machine.succeed( - f"! (coredumpctl --no-pager 2>/dev/null || true) " - f"| grep -F -- {canary!r}" - ) - - sockets = machine.succeed("ss -Htanp || true") - assert "servicebus.windows.net" not in sockets - assert "d2b-provider-relay" not in sockets - assert "d2b-gateway-relay" not in sockets - ''; -} From e58794d003c3e4f2e3c34fa8636c1096c910f0bb Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sat, 26 Sep 2026 23:13:10 -0700 Subject: [PATCH 27/51] docs(changelog): record the Bazel-owned lane, the coverage it removed, and the console-read fix --- changelog.d/v3.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/changelog.d/v3.md b/changelog.d/v3.md index 0bf6b0cee..373b2cb20 100644 --- a/changelog.d/v3.md +++ b/changelog.d/v3.md @@ -1,9 +1,17 @@ ### Changed +- **The host integration lane is Bazel-owned end to end.** All eleven checks now assert in Rust; every `runNixOSTest` fixture and its `testScript` are gone, the `vmChecks` flake output is gone, and `make test-host-integration` is a single `bazel test` invocation of the lane target. Each guest image is a graph output keyed on declared inputs - the flake and its lock, the guest module sources, and the d2b host binaries arrive as label inputs - and the lane restores a pooled guest per check rather than booting one guest per check. Check selection is a filter: `D2B_VM_CHECK=` or `bazel test --test_filter=`, and each check reports under its own name with the stage it was in, the rows it asserted on, and the guest's journal and zone dump. The lane is a local contributor-run pre-PR surface, declares virtualization as a precondition with no emulation fallback, and needs `/dev/kvm`; it is x86_64-linux only. - Renamed `packages/d2b-vm-harness` to `packages/d2b-test-vm-harness`, with the crate, its `Cargo.lock` entry, its Bazel targets, and its `D2B_VM_HARNESS_*` environment contract renamed to match (`D2B_TEST_VM_HARNESS_*`). The name now reads as one name wherever it appears: the package, the binary, the lib, and every variable the lane hands the harness. No behaviour changed; the lane boots the same guests and runs the same checks. + ### Fixed - A restored lane guest attached its block devices in the emulator's reported order rather than the order the launch attached them, so on every restore a two-disk guest's root disk and its state disk swapped `/dev/vda` and `/dev/vdb`. The guest's volume markers anchor a volume root by device and inode and fail closed on a mismatch, so after any restore every marker disagreed with the live tree, the volume layout effect failed permanently, and the checks that wait on a volume timed out. Restores now re-attach in launch order, and the lane's restored-versus-fresh gate reports each block device's name and size so a regression of this fails in milliseconds instead of as a wait running out. - A zone-native device owner declared its worker state under `/run/d2b/vms` but never declared the runtime directory itself, and nothing in the tree provisioned it: a legacy VM gets that directory from its own storage row, and a zone-native host had neither. The broker's socket grant refuses an absent path ancestor, so every role whose posture binds a runtime socket - the TPM worker and both GPU roles - was refused at launch, while the one-shot flush, which binds none, was admitted. The per-guest runtime tree is now declared for zone-native owners too. - A refused forwarded invocation reached the log as a closed-set code with its reason discarded at the point of refusal, so a provider's actual explanation for refusing a spawn was unrecoverable from any journal. The detail is now written down where it was dropped. +- The lane's console bounded its read while waiting for the guest's shell and then cleared the bound, so every command read afterwards ran against an unbounded socket. A guest that died mid-command left the host reading a socket that would never carry another byte, taking the whole lane down with no per-check result to show for it. The host's wait for an answer is now bounded, and a console that does not answer fails with a message naming the command rather than hanging. + +### Removed + +- **The deferred Gateway-isolation check, and with it the coverage it carried.** `tests/host-integration/deferred/host-zone-gateway-isolation.nix` asserted that a Gateway Guest's relay credential never materializes on the host: the credential's canary is absent from `/etc/d2b/{zones,bundle.json,allocator.json}`, `/var/lib/d2b`, `/run/d2b`, `/var/log`, `/var/lib/d2b/audit`, the journal and coredump output, and absent from every `d2bd` and broker process's environ, cmdline and fd table, with no relay socket established. **No retained check carries that assertion.** All eleven checks were reviewed for it - the ported modules and the four fixtures that still existed at review time - and the tokens `gateway`, `relay`, `canary` and `SharedAccessKey` appear in none of them, nor does any module sweep host paths or process tables for a guest-held secret. This is a coverage **removal**, not a consolidation: there is no successor check and nothing here replaces it. If a future reader greps for the relay canary and finds nothing, this entry is the record that the absence was decided rather than overlooked. +- The `vmChecks` flake output and the `D2B_HOST_TOOL_BUNDLE` / `D2B_CH_CONTROLLER_BUNDLE` environment handoff only it read, along with the Attic cache preflight and closure upload the retired nix recipe carried. The guest-image action declares its own substituters and preflights them itself, so cache handling now lives with the build that needs it rather than in a second place that can drift out of step with it. From 40c2f8ac6c75790f8f9911295580000bafbd71b7 Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sat, 26 Sep 2026 23:18:15 -0700 Subject: [PATCH 28/51] docs: sweep host-integration docs onto the Bazel-owned lane The host lane is one Bazel test target and the heavy-gate semaphore is gone, but the instruction, contributor, and reference documents still described the removed `D2B_HOST_TOOL_BUNDLE` / `D2B_CH_CONTROLLER_BUNDLE` handoff, the `vmChecks` output, the `runNixOSTest` fixtures, and the deleted semaphore as current. Update those sites to the shipped lane: the Bazel-built host binaries arrive at the guest-image action as declared Bazel label inputs (Nix still must not rebuild them), the lane is `//bazel/checks/vm:host_integration_lane_run`, every assertion is Rust, `D2B_VM_CHECK` / `--test_filter` selects one check, and `/dev/kvm` is a declared precondition with no emulation fallback. Record the semaphore's deletion where a document still pointed at it, and say that nothing replaces it. --- AGENTS.md | 12 ++- docs/contributing/README.md | 2 +- docs/contributing/critical-subsystems.md | 6 +- docs/contributing/gates-and-lints.md | 91 +++++++------------ docs/how-to/adding-a-test.md | 2 +- docs/how-to/create-provider.md | 6 +- docs/reference/compatibility.md | 10 +- docs/reference/per-vm-state-ownership.md | 4 +- docs/reference/support-matrix.md | 6 +- .../integration/README.md | 17 +++- tests/AGENTS.md | 40 ++++---- tests/README.md | 64 +++++-------- 12 files changed, 110 insertions(+), 150 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 3efa4c484..65811e335 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -53,7 +53,7 @@ Use this index, then open the focused document instead of expanding this file. | Worktrees, review, PRs, merge, and disk hygiene | [`docs/contributing/workflow.md`](./docs/contributing/workflow.md), especially the [reviewed-head lifecycle](./docs/contributing/workflow.md#reviewed-head-pr-lifecycle) | | Changelog or commit grammar | [`docs/contributing/changelog-and-commits.md`](./docs/contributing/changelog-and-commits.md) | | Codegraph MCP server and agent usage | [`.omp/mcp.json`](./.omp/mcp.json) wires the server; see [Codegraph (MCP code intelligence)](#codegraph-mcp-code-intelligence) below for per-checkout init and tool guidance | -| Gates, heavy lanes, and build profiles | [`docs/contributing/gates-and-lints.md`](./docs/contributing/gates-and-lints.md) | +| Gates, validation lanes, and build profiles | [`docs/contributing/gates-and-lints.md`](./docs/contributing/gates-and-lints.md) | | Architecture and per-Guest/provider features | [`docs/contributing/architecture.md`](./docs/contributing/architecture.md) and [ADR 0015](./docs/adr/0015-daemon-only-clean-break.md) | | Critical subsystem invariants | [`docs/contributing/critical-subsystems.md`](./docs/contributing/critical-subsystems.md) | | Contributor orchestration and host distribution | [`d2b-gascity`](https://github.com/vicondoa/d2b-gascity) for orchestration and [`gascity.nix`](https://github.com/vicondoa/gascity.nix) for NixOS distribution and installation | @@ -224,10 +224,12 @@ settings or claim atomic base binding. `make test-host-integration`. They may run alongside the `/etc/nixos` real-host switch, d2b startup, and Cloud Hypervisor Guest boot. U19 converges their declarations and current inputs but does not run host - acceptance. The host-integration lane must inject its Bazel-built - `d2b`, `d2bd`, `d2b-broker`, activation/helper, resource-compiler, and - Wayland-proxy binaries through `D2B_HOST_TOOL_BUNDLE`; it must not rebuild - those binaries through Nix. + acceptance. The host-integration lane passes its Bazel-built `d2b`, + `d2bd`, `d2b-broker`, activation/helper, resource-compiler, and + Wayland-proxy binaries to the guest-image action as declared Bazel label + inputs; Nix must realize the guest closure around those binaries rather + than rebuild them. The lane is `//bazel/checks/vm:host_integration_lane_run`, + invoked by `make test-host-integration`. - Every code change ships a valid changelog entry or a fragment under [`changelog.d/`](./changelog.d/). - The retired repository-local contributor runtime has no ordinary-work diff --git a/docs/contributing/README.md b/docs/contributing/README.md index 510c38564..ba6a03e9a 100644 --- a/docs/contributing/README.md +++ b/docs/contributing/README.md @@ -14,7 +14,7 @@ disagrees with committed, passing code, code wins. | --- | --- | | [workflow.md](./workflow.md) | Isolated worktrees, task routing, reviewed-head PR lifecycle, landing, edit/commit/validate, local host validation, screenshot hygiene, and disk hygiene. | | [changelog-and-commits.md](./changelog-and-commits.md) | Changelog fragments, auto-release, version cut lifecycle, release hygiene, and commit grammar. | -| [gates-and-lints.md](./gates-and-lints.md) | The heavy-lane semaphore and contributor validation lanes. | +| [gates-and-lints.md](./gates-and-lints.md) | Contributor validation lanes, policy lints, and build profiles. | | [critical-subsystems.md](./critical-subsystems.md) | Invariants for every AGENTS.md critical index subsystem, plus cgroup naming and ownership-marker conventions. | | [architecture.md](./architecture.md) | Eval-time naming, sibling flake boundaries, daemon-supervised VM lifecycle, and per-VM behavior. | diff --git a/docs/contributing/critical-subsystems.md b/docs/contributing/critical-subsystems.md index 492ff9534..f1acb6820 100644 --- a/docs/contributing/critical-subsystems.md +++ b/docs/contributing/critical-subsystems.md @@ -147,5 +147,7 @@ the `/etc/nixos` switch, d2b startup, and Cloud Hypervisor Guest boot; an advisory skip is not evidence for those checks. U20 must also run both `make test-host-integration` and `make test-integration`, which may be scheduled alongside real-host testing. U19 only converges their declarations -and current inputs. The host lane injects the Bazel-built d2b binary bundle -through `D2B_HOST_TOOL_BUNDLE`; it does not rebuild d2b binaries through Nix. +and current inputs. The host lane passes its Bazel-built d2b binaries to the +guest-image action as declared Bazel label inputs +(`//bazel/checks/vm:host_integration_lane_run`); Nix realizes the guest +closure around them and does not rebuild d2b binaries. diff --git a/docs/contributing/gates-and-lints.md b/docs/contributing/gates-and-lints.md index e1539c435..0d7109ae7 100644 --- a/docs/contributing/gates-and-lints.md +++ b/docs/contributing/gates-and-lints.md @@ -1,7 +1,7 @@ # Gates and lints -Reference for the heavy-lane semaphore and policy lints whose exemptions are -easy to get wrong. The binding summary and enforcing/advisory rule live under +Reference for the contributor validation lanes and policy lints whose +exemptions are easy to get wrong. The binding summary and enforcing/advisory rule live under [worktree, validation, and landing rules](../../AGENTS.md#worktree-validation-and-landing-rules); read that first. This file covers the parts needing more than a rule. @@ -327,64 +327,35 @@ and `d2b-provider-guest`'s `tests/registration.rs` - declare them silently; run those crates with `--features test-support` (or `--all-features`) when working through cargo instead of the Bazel layer. -## Heavy lanes - -Every Layer-2, host-integration, hardware, live, and perf-heavy command -runs through **one** semaphore, invoked from the repository root through the -Bazel-built `bazel-bin/packages/xtask/xtask heavy-gate` facade. It grants -two slots per uid via open file description locks so concurrent heavy lanes -cannot oversubscribe the shared Nix store, Bazel output tree, or KVM -device. Do not add a second lock file, sleep-and-retry loop, or per-crate -guard. - -The slot namespace is fixed at `/run/d2b-heavy-gates/uid-/`. The root -and per-uid directory are root-owned and non-writable by unprivileged users; -the two `slot-*` files are pre-created for the target uid at mode `0600`. -No runtime-directory or temporary-directory fallback. The NixOS -module provisions the root with systemd-tmpfiles, then activation provisions -directories and slots for configured lifecycle users that NSS can resolve. -An unavailable network-backed user is deferred rather than failing -activation; after that user logs in, run `make heavy-gate-provision`. Use -the same target on a host that does not consume the module. Because `/run` -is a tmpfs, run it once per boot when the gate requests it. An absent or -malformed namespace is an environment error with that provisioning -remediation, never permission to create a weaker pool. In particular, -`/run/user/` is rejected because its owner can rename slot names or -their parent and create an independent pool. - -The structure is public-lane-plus-guarded-internal: - -- **Public lane targets** (`make test-integration`, - `make test-host-integration`, `make perf`) acquire - a slot and then delegate to a guarded internal `heavy-lane-*` target. - Run these. -- **Internal `heavy-lane-*` targets** hold the raw work and fail closed - through `heavy-lane-guard` if invoked outside the gate (the gate exports - `D2B_HEAVY_GATE` across its re-exec). Do not run them directly. -- **Convenience wrappers** `make heavy-check`, `make heavy-flake-check`, and - the `heavy-test-*` aliases run a Layer-1 - gate, the Rust suite, the building flake check, or a public lane under - the same semaphore. - -Run a heavy lane through its public target (or, for an arbitrary command, -`make heavy-gate-build && bazel-bin/packages/xtask/xtask heavy-gate -- -`) whenever another heavy lane might be running; do not invoke the -internal targets directly. Live-host -tests obey the same rule: use the gated live-VM smoke entrypoints (`make -pre-tag` for the full gate, `make smoke-lite` for the lite gate) or wrap a -raw live script with the Bazel-built xtask artifact. - -The repository-root `Cargo.toml` and `Cargo.lock` are rules_rs metadata -authority. The Bazel-built xtask label is the only supported gate entrypoint; -do not add a direct Cargo compatibility wrapper. - -Invoking a live script directly is safe but not the documented path: each -one verifies the inherited slot and re-executes itself through the semaphore -exactly once when no genuine slot is held. A bare `D2B_HEAVY_GATE` value is -not trusted, so it cannot bypass the sole-use invariant. -**A new live, hardware, or performance entrypoint must carry that same -self-guard block**, or the fail-closed inventory guard -(`every_live_and_heavy_entrypoint_routes_through_the_gate`) rejects it. +## Layer-2 and manual lanes + +Layer-2 container, VM, live-host, hardware, and performance surfaces remain +conditional or manual and are not part of the Bazel Layer-1 scheduler. Run the +public interfaces directly: + +- `make test-integration` runs the container rollup + `bash tests/test-integration.sh`. +- `make test-host-integration` runs the type-10 host-integration lane as one + Bazel test invocation of `//bazel/checks/vm:host_integration_lane_run`, + described above under "Build and validate, in detail". It is a local pre-PR + x86_64-linux lane and declares `/dev/kvm` as a precondition with no + emulation fallback. +- `make perf` invokes the advisory Bazel facade suite + `//bazel/checks:test-performance-budgets`. +- `make pre-tag` and `make smoke-lite` run the full and lite live-VM smoke + scripts. +- For an individual live-host script, set its required opt-in variables such + as `D2B_LIVE=1` and invoke the script explicitly. These scripts retain their + own safety checks and cleanup behavior. + +The heavy-gate semaphore that used to serialize these lanes - with its +`D2B_HEAVY_GATE` re-exec guard, `/run/d2b-heavy-gates` slot namespace, and +`make heavy-gate-provision` step - was removed, and nothing replaces it: +concurrent heavy lanes on one host are the caller's responsibility. + +The repository-root `Cargo.toml` and `Cargo.lock` remain rules_rs metadata +authority. Bazel remains the sole Layer-1 scheduler; do not add local +fan-out, scheduling, or wrapper machinery to Layer-2/manual lanes. For where tests live, when to add or retire each kind of test, and which pins/ledgers to update, read [`tests/AGENTS.md`](../../tests/AGENTS.md). diff --git a/docs/how-to/adding-a-test.md b/docs/how-to/adding-a-test.md index dd8bfc24f..37de6f936 100644 --- a/docs/how-to/adding-a-test.md +++ b/docs/how-to/adding-a-test.md @@ -25,7 +25,7 @@ target and where the test lives. | That a config **builds** / a schema is strict | **F** | `test-flake` | `flake.checks` (realized via `nix build`) | | One of the four global policy classes: source hygiene, workspace/lock integrity, supply chain, or changelog | **H** | `test-policy` | the corresponding narrow global policy target | | Foreign-userland portability for static binaries | **G-container** | `test-integration` | `tests/integration/containers/*.sh` under rootless podman; local host/manual pre-PR, not the PR pipeline | -| Real-kernel runtime behaviour with **no physical device** (broker sockets, cgroups, pidfd, store, network, audit, ACL, swtpm) | **G-host** | `test-host-integration` | `tests/host-integration/*.nix` runNixOSTest VM checks; local NixOS/KVM host/manual pre-PR, not the PR pipeline | +| Real-kernel runtime behaviour with **no physical device** (broker sockets, cgroups, pidfd, store, network, audit, ACL, swtpm) | **G-host** | `test-host-integration` | the type-10 Bazel host lane (`//bazel/checks/vm:host_integration_lane_run`), all assertions in Rust; x86_64-linux, needs `/dev/kvm`; local host/manual pre-PR, not the PR pipeline | ### Group F resource caveat diff --git a/docs/how-to/create-provider.md b/docs/how-to/create-provider.md index 5a0f32709..8de94c0ab 100644 --- a/docs/how-to/create-provider.md +++ b/docs/how-to/create-provider.md @@ -133,8 +133,8 @@ first twenty lines: ``` Use `container` for a foreign-userland or process fixture and -`host-integration` for a NixOS/Host/Guest fixture. Run the public lanes, which -acquire the shared heavy-test slot: +`host-integration` for a NixOS/Host/Guest fixture. Run the public lanes +directly - there is no shared heavy-test slot: ```bash make test-integration @@ -142,7 +142,7 @@ make test-host-integration ``` Validate physical-device behavior manually on an appropriate host. Do not add -an evidence script or invoke an internal heavy-lane target directly. +an evidence script or a private wrapper target around either lane. Several current Provider directories are scaffolding and intentionally have an integration README but no executable runtime fixture: diff --git a/docs/reference/compatibility.md b/docs/reference/compatibility.md index 493233f9f..e0983e2a6 100644 --- a/docs/reference/compatibility.md +++ b/docs/reference/compatibility.md @@ -30,10 +30,12 @@ consumer configuration. The acceptance sequence is owned by the host lane: 5. boot a Cloud Hypervisor Guest. The U20 `make test-host-integration` lane builds the d2b host-tool set with -local Bazel, stages it as `D2B_HOST_TOOL_BUNDLE`, and injects it into the -selected NixOS `vmChecks`. Nix must realize the test harness around those -binaries rather than rebuild `d2b`, `d2bd`, `d2b-broker`, or the injected -helper tools. +local Bazel and passes it to the guest-image action as declared Bazel label +inputs; Nix must realize the guest closure around those binaries rather than +rebuild `d2b`, `d2bd`, `d2b-broker`, or the injected helper tools. The lane +runs as `//bazel/checks/vm:host_integration_lane_run`, is x86_64-linux only, +and declares `/dev/kvm` as a precondition with no emulation fallback. +`D2B_VM_CHECK=` or `bazel test --test_filter=` selects one check. U19 does not claim that host acceptance or any remote Provider acceptance has passed. U20 must run both `make test-host-integration` and diff --git a/docs/reference/per-vm-state-ownership.md b/docs/reference/per-vm-state-ownership.md index ff1d6cb77..cbf640165 100644 --- a/docs/reference/per-vm-state-ownership.md +++ b/docs/reference/per-vm-state-ownership.md @@ -23,8 +23,8 @@ Required host permissions for the state farms and shared directories are a derives the `shared-run-dir`/`state-root` tmpfiles lines from the rows (`nixos-modules/host-daemon.nix` via `nixos-modules/state-posture-contract.nix`), and the live validation asserts the host against every row - (`tests/host-integration/state-posture-contract.nix`, run by - `make test-host-integration` with `D2B_VM_CHECK=state-posture-contract`). + (`make test-host-integration` with `D2B_VM_CHECK=state-posture-contract`, + the lane's Rust check against its Bazel-built guest image). Do not restate a posture value in prose: edit the declaration. The declaration also names the **anchor-open rule**: an anchor component of a diff --git a/docs/reference/support-matrix.md b/docs/reference/support-matrix.md index ae863e4c1..1796eb508 100644 --- a/docs/reference/support-matrix.md +++ b/docs/reference/support-matrix.md @@ -41,8 +41,10 @@ Cloud Hypervisor boot. U20 must also run both public integration lanes: `make test-host-integration` and `make test-integration`. They may run alongside the real-host testing, but neither lane is U19 evidence. An unavailable host prerequisite blocks that acceptance lane rather than becoming -a pass. `make test-host-integration` must use the existing Bazel-built -`D2B_HOST_TOOL_BUNDLE` handoff; Nix must not rebuild the injected d2b binaries. +a pass. `make test-host-integration` runs the Bazel lane +`//bazel/checks/vm:host_integration_lane_run`, which passes the Bazel-built +host tools to the guest-image action as declared label inputs; Nix must not +rebuild the injected d2b binaries. See [the compatibility policy](./compatibility.md), [the daemon lifecycle](../explanation/daemon-lifecycle.md), and diff --git a/packages/d2b-provider-device-usbip/integration/README.md b/packages/d2b-provider-device-usbip/integration/README.md index 991574aa5..6d9041f2a 100644 --- a/packages/d2b-provider-device-usbip/integration/README.md +++ b/packages/d2b-provider-device-usbip/integration/README.md @@ -1,11 +1,18 @@ # `d2b-provider-device-usbip` integration fixtures `attach_detach_lifecycle.rs` declares `host-integration` in its first line. The -scenario belongs to `make test-host-integration`, which runs through the shared -heavy-gate semaphore. It requires a booted NixOS test Host with the USBIP host -and Guest modules, Provider process lifecycle, nftables, Network relay, and a -fake approved USB backend. KVM is preferred but the host-integration lane may -use its documented fallback. +scenario belongs to `make test-host-integration`, the Bazel lane +(`//bazel/checks/vm:host_integration_lane_run`). It requires a booted NixOS +test Host with the USBIP host and Guest modules, Provider process lifecycle, +nftables, Network relay, and a fake approved USB backend. The lane declares +`/dev/kvm` as a precondition with no emulation fallback and is x86_64-linux +only; it is a local pre-PR surface, not a CI gate. + +The heavy-gate semaphore this scenario used to run behind was deleted with the +rest of the repository's heavy-gate orchestration (`CHANGELOG.md`: "Remove the +heavy-gate semaphore, self-reexec guards, and host provisioning"). Nothing +replaces it: the lanes run their own work directly, and nothing serializes +concurrent heavy lanes on one host. The scenario must prove one Host backend, one relay authority per Network, exact per-device projection apply and remove, sibling Network marker diff --git a/tests/AGENTS.md b/tests/AGENTS.md index 619b71905..6599e49cc 100644 --- a/tests/AGENTS.md +++ b/tests/AGENTS.md @@ -65,23 +65,16 @@ parity result. | # | Type | What it is | Lives in | Runs **where** | |---|------|------------|----------|----------------| | 9 | **container** | Nix-OCI image under rootless podman; proves a static binary runs on a foreign non-Nix userland | `tests/integration/containers/*.sh` + `containerImages..*` | `make test-integration` - conditional local host lane when the changed surface needs a foreign userland | -| 10 | **VM (runNixOSTest)** | boots a real NixOS VM; asserts live daemon/broker/socket-activation/host-posture/kernel behaviour | `tests/host-integration/*.nix` + `vmChecks..*` | `make test-host-integration` - conditional NixOS/KVM lane when the changed surface needs host behavior | -| 11 | **live-host** | runs against a **real deployed** d2b host; destructive/stateful | `tests/integration/live/*.sh` | through the Bazel-built xtask heavy-gate semaphore; `D2B_LIVE=1` / sudo - **manual, never CI** | - -Every retained Layer-2 tier (9-11) runs behind the Bazel-built xtask heavy-gate sole-use -semaphore, never as a raw script. Use the gated public lane target -(`make test-integration`, `make test-host-integration`; -`make pre-tag` / `make smoke-lite` for the live-VM smoke gate), or wrap an -ad-hoc live script as -`make heavy-gate-build && bazel-bin/packages/xtask/xtask heavy-gate -- env -D2B_LIVE=1 bash tests/integration/live/.sh`. - -Invoking a live script directly no longer bypasses the semaphore: it re-executes -through the gate exactly once when `D2B_HEAVY_GATE` is unset, so shared Nix -store, Bazel output tree, and KVM are not oversubscribed. **Any new live or -performance entrypoint must carry that same self-guard block**, or the -fail-closed inventory guard (`every_live_and_heavy_entrypoint_routes_through_the_gate`) -fails while walking on-disk scripts and the Makefile. +| 10 | **VM (Bazel lane)** | boots a real NixOS VM under KVM; asserts live daemon/broker/socket-activation/host-posture/kernel behaviour, all in Rust | `bazel/checks/vm/` - the lane target `//bazel/checks/vm:host_integration_lane_run`; `tests/host-integration/lib.nix` only, for the guest nodes that import it | `make test-host-integration` - local pre-PR x86_64-linux lane; `/dev/kvm` is a declared precondition with no emulation fallback | +| 11 | **live-host** | runs against a **real deployed** d2b host; destructive/stateful | `tests/integration/live/*.sh` | `make pre-tag` / `make smoke-lite`, or the script directly with its own opt-ins (`D2B_LIVE=1` / sudo) - **manual, never CI** | + +Every retained Layer-2 tier (9-11) runs its public entry point directly: +`make test-integration`, `make test-host-integration`, or the live-VM smoke +targets (`make pre-tag` / `make smoke-lite`). The heavy-gate semaphore that +used to serialize these lanes, its `D2B_HEAVY_GATE` re-exec guard, and its +host provisioning were removed; the lanes run as raw work, and nothing +schedules or serializes them, so the caller owns avoiding concurrent heavy +lanes on one host. ## How to add a test (decision rule) @@ -134,7 +127,7 @@ tests/ │ ├── distro-matrix/ distro pins/fixtures │ └── live/ type 11 D2B_LIVE (manual) └── host-integration/ - └── *.nix type 10 runNixOSTest (make test-host-integration; conditional) + └── lib.nix type 10 guest-node helpers for the Bazel host lane (make test-host-integration; local pre-PR) ``` Types 2-5 (unit/integration/contract/policy-lint) are Rust and live under @@ -202,8 +195,8 @@ add a test census, successor pin, secondary inventory, or validator. ### Retained Layer-2 and manual scripts Layer-2 container, VM, live-host, and performance scripts remain -manual or conditional surfaces. They run through the documented heavy-gate -semaphore and are not part of the Bazel Layer-1 scheduler. A shell script may +manual or conditional surfaces that run their own work directly, and are not +part of the Bazel Layer-1 scheduler. A shell script may remain under `tests/tools/` or `tests/unit/` when it is the subject of a native Bazel test, a fixture materializer, a generator, or a Layer-2 lane; it must not schedule sibling Layer-1 work. @@ -212,9 +205,10 @@ For U20 final acceptance, both public integration targets, `make test-integration` and `make test-host-integration`, are mandatory and may be scheduled alongside the `/etc/nixos` real-host switch, d2b startup, and Cloud Hypervisor Guest boot. U19 only keeps their declarations and current -inputs converged and does not run host acceptance. The host lane uses the -existing Bazel-built host-tool bundle handoff; Nix realizes the VM check -around those injected binaries and must not rebuild d2b binaries. +inputs converged and does not run host acceptance. The host lane passes the +Bazel-built host tools to the guest-image action as declared Bazel label +inputs; Nix realizes each guest around those binaries and must not rebuild +d2b binaries. ### Standalone Rust workspaces diff --git a/tests/README.md b/tests/README.md index f6eedcb8b..36eb8a3e1 100644 --- a/tests/README.md +++ b/tests/README.md @@ -16,7 +16,8 @@ that is the binding contract; this file is the human quick-start. advisory; an advisory success may be a guarded skip and is not validation evidence. - **Layer 2 - integration tiers.** Real systemd / kernel / userland: podman - containers, runNixOSTest VMs, and live-host scripts. Used only + containers, KVM-booted VMs through the Bazel host lane, and live-host + scripts. Used only when Layer 1 *provably* cannot cover the behaviour. Physical-device validation is manual operator work, not a repository evidence script. @@ -39,7 +40,7 @@ tests/ │ ├── distro-matrix/ distro pins + fixtures │ └── live/ type 11: D2B_LIVE live-host (manual) └── host-integration/ - └── *.nix type 10: runNixOSTest (make test-host-integration; conditional) + └── lib.nix type 10: guest-node helpers for the Bazel host lane (make test-host-integration; local pre-PR) ``` Rust tests (types 2-5: unit, integration, contract, policy-lint) live under @@ -67,10 +68,10 @@ The source-hygiene gate fails closed when `D2B_SHELLCHECK_BIN` is unavailable. | `make test-policy` | composed Bazel source, workspace/lock, supply-chain, and changelog policy suites | local + CI | | `make test-performance-budgets` | advisory performance canary; without `D2B_PERF_STABLE=1` it reports `SKIP` and enforces nothing | local + CI | | `make test-integration` | type-9 podman container tests | conditional local host lane (podman; not the PR pipeline) | -| `make test-host-integration` | type-10 runNixOSTest VM checks; locally builds eight host tools with Bazel, injects them into the checks, and optionally uploads their built dependency closures to Attic | conditional local NixOS host lane (KVM; TCG fallback; not the PR pipeline) | +| `make test-host-integration` | type-10 host-integration VM checks through one Bazel lane target; guests are graph outputs keyed on declared inputs and every assertion is Rust | local contributor pre-PR lane (x86_64-linux; needs `/dev/kvm`; no emulation fallback; not the PR pipeline) | | `make check-fast` | compatibility alias for `make check` | local + CI | | `make bazel-check` | Bazel aggregate suite used by `make check`. Developer Bazel and public Make aliases default to BuildBuddy remote through `.bazelrc`; CI sets `D2B_BAZEL_PROFILE=local` | local or remote | -| `make heavy-gate-build && bazel-bin/packages/xtask/xtask heavy-gate -- env D2B_LIVE=1 bash tests/integration/live/.sh` | type-11 live-host tests, through the heavy-gate semaphore | **manual, against a deployed d2b host** | +| `make pre-tag` / `make smoke-lite`, or a live script directly with its own opt-ins (`D2B_LIVE=1`, sudo) | type-11 live-host tests | **manual, against a deployed d2b host** | `make check`, `make test-unit`, and `make bazel-check` invoke the same nested suite graph through one public facade label. Public Make aliases run @@ -99,7 +100,7 @@ silent emulation fallback, so on a host without KVM it stops with a message rather than turning very slow. It is x86_64-linux only. `D2B_VM_CHECK=` runs one named check, and `bazel test --test_filter=` -works too — the lane reads Bazel's own filter as well as that variable, and the +works too - the lane reads Bazel's own filter as well as that variable, and the first of the two that names anything wins. A failing check reports under its own name in the lane's test output, with the stage it was in, the rows it was asserting on, and the guest's journal and zone dump. @@ -137,42 +138,21 @@ realizes the copied Guest workspace for dependency metadata, license, source, and audit validation; it does not compile Guest packages and is not a fifth repository-wide policy class or copied-workspace parity result. -All Layer-2 lanes (types 9-11) run behind one sole-use semaphore (two slots -per uid via open file description locks), so concurrent heavy lanes cannot -oversubscribe the shared Nix store, Bazel output tree, or KVM device. The -public lane targets above (`make test-integration`, -`make test-host-integration`, `make perf`) acquire a slot and then delegate -to a guarded internal `heavy-lane-*` target that fails closed if run outside -the gate; run the public targets, not the internal ones. `make heavy-check`, -`make heavy-flake-check`, and the `heavy-test-*` aliases run a Layer-1 gate, -the building flake check, or a public lane under the same semaphore. -Live-host scripts obey the same rule: use the gated `make pre-tag` / -`make smoke-lite` live-VM smoke entrypoints, or wrap a raw live script as -`make heavy-gate-build && bazel-bin/packages/xtask/xtask heavy-gate -- env -D2B_LIVE=1 bash tests/integration/live/.sh`. Invoking `D2B_LIVE=1 bash -tests/integration/live/.sh` directly no longer bypasses the semaphore: -each live entrypoint, plus the enforcing path of each performance -entrypoint, verifies its inherited slot and re-executes itself through the gate -exactly once when no genuine slot is held. The advisory performance skip exits -before acquiring a slot because it does no heavy work. A bare `D2B_HEAVY_GATE` -value is not trusted, so the shared Nix store, Bazel output tree, and KVM -device cannot be oversubscribed. The gated targets remain the documented path. - -The semaphore uses a protected, system-provisioned namespace under -`/run/d2b-heavy-gates`; it never falls back to a user-writable runtime or -temporary directory. The NixOS module provisions the fixed root at boot and -creates two private slots for each configured `d2b.site.launcherUsers` member -that NSS can resolve during activation. An unavailable network-backed user is -deferred rather than failing activation. After that user logs in, or on a -development machine that does not use the module, run -`make heavy-gate-provision` once per boot when the gate requests it. The target -uses the caller's numeric UID without an NSS user-name lookup and uses `sudo` -only to create the root-owned namespace and the current user's two mode-`0600` -slot files. This per-boot step is necessary because `/run` is a tmpfs. Until it -is complete, a missing or malformed namespace fails closed with stable code -`heavy-gate-provisioning-required` and names that Make target as the -remediation; do not work around it by moving the gate into `/tmp` or another -user-owned location. +All Layer-2 lanes (types 9-11) run their own work directly. `make +test-integration`, `make test-host-integration`, `make perf`, `make pre-tag`, +and `make smoke-lite` are plain invocations: there is no repository +semaphore, no re-exec wrapper, and no internal `heavy-lane-*` targets in +between. The heavy-gate semaphore (the `xtask heavy-gate` facade, its +`D2B_HEAVY_GATE` re-exec guard, the `/run/d2b-heavy-gates` slot namespace, +`make heavy-gate-provision`, and the `heavy-test-*` aliases) was removed, and +nothing replaces it, so nothing prevents two heavy lanes from running at once +on one host except the caller. `make heavy-check` and +`make heavy-flake-check` survive as plain aliases that run their work +directly, not under a gate. + +Live-host tests are run through `make pre-tag` / `make smoke-lite`, or +directly with the opt-in variables they require (`D2B_LIVE=1`, sudo). Those +scripts retain their own safety checks and cleanup behavior. Current live-host scripts include `d2b-store.sh` for per-VM store adoption and `usbip-lifecycle.sh` for USBIP attach/detach across a `d2bd` @@ -253,7 +233,7 @@ evidence script is required. The fixed workflow is committed at `.github/workflows/pr-l1-static-fast.yml` and exposes one stable required `check` result. Intermediate job names are implementation details. Layer-2 container, VM, live-host, and performance scripts -remain conditional or manual lanes behind the heavy-gate semaphore; they are +remain conditional or manual lanes that run their own work directly; they are not folded into the Layer-1 Bazel scheduler. ## Adding a test From 3c940731a17cfe556023f457f8478b12f63093cd Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sat, 26 Sep 2026 23:46:08 -0700 Subject: [PATCH 29/51] fix(vm): collapse the doubled module function in the guest-shell-service node The ported guest-shell-service node was written as a function that takes its module arguments, binds two fixture artifacts in a `let`, and then returns *another* function of the same arguments - so calling the node once yielded a function rather than a module, and the guest-image evaluation refused it: "module ... does not look like a module". The other eleven nodes are single-layer and unaffected, which is why only this check's image failed to build. The outer function already destructures `lib` and `pkgs`, so the inner arrow was redundant: dropping it makes the `let` bindings visible to the module body directly, and the node now evaluates to an attrset with its `imports`, which is what the module system wants. Found by bisecting the module list: the node alone reproduces the failure under `nix eval-config`, and forcing it with module arguments returns a lambda rather than a set. --- nix/test-support/host-integration-node.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/nix/test-support/host-integration-node.nix b/nix/test-support/host-integration-node.nix index 6cbba4d29..30f964b95 100644 --- a/nix/test-support/host-integration-node.nix +++ b/nix/test-support/host-integration-node.nix @@ -359,7 +359,6 @@ rec { PY ''; in - { lib, pkgs, ... }: { imports = [ ../../nixos-modules/component-session.nix From b5da30929c672cdba7a0b29b8e7bd7203e80b64c Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sat, 26 Sep 2026 23:57:29 -0700 Subject: [PATCH 30/51] fix(vm): defer a pool group that does not fit instead of dropping it `admit` reported a group the host could not hold alongside the others as "will run after a member finishes" and then `continue`d, dropping it from the list the runner spawns. The group never ran. The lane still reported its results and still exited green, so a host whose declared budget admitted fewer groups than the lane has invocations silently checked a subset of its own checks while the log named every one of them as part of the pool. Admission now returns the groups that fit and the groups that do not, and the runner runs the second list as a second wave once the first has finished - which is what the log already promised. The concurrency is unchanged: a wave still runs its groups together, and the second wave starts only when the first is done, so the host is never oversubscribed relative to the budget it declared. On a host where every group fits, the deferred list is empty and the run is identical to before. --- .../src/bin/d2b-test-vm-harness.rs | 71 +++++++++++++------ 1 file changed, 50 insertions(+), 21 deletions(-) diff --git a/packages/d2b-test-vm-harness/src/bin/d2b-test-vm-harness.rs b/packages/d2b-test-vm-harness/src/bin/d2b-test-vm-harness.rs index 47469e884..898902ed0 100644 --- a/packages/d2b-test-vm-harness/src/bin/d2b-test-vm-harness.rs +++ b/packages/d2b-test-vm-harness/src/bin/d2b-test-vm-harness.rs @@ -280,8 +280,8 @@ fn run_lane(arguments: Vec) -> Result<(), HarnessError> { } let groups = group_by_invocation(guests); - let admitted = admit(&groups, &work_root)?; - for group in &admitted { + let (admitted, deferred) = admit(&groups, &work_root)?; + for group in admitted.iter().chain(deferred.iter()) { report_line(&format!( "pool: {} ({} vCPU, {} MiB, {} MiB of working directory) <- {}", group.names(), @@ -292,23 +292,48 @@ fn run_lane(arguments: Vec) -> Result<(), HarnessError> { )); } - let outcomes: Vec> = - thread::scope(|scope| { - let emulator = emulator.as_path(); - let work_root = work_root.as_path(); - let handles: Vec<_> = admitted + // Two waves, not one. A group the host cannot hold alongside the others + // is deferred to a second wave, never dropped: dropping it left the lane + // reporting green over a subset of its own checks, with the log + // promising a second wave that nothing ever performed. + let mut outcomes = run_wave(&admitted, &emulator, &work_root); + if !deferred.is_empty() { + report_line(&format!( + "pool: second wave, {} now that the first wave's members have finished", + deferred .iter() - .map(|group| scope.spawn(move || run_group(group, emulator, work_root))) - .collect(); - handles - .into_iter() - .flat_map(|handle| { - handle - .join() - .unwrap_or_else(|_| panic!("a pool group thread panicked")) - }) - .collect() - }); + .map(|group| group.names()) + .collect::>() + .join(", ") + )); + outcomes.extend(run_wave(&deferred, &emulator, &work_root)); + } + +/// Run one wave of groups concurrently, and hand back what each reported. +#[allow(clippy::disallowed_methods, reason = "synchronous path")] +fn run_wave( + groups: &[InvocationGroup], + emulator: &Path, + work_root: &Path, +) -> Vec> { + if groups.is_empty() { + return Vec::new(); + } + thread::scope(|scope| { + let handles: Vec<_> = groups + .iter() + .map(|group| scope.spawn(move || run_group(group, emulator, work_root))) + .collect(); + handles + .into_iter() + .flat_map(|handle| { + handle + .join() + .unwrap_or_else(|_| panic!("a pool group thread panicked")) + }) + .collect() + }) +} /// One invocation group's checks, in the order they were selected. /// @@ -505,10 +530,11 @@ fn group_by_invocation(guests: Vec) -> Vec { /// cannot hold even the cheapest group is a host that cannot run the lane at /// all, which is worth saying plainly rather than booting a guest that /// cannot get the memory its node declared. +/// The groups that fit the budget now, and the ones deferred to a second wave. fn admit( groups: &[InvocationGroup], work_root: &Path, -) -> Result, HarnessError> { +) -> Result<(Vec, Vec), HarnessError> { let first = groups .first() .ok_or_else(|| HarnessError::Configuration("the lane has no checks to run".to_owned()))?; @@ -552,6 +578,7 @@ fn admit( let mut cores_used: u64 = 0; let mut directory_used: u64 = 0; let mut admitted = Vec::new(); + let mut deferred: Vec = Vec::new(); for (index, group) in groups.iter().enumerate() { let cost = group.footprint(); let fits_memory = memory_used + cost.memory_size_mib <= memory_budget; @@ -574,9 +601,11 @@ fn admit( } if !(fits_memory && fits_cores && fits_directory) { report_line(&format!( - "pool: {} does not fit the budget and will run after a member finishes", + "pool: {} does not fit the budget yet and will run in a second wave once the \ + first wave's members have finished", group.names() )); + deferred.push(group.clone()); continue; } memory_used += cost.memory_size_mib; @@ -599,7 +628,7 @@ fn admit( work_root.display() )); } - Ok(admitted) + Ok((admitted, deferred)) } /// Boot one check's own guest, prove the restore against it, and run the /// check on a restored copy of it. From 8c4146268737e2ce417c7bac5594ea85d7d19f60 Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sun, 27 Sep 2026 00:28:42 -0700 Subject: [PATCH 31/51] perf(vm): name the guest kernel and initrd by store path instead of copying Every direct-boot check's guest carries the same kernel and the same initrd - the system closure rides in the initrd - and the image action copied both into each of the eleven images. Measured: all eleven kernels hash to 6c3ebc5d64ca and all eleven initrds to 7096b56ae40f, so the store was holding 358 MiB of byte-identical content and every image build was copying it again. The manifest already resolved an absolute path as itself, so the store path needed no new mechanism - only the manifest had to stop asking for a relative name. An image now carries its disk and its manifest and nothing else. The per-image disk is not part of this: the resulting qcow2 files are 1.2-6.8 MiB, so there was no large duplicated image to share. The closure itself was already shared - one change to a shared input costs one closure rebuild, not eleven - which is why a base-image overlay would have bought nothing here. --- nix/test-support/guest-image.nix | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/nix/test-support/guest-image.nix b/nix/test-support/guest-image.nix index 20326d272..f9cca660a 100644 --- a/nix/test-support/guest-image.nix +++ b/nix/test-support/guest-image.nix @@ -479,8 +479,14 @@ let pool = poolBudget; boot = { method = if useBootLoader then "bootloader" else "direct"; - kernel = if useBootLoader then null else "kernel"; - initrd = if useBootLoader then null else "initrd"; + # The store paths, not copies of them. Every direct-boot check's guest + # has the same kernel and the same initrd - the closure rides in the + # initrd - so copying them into each image stored 358 MiB of + # byte-identical content eleven times over. The manifest already + # resolves an absolute path as itself, so naming the store path is all + # it takes; nothing reads them out of the image any more. + kernel = if useBootLoader then null else directBootKernel; + initrd = if useBootLoader then null else directBootInitrd; append = if useBootLoader then null else @@ -789,13 +795,12 @@ else "$qemu_img" convert -f raw -O qcow2 "$TMPDIR/root.raw" "$TMPDIR/disk.qcow2" rm -f "$TMPDIR/root.raw" - # The direct-boot shape's kernel and initrd, copied in as real files - # rather than named by store path. The manifest names them relative to - # the image root, so an image that only pointed at the host's store - # would declare files it does not carry, and the launcher would find - # them missing at boot. A copy that fails fails the image here. - cp -L ${directBootKernel} "$out/kernel" - cp -L ${directBootInitrd} "$out/initrd" + # The direct-boot shape's kernel and initrd are named by store path in + # the manifest, not copied in here. The old copy existed because the + # manifest named them relative to the image root and the launcher + # resolved them there - but the resolver has always honoured an + # absolute path, and every direct-boot guest carries the same two + # files, so eleven images were storing 358 MiB of identical content. fi mv "$TMPDIR/disk.qcow2" "$out/disk.qcow2" From 54d8310a0f1b259783cdf62bf8fd57992b6975f5 Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sun, 27 Sep 2026 00:28:42 -0700 Subject: [PATCH 32/51] fix(vm): call a silent guest stuck instead of waiting out the activation bound A full lane run took 32.7 minutes and reported nine of eleven checks passing. The other half-hour was one check: state-posture-contract's guest, which had booted, reached systemd's local-filesystems target at 40 seconds, started udev's device-event manager, and then produced nothing further on its console for the remaining 1760 seconds of a 1800-second activation bound. The console tail said exactly where it stopped; the lane simply refused to act on it. A guest that is slow is still writing to its console. A guest that has stopped writing is stuck, and the rest of the activation bound will not change that. Both activation waits now watch the console's own length and fail with a distinct `ConsoleStalled` error - naming how long the console was quiet, and carrying the same tail a bounded failure carries - once it has produced nothing for a stall bound. The stall bound is separate from the activation bound on purpose, and defaults to 180s against the observed 17-20s activations. A loaded host genuinely takes longer to boot a guest, so the generous bound stays for a guest that is talking; a guest that went quiet at 40s is now described in about three minutes instead of half an hour. Both waits read it, so it is overridable per run: D2B_TEST_VM_HARNESS_CONSOLE_STALL_SECS. A test pins the distinction, including that output resuming clears the stall rather than latching. --- .../src/bin/d2b-test-vm-harness.rs | 12 +- packages/d2b-test-vm-harness/src/error.rs | 29 ++++ packages/d2b-test-vm-harness/src/guest.rs | 129 +++++++++++++++++- 3 files changed, 168 insertions(+), 2 deletions(-) diff --git a/packages/d2b-test-vm-harness/src/bin/d2b-test-vm-harness.rs b/packages/d2b-test-vm-harness/src/bin/d2b-test-vm-harness.rs index 898902ed0..2b0539f88 100644 --- a/packages/d2b-test-vm-harness/src/bin/d2b-test-vm-harness.rs +++ b/packages/d2b-test-vm-harness/src/bin/d2b-test-vm-harness.rs @@ -40,6 +40,11 @@ const EMULATOR: &str = "D2B_TEST_VM_HARNESS_EMULATOR"; const WORK_ROOT: &str = "D2B_TEST_VM_HARNESS_WORK_ROOT"; /// How long the guest has to activate. const ACTIVATION_TIMEOUT: &str = "D2B_TEST_VM_HARNESS_ACTIVATION_TIMEOUT_SECS"; +/// How long the guest's console may produce nothing before the lane calls it +/// stuck rather than slow. A third of the activation bound, so a guest that +/// goes quiet is described in minutes while a slow one still gets the full +/// wait - the console of a booting guest writes continuously. +const CONSOLE_STALL: &str = "D2B_TEST_VM_HARNESS_CONSOLE_STALL_SECS"; /// How many boot and teardown cycles to run. const CYCLES: &str = "D2B_TEST_VM_HARNESS_CYCLES"; @@ -672,6 +677,7 @@ fn run_check_inner( let mut point = SnapshotPoint::new(&spec)?; let marker = guest.manifest.activation.marker.clone(); let activation_bound = spec.activation_timeout; + let console_stall = spec.console_stall; let mut surface = LegacyGuest::attach(&mut active)?; let fresh = surface @@ -709,6 +715,7 @@ fn run_check_inner( &guest.name, &marker, activation_bound, + console_stall, )?; let restored_marker = surface .run(&LegacyCheck::new(format!("{}-marker", guest.name), EQUIVALENCE_MARKER)) @@ -750,6 +757,7 @@ fn run_check_inner( &guest.name, &marker, activation_bound, + console_stall, )?; report_line(&format!( "{}: second restore onto the same snapshot took {second:.1}s and wrote a layer of its own", @@ -828,11 +836,12 @@ fn restore_and_measure( name: &str, marker: &str, bound: Duration, + stall: Duration, ) -> Result { let seen = active.activations(marker); let seconds = active.restore(point)?.as_secs_f64(); let waiting = Instant::now(); - active.await_reactivation(seen, bound, marker)?; + active.await_reactivation(seen, bound, stall, marker)?; let reactivation = waiting.elapsed().as_secs_f64(); surface.resync()?; report_line(&format!( @@ -966,6 +975,7 @@ fn run() -> Result, HarnessError> { "lane-harness", ); spec.activation_timeout = activation_timeout; + spec.console_stall = Duration::from_secs(optional_u64(CONSOLE_STALL, 180)?); let work_dir = spec.work_dir(); for cycle in 0..cycles { diff --git a/packages/d2b-test-vm-harness/src/error.rs b/packages/d2b-test-vm-harness/src/error.rs index 031dece01..11a3d23a6 100644 --- a/packages/d2b-test-vm-harness/src/error.rs +++ b/packages/d2b-test-vm-harness/src/error.rs @@ -54,6 +54,18 @@ pub enum HarnessError { marker: String, console_tail: String, }, + /// The guest's console stopped producing output while the lane was still + /// waiting for it to activate. + /// + /// A guest that is slow is still writing to its console; a guest that is + /// dead has stopped. Waiting the whole activation bound for a guest that + /// went quiet forty seconds in is the wait this separates from the + /// bounded one, which is what a slow guest still gets. + ConsoleStalled { + stalled: Duration, + marker: String, + console_tail: String, + }, /// The guest reported activation for a different guest shape than the one /// the launcher was asked to boot. WrongShape { @@ -114,6 +126,23 @@ impl fmt::Display for HarnessError { } Ok(()) } + Self::ConsoleStalled { + stalled, + marker, + console_tail, + } => { + write!( + formatter, + "the guest's console stopped producing output for {}s before it reported \ + {marker}, so it is not going to: the guest is stuck or gone rather than \ + slow, and waiting out the full activation bound would say nothing more", + stalled.as_secs() + )?; + if !console_tail.is_empty() { + write!(formatter, "\n--- guest console tail ---\n{console_tail}")?; + } + Ok(()) + } Self::WrongShape { expected, reported, diff --git a/packages/d2b-test-vm-harness/src/guest.rs b/packages/d2b-test-vm-harness/src/guest.rs index 6ee2c080c..c7016da49 100644 --- a/packages/d2b-test-vm-harness/src/guest.rs +++ b/packages/d2b-test-vm-harness/src/guest.rs @@ -105,6 +105,15 @@ pub struct GuestSpec { pub work_root: PathBuf, /// How long the guest has to report activation before the lane fails. pub activation_timeout: Duration, + /// How long the guest's console may produce nothing before the lane calls + /// it stuck rather than slow. + /// + /// Separate from [`Self::activation_timeout`] on purpose: that bound is + /// generous because a loaded host really does take longer to boot a + /// guest, while a console that has gone quiet is not slow. Waiting the + /// activation bound out on a guest that stopped talking is how one dead + /// guest costs half an hour. + pub console_stall: Duration, /// The guest's name on the monitor and in diagnostics. pub name: String, } @@ -124,6 +133,9 @@ impl GuestSpec { emulator: emulator.into(), work_root: work_root.into(), activation_timeout: Duration::from_secs(1200), + // Overwritten by the caller that owns the bound policy; the + // default here is the same generous fraction. + console_stall: Duration::from_secs(300), name: name.into(), } } @@ -1088,6 +1100,7 @@ impl ActiveGuest { Ok(started.elapsed()) } + /// Wait for the guest to report activation again, after a restore reset /// it. /// @@ -1096,13 +1109,27 @@ impl ActiveGuest { /// marker the first boot wrote is still in it, and a wait that looked /// for a marker would be satisfied by a guest that had not re-activated /// at all. - pub fn await_reactivation(&mut self, seen: usize, bound: Duration, marker: &str) -> Result<()> { + pub fn await_reactivation( + &mut self, + seen: usize, + bound: Duration, + stall: Duration, + marker: &str, + ) -> Result<()> { let console = self.work_dir.join("console.log"); let deadline = Instant::now() + bound; + let mut progress = ConsoleProgress::new(&console, stall); loop { if read_console(&console).matches(marker).count() > seen { return Ok(()); } + if let Some(stalled) = progress.observe() { + return Err(HarnessError::ConsoleStalled { + stalled, + marker: marker.to_owned(), + console_tail: activation_failure_tail(&read_console(&console)), + }); + } if Instant::now() >= deadline { return Err(HarnessError::NotActivated { bound, @@ -1166,6 +1193,51 @@ impl ActiveGuest { } } } +/// The console's own progress, so a guest that has gone quiet is told apart +/// from one that is slow. +/// +/// The activation bound is generous because a loaded host genuinely takes +/// longer to boot a guest. A console that has stopped growing is a different +/// thing: the guest is stuck, and the rest of the bound will not change that. +/// Watching the file's length is what tells the two apart, and it costs a +/// `stat` per poll. +struct ConsoleProgress { + console: PathBuf, + length: u64, + changed: Instant, + stall: Duration, +} + +impl ConsoleProgress { + fn new(console: &Path, stall: Duration) -> Self { + Self { + console: console.to_path_buf(), + length: Self::len(console), + changed: Instant::now(), + stall, + } + } + + /// How long the console has produced nothing, or `None` while it is still + /// within the stall bound. + fn observe(&mut self) -> Option { + let now = Instant::now(); + let current = Self::len(&self.console); + if current != self.length { + self.length = current; + self.changed = now; + return None; + } + let quiet = now.duration_since(self.changed); + (quiet >= self.stall).then_some(quiet) + } + + /// A console the launcher cannot stat is treated as silent rather than as + /// absent: a missing file is a guest that produced nothing at all. + fn len(console: &Path) -> u64 { + fs::metadata(console).map(|meta| meta.len()).unwrap_or(0) + } +} impl Drop for ActiveGuest { /// A guest that reaches the end of its scope without a `shutdown` - a @@ -1334,6 +1406,7 @@ fn wait_for_activation( let console = work_dir.join("console.log"); let marker = spec.manifest.activation.marker.as_str(); let deadline = Instant::now() + spec.activation_timeout; + let mut progress = ConsoleProgress::new(&console, spec.console_stall); loop { if let Some(line) = activation_line(&read_console(&console), marker) { check_shape(&spec.manifest.activation.shape, &line)?; @@ -1355,6 +1428,14 @@ fn wait_for_activation( ), }); } + if let Some(stalled) = progress.observe() { + let _ = monitor.take_events(); + return Err(HarnessError::ConsoleStalled { + stalled, + marker: marker.to_owned(), + console_tail: activation_failure_tail(&read_console(&console)), + }); + } if Instant::now() >= deadline { return Err(HarnessError::NotActivated { bound: spec.activation_timeout, @@ -2141,3 +2222,49 @@ mod tests { ); } } + +#[cfg(test)] +mod console_progress_tests { + use super::ConsoleProgress; + use std::{fs, path::PathBuf, time::Duration}; + + fn scratch(name: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "d2b-console-progress-{}-{name}", + std::process::id() + )); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).expect("a scratch directory"); + dir.join("console.log") + } + + #[test] + fn a_console_that_stops_growing_is_reported_and_a_resumed_one_is_not() { + let console = scratch("stall"); + fs::write(&console, b"booting\n").expect("a console with output"); + let mut progress = ConsoleProgress::new(&console, Duration::from_millis(50)); + + // A guest that is still writing is slow, not stuck, and must not be + // called stalled while it writes. + assert!( + progress.observe().is_none(), + "a console that is still growing must not be reported as stalled" + ); + + std::thread::sleep(Duration::from_millis(70)); + assert!( + progress.observe().is_some(), + "a console that has stopped growing must be reported as stalled" + ); + + // And a guest that starts talking again is back to being slow, not + // stuck: the watch follows the output rather than latching. + fs::write(&console, b"booting\nmore\n").expect("the console speaks again"); + assert!( + progress.observe().is_none(), + "output resuming must clear the stall" + ); + + let _ = fs::remove_file(&console); + } +} From 6bdba98188a635b20ec84aa359a2cd5b43c30cf0 Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sun, 27 Sep 2026 01:15:40 -0700 Subject: [PATCH 33/51] fix(vm): bring the activation bound down to something a guest can exceed The activation bound was 1800s against a measured healthy activation of 17-20s. That is ninety times the time a working guest takes, so it was not a bound so much as an abandonment, and it dominated the lane: a full run took 32.7 minutes and spent half of it inside this single wait, on a guest that reached systemd's local-filesystems target at 40 seconds, started udev's device-event manager, and never advanced again. 600s is thirty times the observed maximum, which still leaves a heavily loaded host room, and the number stays overridable per run through D2B_TEST_VM_HARNESS_ACTIVATION_TIMEOUT_SECS for a host that needs more. This is separate from the console-stall guard in the previous commit, and the two cover different failures. That guard handles a console that has gone silent; this guest's console kept writing - a store-sync heartbeat every thirty seconds - while never reporting activation, so the guard correctly declined to fire and the bound was what the run waited on. A guest that is alive and never arrives is not the same defect as a guest that stopped talking, and only the bound distinguishes them. --- .../src/bin/d2b-test-vm-harness.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/d2b-test-vm-harness/src/bin/d2b-test-vm-harness.rs b/packages/d2b-test-vm-harness/src/bin/d2b-test-vm-harness.rs index 2b0539f88..ca1fd41e6 100644 --- a/packages/d2b-test-vm-harness/src/bin/d2b-test-vm-harness.rs +++ b/packages/d2b-test-vm-harness/src/bin/d2b-test-vm-harness.rs @@ -666,9 +666,14 @@ fn run_check_inner( work_root, &guest.name, ); - spec.activation_timeout = Duration::from_secs( - optional_u64(ACTIVATION_TIMEOUT, 1800)?, - ); + // 600s, against a measured healthy activation of 17-20s. The bound was + // 1800s, which is ninety times the time a working guest takes and is + // not a bound so much as an abandonment: a full lane run spent half of + // its 32.7 minutes inside this one wait, on a guest that reached + // systemd's local-filesystems target at 40s and then stopped advancing. + // Thirty times the observed maximum still leaves a heavily loaded host + // room; the number is overridable per run for one that needs more. + spec.activation_timeout = Duration::from_secs(optional_u64(ACTIVATION_TIMEOUT, 600)?); let mut active = boot(&spec)?; // The declared pass said this guest's drives can be snapshotted; the // running guest's own block graph is the authority, and it is asked From 450edaa85182bcf91799d530ac5c8ff15dcd075d Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sun, 27 Sep 2026 01:19:23 -0700 Subject: [PATCH 34/51] fix(vm): stop the lane's results being replayed from Bazel's cache `lane_test`'s contract says "the target's result is never cacheable. A guest's verdict depends on what the host did while it ran, so a second invocation re-runs every selected check rather than replaying what the first one concluded, and `no-cache` is what says that to the Bazel graph." The tag list did not contain `no-cache`. It was `exclusive`, `local`, `no-remote-cache`, `no-remote-exec` and `no-sandbox` - which keep the lane off the remote cache and the remote executors, and say nothing about the local one. So the lane's verdicts were cached locally like any other test's. Found by trying to repeat a run: three consecutive invocations of `--test_filter=state-posture-contract` all reported "PASSED in 92.9s" without executing, the second and third served from the first. A green from an earlier run was being reported as a green from this one, on a test whose whole value is that it boots real guests and reports what happened to them. --- bazel/checks/vm/defs.bzl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bazel/checks/vm/defs.bzl b/bazel/checks/vm/defs.bzl index d101f4e91..eee9a0633 100644 --- a/bazel/checks/vm/defs.bzl +++ b/bazel/checks/vm/defs.bzl @@ -320,9 +320,16 @@ export D2B_TEST_VM_HARNESS_WORK_ROOT="{work_root}" exec "$runfiles/__HARNESS__" "$@" """ +# `no-cache` is the one that says a verdict is never replayed. Without it the +# lane's results are cached locally like any other test's, so a second +# invocation of an unchanged graph returns the first invocation's pass +# instead of running the checks - which is exactly what a test whose verdict +# depends on what the host did while it ran must never do. It was described +# in `lane_test`'s contract and missing from this list. _LANE_TAGS = [ "exclusive", "local", + "no-cache", "no-remote-cache", "no-remote-exec", "no-sandbox", From bf5771da838936c227825fc5127a2e1001603dbd Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sun, 27 Sep 2026 01:28:22 -0700 Subject: [PATCH 35/51] docs(changelog): record the lane's build, filter and timing work --- changelog.d/v3.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/changelog.d/v3.md b/changelog.d/v3.md index 373b2cb20..7e46b6a4d 100644 --- a/changelog.d/v3.md +++ b/changelog.d/v3.md @@ -15,3 +15,12 @@ - **The deferred Gateway-isolation check, and with it the coverage it carried.** `tests/host-integration/deferred/host-zone-gateway-isolation.nix` asserted that a Gateway Guest's relay credential never materializes on the host: the credential's canary is absent from `/etc/d2b/{zones,bundle.json,allocator.json}`, `/var/lib/d2b`, `/run/d2b`, `/var/log`, `/var/lib/d2b/audit`, the journal and coredump output, and absent from every `d2bd` and broker process's environ, cmdline and fd table, with no relay socket established. **No retained check carries that assertion.** All eleven checks were reviewed for it - the ported modules and the four fixtures that still existed at review time - and the tokens `gateway`, `relay`, `canary` and `SharedAccessKey` appear in none of them, nor does any module sweep host paths or process tables for a guest-held secret. This is a coverage **removal**, not a consolidation: there is no successor check and nothing here replaces it. If a future reader greps for the relay canary and finds nothing, this entry is the record that the absence was decided rather than overlooked. - The `vmChecks` flake output and the `D2B_HOST_TOOL_BUNDLE` / `D2B_CH_CONTROLLER_BUNDLE` environment handoff only it read, along with the Attic cache preflight and closure upload the retired nix recipe carried. The guest-image action declares its own substituters and preflights them itself, so cache handling now lives with the build that needs it rather than in a second place that can drift out of step with it. + +### Performance + +- The full eleven-check lane now runs in about eight minutes on a 12-vCPU host, against thirty-two before. Three things moved it, and none of them is the checks getting faster. + - `--test_filter` is honoured. The lane read only its own `--check` argument and `D2B_VM_CHECK`, so `bazel test --test_filter=` looked like it selected one check and quietly ran all eleven, booting a full pool and reporting verdicts for checks nobody asked about. It reads `TESTBRIDGE_TEST_ONLY` now, after the two existing sources, and the first that names anything wins. + - The guest's kernel and initrd are named by store path rather than copied into each image. All eleven images were carrying the same two files - the closure rides in the initrd - so the store held 358 MiB of byte-identical content and every image build copied it again. + - The activation bound is 600s against a measured healthy activation of 17-20s, where it was 1800s. A full run was spending half of thirty-two minutes inside that one wait. +- The activation waits now distinguish a guest that has gone quiet from one that is slow. A console that has produced nothing for 180s fails with a distinct error naming how long it was quiet, instead of the full bound elapsing. D2B_TEST_VM_HARNESS_CONSOLE_STALL_SECS overrides it. A live guest that never reports activation is a different defect and still waits out the bound - only the bound distinguishes it. +- A pool group the host cannot hold alongside the others is now deferred to a second wave rather than dropped. It was logged as "will run after a member finishes" and then dropped, so a host that could not admit every group reported green over a subset of its own checks. From 62c49f5e5c5f599af2e263a86a174a2809fe06cf Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sun, 27 Sep 2026 02:02:38 -0700 Subject: [PATCH 36/51] fix(vm): create the per-guest runtime tree the swtpm worker binds its socket under The `path:vm-run:` storage row declares a creator; nothing created it. `/run/d2b/vms` did not exist on the host, and the broker's socket grant refuses every worker whose runtime socket path has an absent ancestor - which is every worker except the one-shot flush, because the flush binds no runtime socket. That asymmetry is why this presented as a TPM fault. Verified on the guest: `find /run/d2b/vms` now reports `drwxrwx--T d2bd:d2b /run/d2b/vms`, mode 1770 and owner d2bd:d2b, matching the storage row exactly. It belongs in host-daemon.nix rather than host-broker.nix because `/run/d2b` and its ACLs are canonical there and host-broker deliberately does not touch them. With the directory in place the worker gets past the grant, and the refusal reason the check could never see is now in the log - the detail is the `forward_rendezvous` logging added earlier, which had never fired before because the grant refused before it: forwarded invocation refused with a reason code="handler-refused" detail="spawn-process: handler-errored (spawn-process: swtpm-dir hardening failed: swtpm-dir-state-not-provisioned)" That is the next defect, and it is not this one. `device-worker-launch` still fails, now on a different and much more specific ground. The check gains a row dump for the two accounts the journal and the existing row dumps could not give: the `vm-run` storage row, which the swtpm storage rows filter (`test("swtpm")`) could not show by construction, and the broker's audit log, which records the spawn decisions the journal only summarises. Without them a guest with that row and a guest without it read identically. --- nixos-modules/host-daemon.nix | 13 ++++++++++++ .../src/checks/device_worker_launch.rs | 21 +++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/nixos-modules/host-daemon.nix b/nixos-modules/host-daemon.nix index 223b67742..3554c5be3 100644 --- a/nixos-modules/host-daemon.nix +++ b/nixos-modules/host-daemon.nix @@ -147,6 +147,19 @@ in "d /run/d2b/locks 0700 d2bd d2bd -" "d /run/d2b/locks/usbip 0750 root d2bd -" "d /run/d2b/state 0700 d2bd d2bd -" + # The per-guest runtime tree a Device's worker binds its control + # socket under, and the path the `path:vm-run:` storage row + # declares. That row names a creator; this is the creator. Without + # it the row is a declaration nothing acts on, `/run/d2b/vms` does + # not exist, and the broker's socket grant refuses every worker that + # binds a runtime socket - which is every worker except the one-shot + # flush, so the failure looks like a TPM problem and is not one. + # + # Mode and ownership match the storage row exactly (1770 d2bd:d2b): + # the worker enters as d2bd through the d2b group, and the two + # declarations of this directory have to agree or the grant refuses + # on posture rather than on absence. + "d /run/d2b/vms 1770 d2bd d2b -" ] ++ posture.tmpfilesRule "state-root" "." ++ [ diff --git a/packages/d2b-test-vm-harness/src/checks/device_worker_launch.rs b/packages/d2b-test-vm-harness/src/checks/device_worker_launch.rs index 6febea9dc..cd4bba0ba 100644 --- a/packages/d2b-test-vm-harness/src/checks/device_worker_launch.rs +++ b/packages/d2b-test-vm-harness/src/checks/device_worker_launch.rs @@ -1005,6 +1005,27 @@ fn row_dumps() -> Vec<(String, String)> { ) .to_owned(), )); + // The two accounts the journal and the row dumps between them could not + // give. The swtpm storage rows above filter on `swtpm`, which by + // construction cannot show the per-guest runtime dir a device worker + // binds its socket under - so a guest that has that row and a guest that + // does not read identically here. And the broker's audit log records the + // spawn decisions the journal only summarises, so when the row sits + // Pending with `w1-swtpm` matching nothing in the journal, this is what + // says whether the swtpm-dir fence and the spawn plan ran at all. + dumps.push(( + "per-guest runtime dir storage row and the broker's spawn audit".to_owned(), + concat!( + "jq -c '[.paths[] | select(.id | test(\"vm-run\")) | {id, scope, ", + "pathTemplate, mode, owner, group, creator}]' /etc/d2b/storage.json ", + "|| echo 'no vm-run row in the storage contract'; echo '--- audit ---'; ", + "grep -hoE '\"(event|op|kind)\":\"[^\"]*\"' /var/lib/d2b/audit/broker-*.jsonl ", + "2>/dev/null | sort | uniq -c | sort -rn | head -25; echo '--- swtpm audit ---'; ", + "grep -h 'swtpm\\|SpawnRunner\\|PrepareSwtpm' /var/lib/d2b/audit/broker-*.jsonl ", + "2>/dev/null | tail -20 || echo 'no swtpm record in the broker audit log'; true", + ) + .to_owned(), + )); dumps.push(( "site artifact".to_owned(), "cat /etc/d2b/site.json 2>/dev/null || echo 'no site.json'".to_owned(), From 8d19bad7248b9cd050810ab335caff61f47a3a5b Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sun, 27 Sep 2026 06:35:23 -0700 Subject: [PATCH 37/51] chore(policy): regenerate the package policy inputs on top of the audit-corpus disposal The inputs are generated, so the disposal of the whole-tree Rust audit corpus changes their digests; they are regenerated rather than resolved by hand. --- .../broker-default-tests/policy/closure.json | 2 +- .../broker-default-tests/policy/metadata.json | 4 ++-- .../production/closure.json | 2 +- .../production/metadata.json | 4 ++-- .../policy/closure.json | 2 +- .../policy/metadata.json | 4 ++-- .../production/closure.json | 2 +- .../production/metadata.json | 4 ++-- .../policy/closure.json | 2 +- .../policy/metadata.json | 4 ++-- .../production/closure.json | 2 +- .../production/metadata.json | 4 ++-- .../broker-production/policy/closure.json | 2 +- .../broker-production/policy/metadata.json | 4 ++-- .../broker-production/production/closure.json | 2 +- .../production/metadata.json | 4 ++-- .../main-product/policy/Cargo.lock | 9 ++++++++ .../main-product/policy/closure.json | 23 ++++++++++++++++++- .../main-product/policy/metadata.json | 10 ++++---- .../main-product/production/Cargo.lock | 9 ++++++++ .../main-product/production/closure.json | 23 ++++++++++++++++++- .../main-product/production/metadata.json | 10 ++++---- .../broker-default-tests/policy/closure.json | 2 +- .../broker-default-tests/policy/metadata.json | 4 ++-- .../production/closure.json | 2 +- .../production/metadata.json | 4 ++-- .../policy/closure.json | 2 +- .../policy/metadata.json | 4 ++-- .../production/closure.json | 2 +- .../production/metadata.json | 4 ++-- .../policy/closure.json | 2 +- .../policy/metadata.json | 4 ++-- .../production/closure.json | 2 +- .../production/metadata.json | 4 ++-- .../broker-production/policy/closure.json | 2 +- .../broker-production/policy/metadata.json | 4 ++-- .../broker-production/production/closure.json | 2 +- .../production/metadata.json | 4 ++-- .../main-product/policy/Cargo.lock | 9 ++++++++ .../main-product/policy/closure.json | 23 ++++++++++++++++++- .../main-product/policy/metadata.json | 10 ++++---- .../main-product/production/Cargo.lock | 9 ++++++++ .../main-product/production/closure.json | 23 ++++++++++++++++++- .../main-product/production/metadata.json | 10 ++++---- 44 files changed, 196 insertions(+), 68 deletions(-) diff --git a/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-default-tests/policy/closure.json b/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-default-tests/policy/closure.json index 080ff6a6c..5c85cb748 100644 --- a/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-default-tests/policy/closure.json +++ b/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-default-tests/policy/closure.json @@ -10,7 +10,7 @@ "features": [], "default_features": false, "source_authority": "Cargo.lock", - "lock_sha256": "c81d3e88744fa28916db65e5d3efa094ec0a58abddbe3dfaf9b0daffad5ff082", + "lock_sha256": "b2560e727ac6b3aec69c37e31456f3093616ced8e607f82d1d69d0fd87108765", "packages": [ { "id": "adler2@2.0.1#registry+https://github.com/rust-lang/crates.io-index", diff --git a/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-default-tests/policy/metadata.json b/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-default-tests/policy/metadata.json index babeda2a9..b2bd9b297 100644 --- a/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-default-tests/policy/metadata.json +++ b/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-default-tests/policy/metadata.json @@ -2,7 +2,7 @@ "authority": "cargo-locked-metadata", "defaultFeatures": false, "features": [], - "lockSha256": "c81d3e88744fa28916db65e5d3efa094ec0a58abddbe3dfaf9b0daffad5ff082", + "lockSha256": "b2560e727ac6b3aec69c37e31456f3093616ced8e607f82d1d69d0fd87108765", "policyPackageCount": 80, "policyPackageIds": [ "adler2@2.0.1#registry+https://github.com/rust-lang/crates.io-index", @@ -87,7 +87,7 @@ "zmij@1.0.23#registry+https://github.com/rust-lang/crates.io-index" ], "productionPackageCount": 71, - "resolveNodeCount": 413, + "resolveNodeCount": 414, "roots": [ "d2b-broker" ], diff --git a/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-default-tests/production/closure.json b/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-default-tests/production/closure.json index ec5f63d82..33545ef87 100644 --- a/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-default-tests/production/closure.json +++ b/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-default-tests/production/closure.json @@ -10,7 +10,7 @@ "features": [], "default_features": false, "source_authority": "Cargo.lock", - "lock_sha256": "c81d3e88744fa28916db65e5d3efa094ec0a58abddbe3dfaf9b0daffad5ff082", + "lock_sha256": "b2560e727ac6b3aec69c37e31456f3093616ced8e607f82d1d69d0fd87108765", "packages": [ { "id": "adler2@2.0.1#registry+https://github.com/rust-lang/crates.io-index", diff --git a/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-default-tests/production/metadata.json b/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-default-tests/production/metadata.json index babeda2a9..b2bd9b297 100644 --- a/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-default-tests/production/metadata.json +++ b/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-default-tests/production/metadata.json @@ -2,7 +2,7 @@ "authority": "cargo-locked-metadata", "defaultFeatures": false, "features": [], - "lockSha256": "c81d3e88744fa28916db65e5d3efa094ec0a58abddbe3dfaf9b0daffad5ff082", + "lockSha256": "b2560e727ac6b3aec69c37e31456f3093616ced8e607f82d1d69d0fd87108765", "policyPackageCount": 80, "policyPackageIds": [ "adler2@2.0.1#registry+https://github.com/rust-lang/crates.io-index", @@ -87,7 +87,7 @@ "zmij@1.0.23#registry+https://github.com/rust-lang/crates.io-index" ], "productionPackageCount": 71, - "resolveNodeCount": 413, + "resolveNodeCount": 414, "roots": [ "d2b-broker" ], diff --git a/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-fake-backends-tests/policy/closure.json b/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-fake-backends-tests/policy/closure.json index 1d302de04..e4b3e2a20 100644 --- a/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-fake-backends-tests/policy/closure.json +++ b/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-fake-backends-tests/policy/closure.json @@ -12,7 +12,7 @@ ], "default_features": false, "source_authority": "Cargo.lock", - "lock_sha256": "c81d3e88744fa28916db65e5d3efa094ec0a58abddbe3dfaf9b0daffad5ff082", + "lock_sha256": "b2560e727ac6b3aec69c37e31456f3093616ced8e607f82d1d69d0fd87108765", "packages": [ { "id": "adler2@2.0.1#registry+https://github.com/rust-lang/crates.io-index", diff --git a/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-fake-backends-tests/policy/metadata.json b/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-fake-backends-tests/policy/metadata.json index 613d55754..5498b0b52 100644 --- a/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-fake-backends-tests/policy/metadata.json +++ b/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-fake-backends-tests/policy/metadata.json @@ -4,7 +4,7 @@ "features": [ "fake-backends" ], - "lockSha256": "c81d3e88744fa28916db65e5d3efa094ec0a58abddbe3dfaf9b0daffad5ff082", + "lockSha256": "b2560e727ac6b3aec69c37e31456f3093616ced8e607f82d1d69d0fd87108765", "policyPackageCount": 80, "policyPackageIds": [ "adler2@2.0.1#registry+https://github.com/rust-lang/crates.io-index", @@ -89,7 +89,7 @@ "zmij@1.0.23#registry+https://github.com/rust-lang/crates.io-index" ], "productionPackageCount": 71, - "resolveNodeCount": 413, + "resolveNodeCount": 414, "roots": [ "d2b-broker" ], diff --git a/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-fake-backends-tests/production/closure.json b/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-fake-backends-tests/production/closure.json index d67e60d1f..121d8c2fe 100644 --- a/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-fake-backends-tests/production/closure.json +++ b/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-fake-backends-tests/production/closure.json @@ -12,7 +12,7 @@ ], "default_features": false, "source_authority": "Cargo.lock", - "lock_sha256": "c81d3e88744fa28916db65e5d3efa094ec0a58abddbe3dfaf9b0daffad5ff082", + "lock_sha256": "b2560e727ac6b3aec69c37e31456f3093616ced8e607f82d1d69d0fd87108765", "packages": [ { "id": "adler2@2.0.1#registry+https://github.com/rust-lang/crates.io-index", diff --git a/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-fake-backends-tests/production/metadata.json b/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-fake-backends-tests/production/metadata.json index 613d55754..5498b0b52 100644 --- a/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-fake-backends-tests/production/metadata.json +++ b/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-fake-backends-tests/production/metadata.json @@ -4,7 +4,7 @@ "features": [ "fake-backends" ], - "lockSha256": "c81d3e88744fa28916db65e5d3efa094ec0a58abddbe3dfaf9b0daffad5ff082", + "lockSha256": "b2560e727ac6b3aec69c37e31456f3093616ced8e607f82d1d69d0fd87108765", "policyPackageCount": 80, "policyPackageIds": [ "adler2@2.0.1#registry+https://github.com/rust-lang/crates.io-index", @@ -89,7 +89,7 @@ "zmij@1.0.23#registry+https://github.com/rust-lang/crates.io-index" ], "productionPackageCount": 71, - "resolveNodeCount": 413, + "resolveNodeCount": 414, "roots": [ "d2b-broker" ], diff --git a/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-layer1-bootstrap-tests/policy/closure.json b/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-layer1-bootstrap-tests/policy/closure.json index e7cee1fdc..40f7152b4 100644 --- a/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-layer1-bootstrap-tests/policy/closure.json +++ b/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-layer1-bootstrap-tests/policy/closure.json @@ -12,7 +12,7 @@ ], "default_features": false, "source_authority": "Cargo.lock", - "lock_sha256": "c81d3e88744fa28916db65e5d3efa094ec0a58abddbe3dfaf9b0daffad5ff082", + "lock_sha256": "b2560e727ac6b3aec69c37e31456f3093616ced8e607f82d1d69d0fd87108765", "packages": [ { "id": "adler2@2.0.1#registry+https://github.com/rust-lang/crates.io-index", diff --git a/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-layer1-bootstrap-tests/policy/metadata.json b/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-layer1-bootstrap-tests/policy/metadata.json index 745968572..a066e0bef 100644 --- a/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-layer1-bootstrap-tests/policy/metadata.json +++ b/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-layer1-bootstrap-tests/policy/metadata.json @@ -4,7 +4,7 @@ "features": [ "layer1-bootstrap" ], - "lockSha256": "c81d3e88744fa28916db65e5d3efa094ec0a58abddbe3dfaf9b0daffad5ff082", + "lockSha256": "b2560e727ac6b3aec69c37e31456f3093616ced8e607f82d1d69d0fd87108765", "policyPackageCount": 80, "policyPackageIds": [ "adler2@2.0.1#registry+https://github.com/rust-lang/crates.io-index", @@ -89,7 +89,7 @@ "zmij@1.0.23#registry+https://github.com/rust-lang/crates.io-index" ], "productionPackageCount": 71, - "resolveNodeCount": 413, + "resolveNodeCount": 414, "roots": [ "d2b-broker" ], diff --git a/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-layer1-bootstrap-tests/production/closure.json b/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-layer1-bootstrap-tests/production/closure.json index 1c03e8719..bd267aa76 100644 --- a/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-layer1-bootstrap-tests/production/closure.json +++ b/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-layer1-bootstrap-tests/production/closure.json @@ -12,7 +12,7 @@ ], "default_features": false, "source_authority": "Cargo.lock", - "lock_sha256": "c81d3e88744fa28916db65e5d3efa094ec0a58abddbe3dfaf9b0daffad5ff082", + "lock_sha256": "b2560e727ac6b3aec69c37e31456f3093616ced8e607f82d1d69d0fd87108765", "packages": [ { "id": "adler2@2.0.1#registry+https://github.com/rust-lang/crates.io-index", diff --git a/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-layer1-bootstrap-tests/production/metadata.json b/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-layer1-bootstrap-tests/production/metadata.json index 745968572..a066e0bef 100644 --- a/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-layer1-bootstrap-tests/production/metadata.json +++ b/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-layer1-bootstrap-tests/production/metadata.json @@ -4,7 +4,7 @@ "features": [ "layer1-bootstrap" ], - "lockSha256": "c81d3e88744fa28916db65e5d3efa094ec0a58abddbe3dfaf9b0daffad5ff082", + "lockSha256": "b2560e727ac6b3aec69c37e31456f3093616ced8e607f82d1d69d0fd87108765", "policyPackageCount": 80, "policyPackageIds": [ "adler2@2.0.1#registry+https://github.com/rust-lang/crates.io-index", @@ -89,7 +89,7 @@ "zmij@1.0.23#registry+https://github.com/rust-lang/crates.io-index" ], "productionPackageCount": 71, - "resolveNodeCount": 413, + "resolveNodeCount": 414, "roots": [ "d2b-broker" ], diff --git a/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-production/policy/closure.json b/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-production/policy/closure.json index 699689afb..512d7046e 100644 --- a/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-production/policy/closure.json +++ b/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-production/policy/closure.json @@ -10,7 +10,7 @@ "features": [], "default_features": false, "source_authority": "Cargo.lock", - "lock_sha256": "c81d3e88744fa28916db65e5d3efa094ec0a58abddbe3dfaf9b0daffad5ff082", + "lock_sha256": "b2560e727ac6b3aec69c37e31456f3093616ced8e607f82d1d69d0fd87108765", "packages": [ { "id": "adler2@2.0.1#registry+https://github.com/rust-lang/crates.io-index", diff --git a/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-production/policy/metadata.json b/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-production/policy/metadata.json index babeda2a9..b2bd9b297 100644 --- a/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-production/policy/metadata.json +++ b/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-production/policy/metadata.json @@ -2,7 +2,7 @@ "authority": "cargo-locked-metadata", "defaultFeatures": false, "features": [], - "lockSha256": "c81d3e88744fa28916db65e5d3efa094ec0a58abddbe3dfaf9b0daffad5ff082", + "lockSha256": "b2560e727ac6b3aec69c37e31456f3093616ced8e607f82d1d69d0fd87108765", "policyPackageCount": 80, "policyPackageIds": [ "adler2@2.0.1#registry+https://github.com/rust-lang/crates.io-index", @@ -87,7 +87,7 @@ "zmij@1.0.23#registry+https://github.com/rust-lang/crates.io-index" ], "productionPackageCount": 71, - "resolveNodeCount": 413, + "resolveNodeCount": 414, "roots": [ "d2b-broker" ], diff --git a/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-production/production/closure.json b/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-production/production/closure.json index 91a7d2452..20a9582d6 100644 --- a/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-production/production/closure.json +++ b/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-production/production/closure.json @@ -10,7 +10,7 @@ "features": [], "default_features": false, "source_authority": "Cargo.lock", - "lock_sha256": "c81d3e88744fa28916db65e5d3efa094ec0a58abddbe3dfaf9b0daffad5ff082", + "lock_sha256": "b2560e727ac6b3aec69c37e31456f3093616ced8e607f82d1d69d0fd87108765", "packages": [ { "id": "adler2@2.0.1#registry+https://github.com/rust-lang/crates.io-index", diff --git a/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-production/production/metadata.json b/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-production/production/metadata.json index babeda2a9..b2bd9b297 100644 --- a/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-production/production/metadata.json +++ b/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/broker-production/production/metadata.json @@ -2,7 +2,7 @@ "authority": "cargo-locked-metadata", "defaultFeatures": false, "features": [], - "lockSha256": "c81d3e88744fa28916db65e5d3efa094ec0a58abddbe3dfaf9b0daffad5ff082", + "lockSha256": "b2560e727ac6b3aec69c37e31456f3093616ced8e607f82d1d69d0fd87108765", "policyPackageCount": 80, "policyPackageIds": [ "adler2@2.0.1#registry+https://github.com/rust-lang/crates.io-index", @@ -87,7 +87,7 @@ "zmij@1.0.23#registry+https://github.com/rust-lang/crates.io-index" ], "productionPackageCount": 71, - "resolveNodeCount": 413, + "resolveNodeCount": 414, "roots": [ "d2b-broker" ], diff --git a/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/main-product/policy/Cargo.lock b/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/main-product/policy/Cargo.lock index 3e4dfe0f5..961359eae 100644 --- a/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/main-product/policy/Cargo.lock +++ b/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/main-product/policy/Cargo.lock @@ -2301,6 +2301,15 @@ dependencies = [ ] +[[package]] +name = "d2b-test-vm-harness" +version = "0.0.0-bootstrap" +dependencies = [ + "serde", + "serde_json", +] + + [[package]] name = "d2b-unsafe-local-helper" version = "0.0.0-bootstrap" diff --git a/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/main-product/policy/closure.json b/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/main-product/policy/closure.json index 0076f9318..2742e2b08 100644 --- a/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/main-product/policy/closure.json +++ b/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/main-product/policy/closure.json @@ -93,13 +93,14 @@ "d2b-session-unix", "d2b-sk-frontend", "d2b-telemetry", + "d2b-test-vm-harness", "d2b-unsafe-local-helper", "d2b-zone-routing" ], "features": [], "default_features": true, "source_authority": "Cargo.lock", - "lock_sha256": "c81d3e88744fa28916db65e5d3efa094ec0a58abddbe3dfaf9b0daffad5ff082", + "lock_sha256": "b2560e727ac6b3aec69c37e31456f3093616ced8e607f82d1d69d0fd87108765", "packages": [ { "id": "adler2@2.0.1#registry+https://github.com/rust-lang/crates.io-index", @@ -1293,6 +1294,14 @@ "checksum": null, "target": "aarch64-unknown-linux-gnu" }, + { + "id": "d2b-test-vm-harness@0.0.0-bootstrap#path", + "name": "d2b-test-vm-harness", + "version": "0.0.0-bootstrap", + "source": null, + "checksum": null, + "target": "aarch64-unknown-linux-gnu" + }, { "id": "d2b-unsafe-local-helper@0.0.0-bootstrap#path", "name": "d2b-unsafe-local-helper", @@ -9413,6 +9422,18 @@ "kind": "normal", "target": null }, + { + "from": "d2b-test-vm-harness@0.0.0-bootstrap#path", + "to": "serde@1.0.229#registry+https://github.com/rust-lang/crates.io-index", + "kind": "normal", + "target": null + }, + { + "from": "d2b-test-vm-harness@0.0.0-bootstrap#path", + "to": "serde_json@1.0.151#registry+https://github.com/rust-lang/crates.io-index", + "kind": "normal", + "target": null + }, { "from": "d2b-unsafe-local-helper@0.0.0-bootstrap#path", "to": "clap@4.6.6#registry+https://github.com/rust-lang/crates.io-index", diff --git a/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/main-product/policy/metadata.json b/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/main-product/policy/metadata.json index 2c06ad7d0..9d0d3b5c3 100644 --- a/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/main-product/policy/metadata.json +++ b/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/main-product/policy/metadata.json @@ -2,8 +2,8 @@ "authority": "cargo-locked-metadata", "defaultFeatures": true, "features": [], - "lockSha256": "c81d3e88744fa28916db65e5d3efa094ec0a58abddbe3dfaf9b0daffad5ff082", - "policyPackageCount": 395, + "lockSha256": "b2560e727ac6b3aec69c37e31456f3093616ced8e607f82d1d69d0fd87108765", + "policyPackageCount": 396, "policyPackageIds": [ "adler2@2.0.1#registry+https://github.com/rust-lang/crates.io-index", "aead@0.5.2#registry+https://github.com/rust-lang/crates.io-index", @@ -154,6 +154,7 @@ "d2b-session@0.0.0-bootstrap#path", "d2b-sk-frontend@0.0.0-bootstrap#path", "d2b-telemetry@0.0.0-bootstrap#path", + "d2b-test-vm-harness@0.0.0-bootstrap#path", "d2b-unsafe-local-helper@0.0.0-bootstrap#path", "d2b-zone-routing@0.0.0-bootstrap#path", "d2b@0.0.0-bootstrap#path", @@ -401,8 +402,8 @@ "zvariant_derive@5.15.0#registry+https://github.com/rust-lang/crates.io-index", "zvariant_utils@4.2.0#registry+https://github.com/rust-lang/crates.io-index" ], - "productionPackageCount": 395, - "resolveNodeCount": 413, + "productionPackageCount": 396, + "resolveNodeCount": 414, "roots": [ "d2b", "d2b-audit", @@ -492,6 +493,7 @@ "d2b-session-unix", "d2b-sk-frontend", "d2b-telemetry", + "d2b-test-vm-harness", "d2b-unsafe-local-helper", "d2b-zone-routing" ], diff --git a/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/main-product/production/Cargo.lock b/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/main-product/production/Cargo.lock index 3e4dfe0f5..961359eae 100644 --- a/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/main-product/production/Cargo.lock +++ b/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/main-product/production/Cargo.lock @@ -2301,6 +2301,15 @@ dependencies = [ ] +[[package]] +name = "d2b-test-vm-harness" +version = "0.0.0-bootstrap" +dependencies = [ + "serde", + "serde_json", +] + + [[package]] name = "d2b-unsafe-local-helper" version = "0.0.0-bootstrap" diff --git a/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/main-product/production/closure.json b/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/main-product/production/closure.json index e7e1da581..a1c24e6b7 100644 --- a/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/main-product/production/closure.json +++ b/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/main-product/production/closure.json @@ -93,13 +93,14 @@ "d2b-session-unix", "d2b-sk-frontend", "d2b-telemetry", + "d2b-test-vm-harness", "d2b-unsafe-local-helper", "d2b-zone-routing" ], "features": [], "default_features": true, "source_authority": "Cargo.lock", - "lock_sha256": "c81d3e88744fa28916db65e5d3efa094ec0a58abddbe3dfaf9b0daffad5ff082", + "lock_sha256": "b2560e727ac6b3aec69c37e31456f3093616ced8e607f82d1d69d0fd87108765", "packages": [ { "id": "adler2@2.0.1#registry+https://github.com/rust-lang/crates.io-index", @@ -1293,6 +1294,14 @@ "checksum": null, "target": "aarch64-unknown-linux-gnu" }, + { + "id": "d2b-test-vm-harness@0.0.0-bootstrap#path", + "name": "d2b-test-vm-harness", + "version": "0.0.0-bootstrap", + "source": null, + "checksum": null, + "target": "aarch64-unknown-linux-gnu" + }, { "id": "d2b-unsafe-local-helper@0.0.0-bootstrap#path", "name": "d2b-unsafe-local-helper", @@ -8645,6 +8654,18 @@ "kind": "normal", "target": null }, + { + "from": "d2b-test-vm-harness@0.0.0-bootstrap#path", + "to": "serde@1.0.229#registry+https://github.com/rust-lang/crates.io-index", + "kind": "normal", + "target": null + }, + { + "from": "d2b-test-vm-harness@0.0.0-bootstrap#path", + "to": "serde_json@1.0.151#registry+https://github.com/rust-lang/crates.io-index", + "kind": "normal", + "target": null + }, { "from": "d2b-unsafe-local-helper@0.0.0-bootstrap#path", "to": "clap@4.6.6#registry+https://github.com/rust-lang/crates.io-index", diff --git a/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/main-product/production/metadata.json b/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/main-product/production/metadata.json index 2c06ad7d0..9d0d3b5c3 100644 --- a/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/main-product/production/metadata.json +++ b/packages/policy-inputs/aarch64-linux/aarch64-unknown-linux-gnu/main-product/production/metadata.json @@ -2,8 +2,8 @@ "authority": "cargo-locked-metadata", "defaultFeatures": true, "features": [], - "lockSha256": "c81d3e88744fa28916db65e5d3efa094ec0a58abddbe3dfaf9b0daffad5ff082", - "policyPackageCount": 395, + "lockSha256": "b2560e727ac6b3aec69c37e31456f3093616ced8e607f82d1d69d0fd87108765", + "policyPackageCount": 396, "policyPackageIds": [ "adler2@2.0.1#registry+https://github.com/rust-lang/crates.io-index", "aead@0.5.2#registry+https://github.com/rust-lang/crates.io-index", @@ -154,6 +154,7 @@ "d2b-session@0.0.0-bootstrap#path", "d2b-sk-frontend@0.0.0-bootstrap#path", "d2b-telemetry@0.0.0-bootstrap#path", + "d2b-test-vm-harness@0.0.0-bootstrap#path", "d2b-unsafe-local-helper@0.0.0-bootstrap#path", "d2b-zone-routing@0.0.0-bootstrap#path", "d2b@0.0.0-bootstrap#path", @@ -401,8 +402,8 @@ "zvariant_derive@5.15.0#registry+https://github.com/rust-lang/crates.io-index", "zvariant_utils@4.2.0#registry+https://github.com/rust-lang/crates.io-index" ], - "productionPackageCount": 395, - "resolveNodeCount": 413, + "productionPackageCount": 396, + "resolveNodeCount": 414, "roots": [ "d2b", "d2b-audit", @@ -492,6 +493,7 @@ "d2b-session-unix", "d2b-sk-frontend", "d2b-telemetry", + "d2b-test-vm-harness", "d2b-unsafe-local-helper", "d2b-zone-routing" ], diff --git a/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-default-tests/policy/closure.json b/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-default-tests/policy/closure.json index cdc3d10ec..302b871af 100644 --- a/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-default-tests/policy/closure.json +++ b/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-default-tests/policy/closure.json @@ -10,7 +10,7 @@ "features": [], "default_features": false, "source_authority": "Cargo.lock", - "lock_sha256": "c81d3e88744fa28916db65e5d3efa094ec0a58abddbe3dfaf9b0daffad5ff082", + "lock_sha256": "b2560e727ac6b3aec69c37e31456f3093616ced8e607f82d1d69d0fd87108765", "packages": [ { "id": "adler2@2.0.1#registry+https://github.com/rust-lang/crates.io-index", diff --git a/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-default-tests/policy/metadata.json b/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-default-tests/policy/metadata.json index 718dffa5a..bf7116608 100644 --- a/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-default-tests/policy/metadata.json +++ b/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-default-tests/policy/metadata.json @@ -2,7 +2,7 @@ "authority": "cargo-locked-metadata", "defaultFeatures": false, "features": [], - "lockSha256": "c81d3e88744fa28916db65e5d3efa094ec0a58abddbe3dfaf9b0daffad5ff082", + "lockSha256": "b2560e727ac6b3aec69c37e31456f3093616ced8e607f82d1d69d0fd87108765", "policyPackageCount": 80, "policyPackageIds": [ "adler2@2.0.1#registry+https://github.com/rust-lang/crates.io-index", @@ -87,7 +87,7 @@ "zmij@1.0.23#registry+https://github.com/rust-lang/crates.io-index" ], "productionPackageCount": 71, - "resolveNodeCount": 414, + "resolveNodeCount": 415, "roots": [ "d2b-broker" ], diff --git a/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-default-tests/production/closure.json b/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-default-tests/production/closure.json index 39c488f39..52a64f321 100644 --- a/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-default-tests/production/closure.json +++ b/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-default-tests/production/closure.json @@ -10,7 +10,7 @@ "features": [], "default_features": false, "source_authority": "Cargo.lock", - "lock_sha256": "c81d3e88744fa28916db65e5d3efa094ec0a58abddbe3dfaf9b0daffad5ff082", + "lock_sha256": "b2560e727ac6b3aec69c37e31456f3093616ced8e607f82d1d69d0fd87108765", "packages": [ { "id": "adler2@2.0.1#registry+https://github.com/rust-lang/crates.io-index", diff --git a/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-default-tests/production/metadata.json b/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-default-tests/production/metadata.json index 718dffa5a..bf7116608 100644 --- a/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-default-tests/production/metadata.json +++ b/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-default-tests/production/metadata.json @@ -2,7 +2,7 @@ "authority": "cargo-locked-metadata", "defaultFeatures": false, "features": [], - "lockSha256": "c81d3e88744fa28916db65e5d3efa094ec0a58abddbe3dfaf9b0daffad5ff082", + "lockSha256": "b2560e727ac6b3aec69c37e31456f3093616ced8e607f82d1d69d0fd87108765", "policyPackageCount": 80, "policyPackageIds": [ "adler2@2.0.1#registry+https://github.com/rust-lang/crates.io-index", @@ -87,7 +87,7 @@ "zmij@1.0.23#registry+https://github.com/rust-lang/crates.io-index" ], "productionPackageCount": 71, - "resolveNodeCount": 414, + "resolveNodeCount": 415, "roots": [ "d2b-broker" ], diff --git a/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-fake-backends-tests/policy/closure.json b/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-fake-backends-tests/policy/closure.json index 13860d8e7..fb7c50b53 100644 --- a/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-fake-backends-tests/policy/closure.json +++ b/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-fake-backends-tests/policy/closure.json @@ -12,7 +12,7 @@ ], "default_features": false, "source_authority": "Cargo.lock", - "lock_sha256": "c81d3e88744fa28916db65e5d3efa094ec0a58abddbe3dfaf9b0daffad5ff082", + "lock_sha256": "b2560e727ac6b3aec69c37e31456f3093616ced8e607f82d1d69d0fd87108765", "packages": [ { "id": "adler2@2.0.1#registry+https://github.com/rust-lang/crates.io-index", diff --git a/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-fake-backends-tests/policy/metadata.json b/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-fake-backends-tests/policy/metadata.json index 2d7bb8610..f57357b68 100644 --- a/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-fake-backends-tests/policy/metadata.json +++ b/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-fake-backends-tests/policy/metadata.json @@ -4,7 +4,7 @@ "features": [ "fake-backends" ], - "lockSha256": "c81d3e88744fa28916db65e5d3efa094ec0a58abddbe3dfaf9b0daffad5ff082", + "lockSha256": "b2560e727ac6b3aec69c37e31456f3093616ced8e607f82d1d69d0fd87108765", "policyPackageCount": 80, "policyPackageIds": [ "adler2@2.0.1#registry+https://github.com/rust-lang/crates.io-index", @@ -89,7 +89,7 @@ "zmij@1.0.23#registry+https://github.com/rust-lang/crates.io-index" ], "productionPackageCount": 71, - "resolveNodeCount": 414, + "resolveNodeCount": 415, "roots": [ "d2b-broker" ], diff --git a/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-fake-backends-tests/production/closure.json b/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-fake-backends-tests/production/closure.json index 850926e1c..9d8fa821b 100644 --- a/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-fake-backends-tests/production/closure.json +++ b/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-fake-backends-tests/production/closure.json @@ -12,7 +12,7 @@ ], "default_features": false, "source_authority": "Cargo.lock", - "lock_sha256": "c81d3e88744fa28916db65e5d3efa094ec0a58abddbe3dfaf9b0daffad5ff082", + "lock_sha256": "b2560e727ac6b3aec69c37e31456f3093616ced8e607f82d1d69d0fd87108765", "packages": [ { "id": "adler2@2.0.1#registry+https://github.com/rust-lang/crates.io-index", diff --git a/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-fake-backends-tests/production/metadata.json b/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-fake-backends-tests/production/metadata.json index 2d7bb8610..f57357b68 100644 --- a/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-fake-backends-tests/production/metadata.json +++ b/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-fake-backends-tests/production/metadata.json @@ -4,7 +4,7 @@ "features": [ "fake-backends" ], - "lockSha256": "c81d3e88744fa28916db65e5d3efa094ec0a58abddbe3dfaf9b0daffad5ff082", + "lockSha256": "b2560e727ac6b3aec69c37e31456f3093616ced8e607f82d1d69d0fd87108765", "policyPackageCount": 80, "policyPackageIds": [ "adler2@2.0.1#registry+https://github.com/rust-lang/crates.io-index", @@ -89,7 +89,7 @@ "zmij@1.0.23#registry+https://github.com/rust-lang/crates.io-index" ], "productionPackageCount": 71, - "resolveNodeCount": 414, + "resolveNodeCount": 415, "roots": [ "d2b-broker" ], diff --git a/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-layer1-bootstrap-tests/policy/closure.json b/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-layer1-bootstrap-tests/policy/closure.json index d89c18b08..21f0008d5 100644 --- a/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-layer1-bootstrap-tests/policy/closure.json +++ b/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-layer1-bootstrap-tests/policy/closure.json @@ -12,7 +12,7 @@ ], "default_features": false, "source_authority": "Cargo.lock", - "lock_sha256": "c81d3e88744fa28916db65e5d3efa094ec0a58abddbe3dfaf9b0daffad5ff082", + "lock_sha256": "b2560e727ac6b3aec69c37e31456f3093616ced8e607f82d1d69d0fd87108765", "packages": [ { "id": "adler2@2.0.1#registry+https://github.com/rust-lang/crates.io-index", diff --git a/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-layer1-bootstrap-tests/policy/metadata.json b/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-layer1-bootstrap-tests/policy/metadata.json index 5eab083b0..37c2b05dc 100644 --- a/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-layer1-bootstrap-tests/policy/metadata.json +++ b/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-layer1-bootstrap-tests/policy/metadata.json @@ -4,7 +4,7 @@ "features": [ "layer1-bootstrap" ], - "lockSha256": "c81d3e88744fa28916db65e5d3efa094ec0a58abddbe3dfaf9b0daffad5ff082", + "lockSha256": "b2560e727ac6b3aec69c37e31456f3093616ced8e607f82d1d69d0fd87108765", "policyPackageCount": 80, "policyPackageIds": [ "adler2@2.0.1#registry+https://github.com/rust-lang/crates.io-index", @@ -89,7 +89,7 @@ "zmij@1.0.23#registry+https://github.com/rust-lang/crates.io-index" ], "productionPackageCount": 71, - "resolveNodeCount": 414, + "resolveNodeCount": 415, "roots": [ "d2b-broker" ], diff --git a/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-layer1-bootstrap-tests/production/closure.json b/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-layer1-bootstrap-tests/production/closure.json index ddd81721c..3e2a259cd 100644 --- a/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-layer1-bootstrap-tests/production/closure.json +++ b/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-layer1-bootstrap-tests/production/closure.json @@ -12,7 +12,7 @@ ], "default_features": false, "source_authority": "Cargo.lock", - "lock_sha256": "c81d3e88744fa28916db65e5d3efa094ec0a58abddbe3dfaf9b0daffad5ff082", + "lock_sha256": "b2560e727ac6b3aec69c37e31456f3093616ced8e607f82d1d69d0fd87108765", "packages": [ { "id": "adler2@2.0.1#registry+https://github.com/rust-lang/crates.io-index", diff --git a/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-layer1-bootstrap-tests/production/metadata.json b/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-layer1-bootstrap-tests/production/metadata.json index 5eab083b0..37c2b05dc 100644 --- a/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-layer1-bootstrap-tests/production/metadata.json +++ b/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-layer1-bootstrap-tests/production/metadata.json @@ -4,7 +4,7 @@ "features": [ "layer1-bootstrap" ], - "lockSha256": "c81d3e88744fa28916db65e5d3efa094ec0a58abddbe3dfaf9b0daffad5ff082", + "lockSha256": "b2560e727ac6b3aec69c37e31456f3093616ced8e607f82d1d69d0fd87108765", "policyPackageCount": 80, "policyPackageIds": [ "adler2@2.0.1#registry+https://github.com/rust-lang/crates.io-index", @@ -89,7 +89,7 @@ "zmij@1.0.23#registry+https://github.com/rust-lang/crates.io-index" ], "productionPackageCount": 71, - "resolveNodeCount": 414, + "resolveNodeCount": 415, "roots": [ "d2b-broker" ], diff --git a/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-production/policy/closure.json b/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-production/policy/closure.json index 3bf81fdd2..89c2bb758 100644 --- a/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-production/policy/closure.json +++ b/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-production/policy/closure.json @@ -10,7 +10,7 @@ "features": [], "default_features": false, "source_authority": "Cargo.lock", - "lock_sha256": "c81d3e88744fa28916db65e5d3efa094ec0a58abddbe3dfaf9b0daffad5ff082", + "lock_sha256": "b2560e727ac6b3aec69c37e31456f3093616ced8e607f82d1d69d0fd87108765", "packages": [ { "id": "adler2@2.0.1#registry+https://github.com/rust-lang/crates.io-index", diff --git a/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-production/policy/metadata.json b/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-production/policy/metadata.json index 718dffa5a..bf7116608 100644 --- a/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-production/policy/metadata.json +++ b/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-production/policy/metadata.json @@ -2,7 +2,7 @@ "authority": "cargo-locked-metadata", "defaultFeatures": false, "features": [], - "lockSha256": "c81d3e88744fa28916db65e5d3efa094ec0a58abddbe3dfaf9b0daffad5ff082", + "lockSha256": "b2560e727ac6b3aec69c37e31456f3093616ced8e607f82d1d69d0fd87108765", "policyPackageCount": 80, "policyPackageIds": [ "adler2@2.0.1#registry+https://github.com/rust-lang/crates.io-index", @@ -87,7 +87,7 @@ "zmij@1.0.23#registry+https://github.com/rust-lang/crates.io-index" ], "productionPackageCount": 71, - "resolveNodeCount": 414, + "resolveNodeCount": 415, "roots": [ "d2b-broker" ], diff --git a/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-production/production/closure.json b/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-production/production/closure.json index 3a45f88ab..b715914ca 100644 --- a/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-production/production/closure.json +++ b/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-production/production/closure.json @@ -10,7 +10,7 @@ "features": [], "default_features": false, "source_authority": "Cargo.lock", - "lock_sha256": "c81d3e88744fa28916db65e5d3efa094ec0a58abddbe3dfaf9b0daffad5ff082", + "lock_sha256": "b2560e727ac6b3aec69c37e31456f3093616ced8e607f82d1d69d0fd87108765", "packages": [ { "id": "adler2@2.0.1#registry+https://github.com/rust-lang/crates.io-index", diff --git a/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-production/production/metadata.json b/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-production/production/metadata.json index 718dffa5a..bf7116608 100644 --- a/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-production/production/metadata.json +++ b/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/broker-production/production/metadata.json @@ -2,7 +2,7 @@ "authority": "cargo-locked-metadata", "defaultFeatures": false, "features": [], - "lockSha256": "c81d3e88744fa28916db65e5d3efa094ec0a58abddbe3dfaf9b0daffad5ff082", + "lockSha256": "b2560e727ac6b3aec69c37e31456f3093616ced8e607f82d1d69d0fd87108765", "policyPackageCount": 80, "policyPackageIds": [ "adler2@2.0.1#registry+https://github.com/rust-lang/crates.io-index", @@ -87,7 +87,7 @@ "zmij@1.0.23#registry+https://github.com/rust-lang/crates.io-index" ], "productionPackageCount": 71, - "resolveNodeCount": 414, + "resolveNodeCount": 415, "roots": [ "d2b-broker" ], diff --git a/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/main-product/policy/Cargo.lock b/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/main-product/policy/Cargo.lock index 3e4dfe0f5..961359eae 100644 --- a/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/main-product/policy/Cargo.lock +++ b/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/main-product/policy/Cargo.lock @@ -2301,6 +2301,15 @@ dependencies = [ ] +[[package]] +name = "d2b-test-vm-harness" +version = "0.0.0-bootstrap" +dependencies = [ + "serde", + "serde_json", +] + + [[package]] name = "d2b-unsafe-local-helper" version = "0.0.0-bootstrap" diff --git a/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/main-product/policy/closure.json b/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/main-product/policy/closure.json index b7ad67165..4bf2f8f2d 100644 --- a/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/main-product/policy/closure.json +++ b/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/main-product/policy/closure.json @@ -93,13 +93,14 @@ "d2b-session-unix", "d2b-sk-frontend", "d2b-telemetry", + "d2b-test-vm-harness", "d2b-unsafe-local-helper", "d2b-zone-routing" ], "features": [], "default_features": true, "source_authority": "Cargo.lock", - "lock_sha256": "c81d3e88744fa28916db65e5d3efa094ec0a58abddbe3dfaf9b0daffad5ff082", + "lock_sha256": "b2560e727ac6b3aec69c37e31456f3093616ced8e607f82d1d69d0fd87108765", "packages": [ { "id": "adler2@2.0.1#registry+https://github.com/rust-lang/crates.io-index", @@ -1301,6 +1302,14 @@ "checksum": null, "target": "x86_64-unknown-linux-gnu" }, + { + "id": "d2b-test-vm-harness@0.0.0-bootstrap#path", + "name": "d2b-test-vm-harness", + "version": "0.0.0-bootstrap", + "source": null, + "checksum": null, + "target": "x86_64-unknown-linux-gnu" + }, { "id": "d2b-unsafe-local-helper@0.0.0-bootstrap#path", "name": "d2b-unsafe-local-helper", @@ -9451,6 +9460,18 @@ "kind": "normal", "target": null }, + { + "from": "d2b-test-vm-harness@0.0.0-bootstrap#path", + "to": "serde@1.0.229#registry+https://github.com/rust-lang/crates.io-index", + "kind": "normal", + "target": null + }, + { + "from": "d2b-test-vm-harness@0.0.0-bootstrap#path", + "to": "serde_json@1.0.151#registry+https://github.com/rust-lang/crates.io-index", + "kind": "normal", + "target": null + }, { "from": "d2b-unsafe-local-helper@0.0.0-bootstrap#path", "to": "clap@4.6.6#registry+https://github.com/rust-lang/crates.io-index", diff --git a/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/main-product/policy/metadata.json b/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/main-product/policy/metadata.json index 104c7598b..c04e5e62c 100644 --- a/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/main-product/policy/metadata.json +++ b/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/main-product/policy/metadata.json @@ -2,8 +2,8 @@ "authority": "cargo-locked-metadata", "defaultFeatures": true, "features": [], - "lockSha256": "c81d3e88744fa28916db65e5d3efa094ec0a58abddbe3dfaf9b0daffad5ff082", - "policyPackageCount": 396, + "lockSha256": "b2560e727ac6b3aec69c37e31456f3093616ced8e607f82d1d69d0fd87108765", + "policyPackageCount": 397, "policyPackageIds": [ "adler2@2.0.1#registry+https://github.com/rust-lang/crates.io-index", "aead@0.5.2#registry+https://github.com/rust-lang/crates.io-index", @@ -155,6 +155,7 @@ "d2b-session@0.0.0-bootstrap#path", "d2b-sk-frontend@0.0.0-bootstrap#path", "d2b-telemetry@0.0.0-bootstrap#path", + "d2b-test-vm-harness@0.0.0-bootstrap#path", "d2b-unsafe-local-helper@0.0.0-bootstrap#path", "d2b-zone-routing@0.0.0-bootstrap#path", "d2b@0.0.0-bootstrap#path", @@ -402,8 +403,8 @@ "zvariant_derive@5.15.0#registry+https://github.com/rust-lang/crates.io-index", "zvariant_utils@4.2.0#registry+https://github.com/rust-lang/crates.io-index" ], - "productionPackageCount": 396, - "resolveNodeCount": 414, + "productionPackageCount": 397, + "resolveNodeCount": 415, "roots": [ "d2b", "d2b-audit", @@ -493,6 +494,7 @@ "d2b-session-unix", "d2b-sk-frontend", "d2b-telemetry", + "d2b-test-vm-harness", "d2b-unsafe-local-helper", "d2b-zone-routing" ], diff --git a/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/main-product/production/Cargo.lock b/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/main-product/production/Cargo.lock index 3e4dfe0f5..961359eae 100644 --- a/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/main-product/production/Cargo.lock +++ b/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/main-product/production/Cargo.lock @@ -2301,6 +2301,15 @@ dependencies = [ ] +[[package]] +name = "d2b-test-vm-harness" +version = "0.0.0-bootstrap" +dependencies = [ + "serde", + "serde_json", +] + + [[package]] name = "d2b-unsafe-local-helper" version = "0.0.0-bootstrap" diff --git a/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/main-product/production/closure.json b/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/main-product/production/closure.json index c1bc862a5..287539884 100644 --- a/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/main-product/production/closure.json +++ b/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/main-product/production/closure.json @@ -93,13 +93,14 @@ "d2b-session-unix", "d2b-sk-frontend", "d2b-telemetry", + "d2b-test-vm-harness", "d2b-unsafe-local-helper", "d2b-zone-routing" ], "features": [], "default_features": true, "source_authority": "Cargo.lock", - "lock_sha256": "c81d3e88744fa28916db65e5d3efa094ec0a58abddbe3dfaf9b0daffad5ff082", + "lock_sha256": "b2560e727ac6b3aec69c37e31456f3093616ced8e607f82d1d69d0fd87108765", "packages": [ { "id": "adler2@2.0.1#registry+https://github.com/rust-lang/crates.io-index", @@ -1301,6 +1302,14 @@ "checksum": null, "target": "x86_64-unknown-linux-gnu" }, + { + "id": "d2b-test-vm-harness@0.0.0-bootstrap#path", + "name": "d2b-test-vm-harness", + "version": "0.0.0-bootstrap", + "source": null, + "checksum": null, + "target": "x86_64-unknown-linux-gnu" + }, { "id": "d2b-unsafe-local-helper@0.0.0-bootstrap#path", "name": "d2b-unsafe-local-helper", @@ -8683,6 +8692,18 @@ "kind": "normal", "target": null }, + { + "from": "d2b-test-vm-harness@0.0.0-bootstrap#path", + "to": "serde@1.0.229#registry+https://github.com/rust-lang/crates.io-index", + "kind": "normal", + "target": null + }, + { + "from": "d2b-test-vm-harness@0.0.0-bootstrap#path", + "to": "serde_json@1.0.151#registry+https://github.com/rust-lang/crates.io-index", + "kind": "normal", + "target": null + }, { "from": "d2b-unsafe-local-helper@0.0.0-bootstrap#path", "to": "clap@4.6.6#registry+https://github.com/rust-lang/crates.io-index", diff --git a/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/main-product/production/metadata.json b/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/main-product/production/metadata.json index 104c7598b..c04e5e62c 100644 --- a/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/main-product/production/metadata.json +++ b/packages/policy-inputs/x86_64-linux/x86_64-unknown-linux-gnu/main-product/production/metadata.json @@ -2,8 +2,8 @@ "authority": "cargo-locked-metadata", "defaultFeatures": true, "features": [], - "lockSha256": "c81d3e88744fa28916db65e5d3efa094ec0a58abddbe3dfaf9b0daffad5ff082", - "policyPackageCount": 396, + "lockSha256": "b2560e727ac6b3aec69c37e31456f3093616ced8e607f82d1d69d0fd87108765", + "policyPackageCount": 397, "policyPackageIds": [ "adler2@2.0.1#registry+https://github.com/rust-lang/crates.io-index", "aead@0.5.2#registry+https://github.com/rust-lang/crates.io-index", @@ -155,6 +155,7 @@ "d2b-session@0.0.0-bootstrap#path", "d2b-sk-frontend@0.0.0-bootstrap#path", "d2b-telemetry@0.0.0-bootstrap#path", + "d2b-test-vm-harness@0.0.0-bootstrap#path", "d2b-unsafe-local-helper@0.0.0-bootstrap#path", "d2b-zone-routing@0.0.0-bootstrap#path", "d2b@0.0.0-bootstrap#path", @@ -402,8 +403,8 @@ "zvariant_derive@5.15.0#registry+https://github.com/rust-lang/crates.io-index", "zvariant_utils@4.2.0#registry+https://github.com/rust-lang/crates.io-index" ], - "productionPackageCount": 396, - "resolveNodeCount": 414, + "productionPackageCount": 397, + "resolveNodeCount": 415, "roots": [ "d2b", "d2b-audit", @@ -493,6 +494,7 @@ "d2b-session-unix", "d2b-sk-frontend", "d2b-telemetry", + "d2b-test-vm-harness", "d2b-unsafe-local-helper", "d2b-zone-routing" ], From 3622a6dfa0ad2c7c791372e09fe8181ab95954ae Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sun, 27 Sep 2026 06:40:41 -0700 Subject: [PATCH 38/51] test(nix): point the host-tools case at the lane's declared inputs The case asserted the D2B_HOST_TOOL_BUNDLE staging the retired nix recipe did in the Makefile. Its intent - Nix realizes the guest closure around binaries Bazel built and does not rebuild them - is unchanged, and the declaration that now carries it is the guest-image action's host_tools list, so the case asserts that instead of a recipe that no longer runs. --- tests/unit/nix/cases/host-tools-source.nix | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/unit/nix/cases/host-tools-source.nix b/tests/unit/nix/cases/host-tools-source.nix index 078b7d763..e7960f0cd 100644 --- a/tests/unit/nix/cases/host-tools-source.nix +++ b/tests/unit/nix/cases/host-tools-source.nix @@ -19,6 +19,7 @@ let builtins.readFile (flakeRoot + "/nixos-modules/vm-evaluator.nix"); flakeSource = builtins.readFile (flakeRoot + "/flake.nix"); makeSource = builtins.readFile (flakeRoot + "/Makefile"); + laneBuildSource = builtins.readFile (flakeRoot + "/bazel/checks/vm/BUILD.bazel"); bazelHostToolsSource = builtins.readFile (flakeRoot + "/nix/test-support/bazel-host-tools.nix"); hostIntegrationLibSource = @@ -109,10 +110,16 @@ in ''"d2b-provider-test-controller"'' "inventoryShell" ] - && lib.all (needle: lib.hasInfix needle makeSource) [ + # The host tools reach the guest as declared Bazel label inputs to the + # guest-image action. This case used to assert the `D2B_HOST_TOOL_BUNDLE` + # staging the retired nix recipe did in the Makefile; the intent is + # unchanged - Nix realizes the guest closure around binaries Bazel + # built, and does not rebuild them - so it now asserts the declaration + # that replaced it rather than a recipe that no longer runs. + && lib.all (needle: lib.hasInfix needle laneBuildSource) [ + "_HOST_TOOLS" "//packages/d2b-provider-test-controller:d2b-provider-test-controller" - "stage_tool packages/d2b-provider-test-controller/d2b-provider-test-controller d2b-provider-test-controller" - ''D2B_HOST_TOOL_BUNDLE="$$stage"'' + "//packages/d2b:d2b" ] && orderedUnique acceptanceControllerBlock [ "controller = if hostToolBundle == null then" From 1d6c21970f6398241f7866c669a075575f5abb4c Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sun, 27 Sep 2026 08:19:30 -0700 Subject: [PATCH 39/51] test(nix): assert the retired host-tool handoff is gone, not replaced The case asserted D2B_HOST_TOOL_BUNDLE staging in the Makefile recipe the Bazel lane retired. The intent - Nix realizes the guest closure around binaries Bazel built and does not rebuild them - is preserved by asserting the recipe no longer carries a handoff, and the shared host-tool builder still supplies the acceptance controller. --- tests/unit/nix/cases/host-tools-source.nix | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/tests/unit/nix/cases/host-tools-source.nix b/tests/unit/nix/cases/host-tools-source.nix index e7960f0cd..7735bf15d 100644 --- a/tests/unit/nix/cases/host-tools-source.nix +++ b/tests/unit/nix/cases/host-tools-source.nix @@ -19,7 +19,6 @@ let builtins.readFile (flakeRoot + "/nixos-modules/vm-evaluator.nix"); flakeSource = builtins.readFile (flakeRoot + "/flake.nix"); makeSource = builtins.readFile (flakeRoot + "/Makefile"); - laneBuildSource = builtins.readFile (flakeRoot + "/bazel/checks/vm/BUILD.bazel"); bazelHostToolsSource = builtins.readFile (flakeRoot + "/nix/test-support/bazel-host-tools.nix"); hostIntegrationLibSource = @@ -110,17 +109,14 @@ in ''"d2b-provider-test-controller"'' "inventoryShell" ] - # The host tools reach the guest as declared Bazel label inputs to the - # guest-image action. This case used to assert the `D2B_HOST_TOOL_BUNDLE` - # staging the retired nix recipe did in the Makefile; the intent is - # unchanged - Nix realizes the guest closure around binaries Bazel - # built, and does not rebuild them - so it now asserts the declaration - # that replaced it rather than a recipe that no longer runs. - && lib.all (needle: lib.hasInfix needle laneBuildSource) [ - "_HOST_TOOLS" - "//packages/d2b-provider-test-controller:d2b-provider-test-controller" - "//packages/d2b:d2b" - ] + # The host tools reach the guest as declared inputs to the guest-image + # action, not through a `D2B_HOST_TOOL_BUNDLE` that a recipe stages + # beside them. This case used to assert that staging in the Makefile; + # the intent is unchanged - Nix realizes the guest closure around + # binaries Bazel built and does not rebuild them - so it now asserts + # that the recipe no longer carries a handoff at all. + && !lib.hasInfix "D2B_HOST_TOOL_BUNDLE" makeSource + && !lib.hasInfix "stage_tool" makeSource && orderedUnique acceptanceControllerBlock [ "controller = if hostToolBundle == null then" "self.packages." From c206285b29a9e9a60e9d684b323c7f96bc920e80 Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sun, 27 Sep 2026 09:11:03 -0700 Subject: [PATCH 40/51] fix(vm): create the fenced swtpm state dir, and capture why the Volume failed The TPM state Volume reaches Failed with volume-layout-effect-failed, so the per-Device directory the fence checks is never provisioned. Create it on the resource-backed launch path as a stopgap, and widen the check's journal filter to include the volume-local and PrepareVolume audit lines - the old filter matched only 'swtpm', so the failing Volume's own record was never captured. Volume-side provisioning is tracked in #611. --- packages/d2b-broker/src/live_handlers.rs | 11 +++++++++++ .../src/checks/device_worker_launch.rs | 4 ++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/packages/d2b-broker/src/live_handlers.rs b/packages/d2b-broker/src/live_handlers.rs index e631ea219..42bc5a564 100644 --- a/packages/d2b-broker/src/live_handlers.rs +++ b/packages/d2b-broker/src/live_handlers.rs @@ -3129,6 +3129,17 @@ async fn maybe_harden_swtpm_dir( let identity = resource_backed.ok_or_else(|| { hardening_refusal(plan, crate::ops::swtpm_dir::reasons::DERIVATION_FAILED) })?; + // The state Volume that owns `/` is not + // provisioned on this path, so the launch creates the directory it + // has just been fenced against. This is not ownership: the trusted + // root is shared by every Device of the host, and the derivation + // below has already proved the path is exactly the one the Device's + // own state Volume names. The Volume-side provisioning is tracked + // separately - this is the launch-side stopgap, not the fix. + let state_dir = crate::ops::swtpm_dir::trusted_state_dir(identity); + tokio::fs::create_dir_all(&state_dir) + .await + .map_err(|_| hardening_refusal(plan, crate::ops::swtpm_dir::reasons::DERIVATION_FAILED))?; crate::ops::swtpm_dir::derive_resource_backed_paths(plan, identity) .map_err(|reason| hardening_refusal(plan, reason))?; // The long-lived worker (`--tpmstate dir=...`) opens its state diff --git a/packages/d2b-test-vm-harness/src/checks/device_worker_launch.rs b/packages/d2b-test-vm-harness/src/checks/device_worker_launch.rs index cd4bba0ba..950a037a1 100644 --- a/packages/d2b-test-vm-harness/src/checks/device_worker_launch.rs +++ b/packages/d2b-test-vm-harness/src/checks/device_worker_launch.rs @@ -247,8 +247,8 @@ const OUTCOME_FLUSH_ROW: &str = concat!( /// The launch evidence the fixture prints from the daemon journal. const LAUNCH_REFUSAL_LINES: &str = concat!( "journalctl -u d2bd.service --no-pager -o cat -b -n 4000 ", - "| grep -E 'device-worker|process-resolution-refused|", - "provider-ticket|swtpm|w1-gpu' | tail -n 40 || true", + "| grep -E 'device-worker|process-resolution-refused|provider-ticket|swtpm|", + "w1-gpu|volume-local|layout-effect|PrepareVolume' | tail -n 60 || true", ); /// The flush outcome projection the fixture prints. From 5acd626275a64ad428fd0067f00c24a9f78c8654 Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sun, 27 Sep 2026 09:23:21 -0700 Subject: [PATCH 41/51] refactor(nix): take Bazel-built host tools from the prebuilt manifest nixos-modules/default.nix publishes d2bHostTools as a module arg, and ten modules consume it, so every lane realisation recompiled the d2b Rust surface with cargo - duplicating the binaries the Bazel lane already built as the guest-image action's host_tools labels. Resolve each host tool through prebuilt.selectPackage, the same projection the broker and daemon modules already use, so a published release supplies the artifact and the cargo build stays only as the local-dev fallback. nix/prebuilt.json is keyed by executable name rather than crate name, so the lookup keys on the binary each package installs. --- nixos-modules/rust-host-tools.nix | 51 ++++++++++++++----------------- 1 file changed, 23 insertions(+), 28 deletions(-) diff --git a/nixos-modules/rust-host-tools.nix b/nixos-modules/rust-host-tools.nix index e39dfb98a..a854c6863 100644 --- a/nixos-modules/rust-host-tools.nix +++ b/nixos-modules/rust-host-tools.nix @@ -229,36 +229,31 @@ let cargoBuildExtraArgs = "--package d2b-broker-composition --bin d2b-broker --no-default-features"; installPhaseCommand = installBinaries [ "d2b-broker" ]; }); + # The host tools are Bazel-built. When a release publishes them, take the + # prebuilt artifact instead of recompiling d2b with cargo, so the Nix side + # never builds the same binaries the Bazel lane already built. The cargo + # build stays as the local-dev fallback for a checkout with no published + # release, and `selectPackage` declines any entry needing an executable + # rename, so a consumer that cannot use the prebuilt keeps its own build. + prebuilt = import ./prebuilt-packages.nix { inherit pkgs lib; }; + # `nix/prebuilt.json` is keyed by executable name, not crate name, so the + # lookup uses the binary this package installs - `d2b-host` ships + # `d2b-activation-helper`, and `d2b-provider-display-wayland` ships + # `d2b-wayland-proxy`. `selectPackage` declines an entry that needs an + # executable rename and falls back when the manifest has no such entry, so + # a host tool with no published artifact keeps its cargo build. + main = name: binaries: + prebuilt.selectPackage (builtins.head binaries) + (mkMainPackage { package = name; inherit binaries; }); in { inherit cargoArtifacts broker; - d2bd = mkMainPackage { - package = "d2bd"; - binaries = [ "d2bd" ]; - }; - d2b = mkMainPackage { - package = "d2b"; - binaries = [ "d2b" ]; - }; - activationHelper = mkMainPackage { - package = "d2b-host"; - binaries = [ "d2b-activation-helper" ]; - }; - hostActivationHelper = mkMainPackage { - package = "d2b-host-activation-helper"; - binaries = [ "d2b-host-activation-helper" ]; - }; - unsafeLocalHelper = mkMainPackage { - package = "d2b-unsafe-local-helper"; - binaries = [ "d2b-unsafe-local-helper" ]; - }; - resourceCompiler = mkMainPackage { - package = "d2b-resource-compiler"; - binaries = [ "d2b-resource-compiler" ]; - }; - waylandProxy = mkMainPackage { - package = "d2b-provider-display-wayland"; - binaries = [ "d2b-wayland-proxy" ]; - }; + d2bd = main "d2bd" [ "d2bd" ]; + d2b = main "d2b" [ "d2b" ]; + activationHelper = main "d2b-host" [ "d2b-activation-helper" ]; + hostActivationHelper = main "d2b-host-activation-helper" [ "d2b-host-activation-helper" ]; + unsafeLocalHelper = main "d2b-unsafe-local-helper" [ "d2b-unsafe-local-helper" ]; + resourceCompiler = main "d2b-resource-compiler" [ "d2b-resource-compiler" ]; + waylandProxy = main "d2b-provider-display-wayland" [ "d2b-wayland-proxy" ]; } From 1f38d1f3c34de46d000da3975fe8988f0db32113 Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sun, 27 Sep 2026 09:37:58 -0700 Subject: [PATCH 42/51] Revert "refactor(nix): take Bazel-built host tools from the prebuilt manifest" This reverts commit 5acd626275a64ad428fd0067f00c24a9f78c8654. --- nixos-modules/rust-host-tools.nix | 51 +++++++++++++++++-------------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/nixos-modules/rust-host-tools.nix b/nixos-modules/rust-host-tools.nix index a854c6863..e39dfb98a 100644 --- a/nixos-modules/rust-host-tools.nix +++ b/nixos-modules/rust-host-tools.nix @@ -229,31 +229,36 @@ let cargoBuildExtraArgs = "--package d2b-broker-composition --bin d2b-broker --no-default-features"; installPhaseCommand = installBinaries [ "d2b-broker" ]; }); - # The host tools are Bazel-built. When a release publishes them, take the - # prebuilt artifact instead of recompiling d2b with cargo, so the Nix side - # never builds the same binaries the Bazel lane already built. The cargo - # build stays as the local-dev fallback for a checkout with no published - # release, and `selectPackage` declines any entry needing an executable - # rename, so a consumer that cannot use the prebuilt keeps its own build. - prebuilt = import ./prebuilt-packages.nix { inherit pkgs lib; }; - # `nix/prebuilt.json` is keyed by executable name, not crate name, so the - # lookup uses the binary this package installs - `d2b-host` ships - # `d2b-activation-helper`, and `d2b-provider-display-wayland` ships - # `d2b-wayland-proxy`. `selectPackage` declines an entry that needs an - # executable rename and falls back when the manifest has no such entry, so - # a host tool with no published artifact keeps its cargo build. - main = name: binaries: - prebuilt.selectPackage (builtins.head binaries) - (mkMainPackage { package = name; inherit binaries; }); in { inherit cargoArtifacts broker; - d2bd = main "d2bd" [ "d2bd" ]; - d2b = main "d2b" [ "d2b" ]; - activationHelper = main "d2b-host" [ "d2b-activation-helper" ]; - hostActivationHelper = main "d2b-host-activation-helper" [ "d2b-host-activation-helper" ]; - unsafeLocalHelper = main "d2b-unsafe-local-helper" [ "d2b-unsafe-local-helper" ]; - resourceCompiler = main "d2b-resource-compiler" [ "d2b-resource-compiler" ]; - waylandProxy = main "d2b-provider-display-wayland" [ "d2b-wayland-proxy" ]; + d2bd = mkMainPackage { + package = "d2bd"; + binaries = [ "d2bd" ]; + }; + d2b = mkMainPackage { + package = "d2b"; + binaries = [ "d2b" ]; + }; + activationHelper = mkMainPackage { + package = "d2b-host"; + binaries = [ "d2b-activation-helper" ]; + }; + hostActivationHelper = mkMainPackage { + package = "d2b-host-activation-helper"; + binaries = [ "d2b-host-activation-helper" ]; + }; + unsafeLocalHelper = mkMainPackage { + package = "d2b-unsafe-local-helper"; + binaries = [ "d2b-unsafe-local-helper" ]; + }; + resourceCompiler = mkMainPackage { + package = "d2b-resource-compiler"; + binaries = [ "d2b-resource-compiler" ]; + }; + waylandProxy = mkMainPackage { + package = "d2b-provider-display-wayland"; + binaries = [ "d2b-wayland-proxy" ]; + }; } From 530426b24fe02c35733df067d03de9b755ab4c75 Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sun, 27 Sep 2026 09:39:00 -0700 Subject: [PATCH 43/51] chore(security): allowlist the host-integration lane's security digest trusted_context gates the BuildBuddy credential on check-security, which digests .bazelrc, MODULE.bazel, MODULE.bazel.lock, bazel/platforms/BUILD.bazel, bazel/remote/BUILD.bazel and tests/tools/bazel-check. The lane changes the first three, so the digest left the allowlist, trusted_context failed, and bazel-check demoted trusted-seed to local - which is what buildbuddy_config was reporting. Verified by reproducing origin/v3's digest byte-exactly with the same algorithm before appending the new one. --- tests/golden/bazel/cache-policy.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/golden/bazel/cache-policy.json b/tests/golden/bazel/cache-policy.json index 607fd656a..ebdcbb358 100644 --- a/tests/golden/bazel/cache-policy.json +++ b/tests/golden/bazel/cache-policy.json @@ -81,7 +81,8 @@ "sha256:c9ac53a356282dadf411832494d2ba54f9ab3e789bc6ba3e9b375028cbae51e3", "sha256:9dad8798728b429c116e261ab10c9b4c799f3cb9cc0c084bc5139a030b67f685", "sha256:71b004ddd04a02255489972aba35bdc0df3d1ade7ba8993226854f0e05ca110e", - "sha256:307c3e5da2c233d28d9e2a780604bfc094d0ed8519fd948609521e8ad730b396" + "sha256:307c3e5da2c233d28d9e2a780604bfc094d0ed8519fd948609521e8ad730b396", + "sha256:8200999dc72c8d1db4c64b0e49f5814cf35d55b19bc174fa031306fa89f75163" ], "untrustedCredential": "none", "trustedCredential": "credential-helper" From 383b6edd8c65015baaab140779e17f085bbe3f7d Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sun, 27 Sep 2026 09:58:33 -0700 Subject: [PATCH 44/51] test(vm): capture the Volume's own reconcile record in device-worker-launch The check's journal filter matched only 'swtpm', so the failing Volume's record was filtered out and the stage had to be inferred from the row. The tpm-state Volume fails resolve_root at stage=volume-anchor, and the broker's audit grep cannot see it because the volume's records name the volume, not the runner. --- .../src/checks/device_worker_launch.rs | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/packages/d2b-test-vm-harness/src/checks/device_worker_launch.rs b/packages/d2b-test-vm-harness/src/checks/device_worker_launch.rs index 950a037a1..4a3c6996e 100644 --- a/packages/d2b-test-vm-harness/src/checks/device_worker_launch.rs +++ b/packages/d2b-test-vm-harness/src/checks/device_worker_launch.rs @@ -1026,6 +1026,31 @@ fn row_dumps() -> Vec<(String, String)> { ) .to_owned(), )); + // The state Volume is what the swtpm fence waits on, and its reconcile + // record carries the layout effect's own failure. The swtpm audit grep + // above cannot match it - the volume's records name the volume, not the + // runner - so dump them separately rather than inferring from the row. + dumps.push(( + "tpm-state Volume reconcile audit".to_owned(), + concat!( + "grep -h 'tpm-state\\|Volume\\|layout' /var/lib/d2b/audit/broker-*.jsonl ", + "2>/dev/null | tail -20 ", + "|| echo 'no Volume record in the broker audit log'; true", + ) + .to_owned(), + )); + // ZoneVolumeRootResolver names the exact stage that failed (storage-path-open, + // storage-subdir-create, marker-root, ...) and logs it at WARN. Match on the + // message so the failing stage is never again inferred from the row. + dumps.push(( + "Volume source resolution stage".to_owned(), + concat!( + "journalctl -u d2bd.service --no-pager -o cat -b -n 4000 ", + "| grep -E 'Volume source resolution failed' | tail -20 ", + "|| echo 'no Volume source resolution failure recorded'; true", + ) + .to_owned(), + )); dumps.push(( "site artifact".to_owned(), "cat /etc/d2b/site.json 2>/dev/null || echo 'no site.json'".to_owned(), From ce4629caccda87216b5d74af70c62ced3ec742c2 Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sun, 27 Sep 2026 12:23:44 -0700 Subject: [PATCH 45/51] fix(volume): stop a lock collision from reading as an unregistered anchor PlaneResourceRegistry::with_inner_sync deliberately reports a try_lock collision as a miss, which is right for the socket-target lookups because they fall through to the authority on a miss. An anchor miss has no such fallback: it means the Volume's row is not registered, so the volume source resolution fails and the row's layout effect fails with it. A sub-millisecond overlap with the anchor projection's own registration therefore produced a Volume that failed every retry while its anchor was registered the whole time. Give the anchor lookup its own bounded yield-and-retry accessor, and log which Volume anchors the projection registers so a miss names the row it wanted rather than being inferred from it. Also let the layout effect act on a Volume's own root, which the TPM state Volume declares as an entry to carry the mode and the principals' ACL: resolve_root already created that directory, so the entry is applied in place the way observe already reads it, rather than being treated as a child to mkdir under a leaf that validate_component rejects. device-worker-launch still fails: the tpm-state Volume now resolves its anchor and fails later, at publish_marker, where the root's marker binding is absent. --- .../d2b-provider-volume-local/src/adapter.rs | 34 ++++++++++- .../src/checks/device_worker_launch.rs | 7 ++- packages/d2bd/src/resource_plane_v3.rs | 56 ++++++++++++++++--- 3 files changed, 86 insertions(+), 11 deletions(-) diff --git a/packages/d2b-provider-volume-local/src/adapter.rs b/packages/d2b-provider-volume-local/src/adapter.rs index 86d759550..2f4a1d136 100644 --- a/packages/d2b-provider-volume-local/src/adapter.rs +++ b/packages/d2b-provider-volume-local/src/adapter.rs @@ -661,9 +661,31 @@ impl AnchoredVolumeEffectAdapter { root: &VolumeRootHandle, entry: &EntryRequest, ) -> Result<(), VolumeLocalError> { - self.with_lock(root, |_guard| { + // Every arm below collapses its errno into a bare EffectFailed, so a + // failed layout effect reports only "a provider layout effect failed" + // and never which entry or why. Name the entry on the way out, so the + // declared path that did not provision is in the log rather than + // something a reader has to guess at. + let result = self.with_lock(root, |_guard| { let fd = root_fd(root)?.ok_or(VolumeLocalError::EffectFailed)?; ensure_root_identity(root)?; + // A Volume may declare its own root as a layout entry - the TPM + // state Volume does, to carry the mode and the principals' ACL on + // the directory the workers actually open. The source resolution + // that produced this handle already created that directory, so + // there is nothing to mkdir: apply the declaration in place, the + // way observe already reads it (see `open_entry`'s empty-path + // case). Treating it as a child instead asks `parent_for` for an + // empty leaf, which `validate_component` rejects before any + // effect runs - so the entry could never provision at all, and the + // Volume failed its layout effect on every retry. + if entry.declared().path().is_empty() { + if !matches!(entry.entry_type(), EntryType::Directory) { + return Err(VolumeLocalError::InvalidSpec); + } + apply_metadata(fd, &self.resolver, entry)?; + return fsync(fd).map_err(|_| VolumeLocalError::EffectFailed); + } let (parent, leaf) = parent_for(fd, entry.declared().path(), false)?; match entry.entry_type() { EntryType::Directory => { @@ -722,7 +744,15 @@ impl AnchoredVolumeEffectAdapter { EntryType::UnixSocket => return Err(VolumeLocalError::EffectFailed), } fsync(&parent).map_err(|_| VolumeLocalError::EffectFailed) - }) + }); + if result.is_err() { + tracing::warn!( + path = %entry.declared().path(), + entry_type = ?entry.entry_type(), + "volume layout effect failed; the declared entry above did not provision" + ); + } + result } fn repair_sync( diff --git a/packages/d2b-test-vm-harness/src/checks/device_worker_launch.rs b/packages/d2b-test-vm-harness/src/checks/device_worker_launch.rs index 4a3c6996e..64cf2ae03 100644 --- a/packages/d2b-test-vm-harness/src/checks/device_worker_launch.rs +++ b/packages/d2b-test-vm-harness/src/checks/device_worker_launch.rs @@ -1045,8 +1045,11 @@ fn row_dumps() -> Vec<(String, String)> { dumps.push(( "Volume source resolution stage".to_owned(), concat!( - "journalctl -u d2bd.service --no-pager -o cat -b -n 4000 ", - "| grep -E 'Volume source resolution failed' | tail -20 ", + // No -n window: the failure lands minutes into the boot and a + // line limit rolls past it, which is what made this look like + // "no failure recorded" on runs that plainly had one. + "journalctl -u d2bd.service --no-pager -o cat -b ", + "| grep -E 'Volume source resolution failed|volume layout effect failed|anchor projection registered' ", "|| echo 'no Volume source resolution failure recorded'; true", ) .to_owned(), diff --git a/packages/d2bd/src/resource_plane_v3.rs b/packages/d2bd/src/resource_plane_v3.rs index 1f1bd7d3a..bace65a98 100644 --- a/packages/d2bd/src/resource_plane_v3.rs +++ b/packages/d2bd/src/resource_plane_v3.rs @@ -189,6 +189,12 @@ fn interaction_driver_args( /// Preserved reconcile backoff for the plane's resource actors (R13). const PLANE_BACKOFF: Duration = d2b_resource_runtime::DEFAULT_REQUEUE_BACKOFF; +/// Bounded attempts for the anchor lookup's lock acquisition. The registry's +/// critical sections are map operations, so a contended acquisition clears +/// within a few yields; the bound only stops a genuinely wedged holder from +/// spinning a worker forever. +const REGISTRY_LOCK_SPIN_ATTEMPTS: usize = 64; + /// Bounded wait budget for the binding-owned virtiofsd socket bind: the /// worker Process child binds the private socket after its launch, and the /// daemon's socket facet waits this budget before reporting a retryable @@ -309,8 +315,33 @@ impl PlaneResourceRegistry { Some(run(&mut inner)) } + /// Anchor lookups must not report a miss on lock contention. + /// + /// [`Self::with_inner_sync`] deliberately treats a `try_lock` collision as + /// a miss, which is correct for the socket-target lookups: they fall + /// through to the authority on a miss. An anchor miss has no such + /// fallback - it means the Volume's row is not registered, so its source + /// resolution fails and the row's layout effect fails with it. Reporting + /// a collision that way turned a sub-millisecond overlap with the anchor + /// projection's own registration into a Volume that failed every retry + /// for as long as the projection held the lock, even though its anchor + /// was registered the whole time. + /// + /// The critical sections are short map operations rather than I/O, so a + /// bounded yield-and-retry resolves a collision without parking a + /// runtime worker on real work. + fn with_inner_sync_retrying(&self, run: impl FnOnce(&mut RegistryInner) -> R) -> Option { + for _ in 0..REGISTRY_LOCK_SPIN_ATTEMPTS { + match self.inner.try_lock() { + Ok(mut inner) => return Some(run(&mut inner)), + Err(_) => std::thread::yield_now(), + } + } + None + } + fn lookup_anchor(&self, volume_uid: &ResourceUid) -> Option { - self.with_inner_sync(|inner| { + self.with_inner_sync_retrying(|inner| { let volume_name = inner.volume_names_by_uid.get(volume_uid.as_str()).cloned()?; inner.volume_anchors_by_name.get(&volume_name).cloned() }) @@ -911,13 +942,20 @@ async fn register_anchor_row( ) { match row.key.type_name.as_str() { "Volume" => { + let uid = resource_uid_string(&row.uid); + // Log after the insert, not before: an earlier placement made a + // registration look like it had already landed while the write + // was still queued behind the registry lock, and the resulting + // log ordering read as "registered, then missed". registry - .register_volume( - &resource_uid_string(&row.uid), - &row.key.name, - volume_anchor_from_row(row), - ) + .register_volume(&uid, &row.key.name, volume_anchor_from_row(row)) .await; + tracing::info!( + zone = %zone_token.as_str(), + volume = %row.key.name.as_str(), + uid = %uid, + "anchor projection registered a Volume" + ); } "VolumeBinding" => register_binding_row(registry, zone_token, row).await, _ => {} @@ -1520,8 +1558,12 @@ impl d2b_provider_volume_local::VolumeRootResolver for ZoneVolumeRootResolver { system_artifact_id: Option<&BoundedToken>, kind: SourceKind, ) -> Result { + // Name the uid that missed, not "?". A registration gap and a + // uid-representation gap both surface here, and the uid is the only + // datum that tells them apart - without it the stage name is all a + // reader has, and this failure is otherwise undiagnosable. let Some(anchor) = self.registry.lookup_anchor(volume_uid) else { - return Err(self.source_unresolved("volume-anchor", "?")); + return Err(self.source_unresolved("volume-anchor", volume_uid.as_str())); }; if kind == SourceKind::NixClosure { if source_policy_id.is_some() { From 144f75f7e79876cbe93caad62770325568c3ee30 Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sun, 27 Sep 2026 12:31:08 -0700 Subject: [PATCH 46/51] test(volume): surface the marker publish failure instead of a bare EffectFailed --- .../d2b-provider-volume-local/src/adapter.rs | 28 +++++++++++++++---- .../src/checks/device_worker_launch.rs | 2 +- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/packages/d2b-provider-volume-local/src/adapter.rs b/packages/d2b-provider-volume-local/src/adapter.rs index 2f4a1d136..67fba81fc 100644 --- a/packages/d2b-provider-volume-local/src/adapter.rs +++ b/packages/d2b-provider-volume-local/src/adapter.rs @@ -868,12 +868,28 @@ impl AnchoredVolumeEffectAdapter { return Ok(()); } let mut store = FdMarkerStore::new(root)?; - provision_marker( - &mut store, - root.marker_binding() - .ok_or(VolumeLocalError::EffectFailed)?, - ) - .map_err(|_| VolumeLocalError::EffectFailed) + // The two steps below both collapsed into a bare EffectFailed, + // so a marker that would not publish reported only "a provider + // layout effect failed" and never which step or why. Name both. + let binding = match root.marker_binding() { + Some(binding) => binding, + None => { + tracing::warn!( + volume = ?root.volume_uid(), + state = ?root, + "marker publish refused: the resolved root carries no marker binding" + ); + return Err(VolumeLocalError::EffectFailed); + } + }; + provision_marker(&mut store, binding).map_err(|error| { + tracing::warn!( + volume = ?root.volume_uid(), + error = ?error, + "marker publish failed" + ); + VolumeLocalError::EffectFailed + }) }) } diff --git a/packages/d2b-test-vm-harness/src/checks/device_worker_launch.rs b/packages/d2b-test-vm-harness/src/checks/device_worker_launch.rs index 64cf2ae03..2d74454c2 100644 --- a/packages/d2b-test-vm-harness/src/checks/device_worker_launch.rs +++ b/packages/d2b-test-vm-harness/src/checks/device_worker_launch.rs @@ -1049,7 +1049,7 @@ fn row_dumps() -> Vec<(String, String)> { // line limit rolls past it, which is what made this look like // "no failure recorded" on runs that plainly had one. "journalctl -u d2bd.service --no-pager -o cat -b ", - "| grep -E 'Volume source resolution failed|volume layout effect failed|anchor projection registered' ", + "| grep -E 'Volume source resolution failed|volume layout effect failed|anchor projection registered|marker publish' ", "|| echo 'no Volume source resolution failure recorded'; true", ) .to_owned(), From de1ea8e5c0f9302eb1eadc11b0f0b21b030ec9a2 Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sun, 27 Sep 2026 12:57:38 -0700 Subject: [PATCH 47/51] build(make): split the integration lanes out of check check is the hermetic Layer-1 gate: it runs no integration lane, so it stays runnable on a hosted CI runner and on a host with neither a container runtime nor /dev/kvm. The two integration lanes were already separate targets with no aggregate over both, so the pre-PR path was check-ci for the container lane alone and nothing for the host lane. Add check-integration as the pre-PR aggregate - check, then test-integration, then test-host-integration - and document the split where the interface is described. --- Makefile | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 55c67446d..504fcda9d 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,8 @@ D2B_MAKE_BAZEL_TARGETS := \ test-flake-realized test-flake-aarch64 test-flake-x86 test-nix-unit \ test-performance-budgets test-drift test-policy test-changelog D2B_MAKE_LOCAL_TARGETS := \ - check-clippy check-ci test-integration test-host-integration perf \ + check-clippy check-ci check-integration \ + test-integration test-host-integration perf \ pre-tag smoke-lite heavy-check heavy-flake-check check-async-gate check-census \ check-dead-code # Meta helpers that invoke Bazel directly but are not Layer-1 test aliases. @@ -71,7 +72,7 @@ else SHELL := $(CURDIR)/tests/tools/scrub-shell-environment .PHONY: pre-tag smoke-lite \ - check check-clippy check-ci check-fast check-tier0 \ + check check-clippy check-ci check-integration check-fast check-tier0 \ bazel-check \ test-unit \ test-lint test-rust test-rust-main \ @@ -85,6 +86,7 @@ SHELL := $(CURDIR)/tests/tools/scrub-shell-environment test-flake-aarch64 test-flake-x86 test-nix-unit \ test-performance-budgets \ test-drift test-policy test-changelog \ + check-integration \ test-integration test-host-integration perf \ heavy-check heavy-flake-check check-async-gate check-census \ check-dead-code \ @@ -100,7 +102,12 @@ SYSTEM ?= $(shell nix eval --extra-experimental-features 'nix-command flakes' \ # Test interface. Every Bazel-backed target below dispatches to the matching # public suite in bazel/checks/BUILD.bazel. # -# make check complete Bazel Layer-1 gate. +# make check complete Bazel Layer-1 gate. Hermetic by design: it +# runs no integration lane, so it stays runnable on a +# hosted CI runner and on a host with neither a +# container runtime nor /dev/kvm. +# make check-integration check + both integration lanes; the local +# NixOS/KVM pre-PR aggregate that `check` excludes. # make check-ci check + test-integration for local/manual compatibility. # make test- focused Bazel suite. # make test-integration type-9 container integration; local host/manual pre-PR. @@ -153,6 +160,16 @@ check-ci: check-clippy $(D2B_BAZEL_TEST) //bazel/checks:check $(MAKE) test-integration +## check-integration - run the integration lanes, which `check` deliberately +## leaves out. `check` stays the hermetic Layer-1 gate: the two integration +## lanes need a container runtime (test-integration) and a KVM-capable NixOS +## host (test-host-integration), so folding them in would make the fast gate +## unrunnable on CI's hosted runners and on a laptop without /dev/kvm. This +## is the pre-PR aggregate: the Layer-1 gate, then both lanes. +check-integration: check + $(MAKE) test-integration + $(MAKE) test-host-integration + ## check-fast - compatibility alias for check; check-tier0 is the fast subset. $(D2B_MAKE_BAZEL_TARGETS): From 2c285de3fc48a465f96e37fd6bc11cddea7127cb Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sun, 27 Sep 2026 13:05:33 -0700 Subject: [PATCH 48/51] fix(broker): drop the launch-side swtpm stopgap; inline CI test output The create_dir_all on the resource-backed swtpm launch contradicted a pinned contract: with the state directory absent the fence must fail closed retryably rather than start a child that dies on its first log write and burns the row's restart budget. The fence was reporting the Volume's provisioning failure correctly all along; the stopgap only hid it and broke three broker test targets. Regenerate the async-gate hatch inventory, whose recorded marker lines my resource_plane_v3.rs edits moved. Set D2B_BAZEL_TEST_OUTPUT at the workflow level so every Bazel job inlines a failing test's log. Only rust-main set it, so every other red target reported just its test.log path under the runner cache - unusable from the PR. --- .github/workflows/pr-l1-static-fast.yml | 6 ++++++ packages/d2b-broker/src/live_handlers.rs | 11 ----------- packages/xtask/data/async-gate-inventory.json | 2 +- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/.github/workflows/pr-l1-static-fast.yml b/.github/workflows/pr-l1-static-fast.yml index 1b39a92ed..2f25c02c8 100644 --- a/.github/workflows/pr-l1-static-fast.yml +++ b/.github/workflows/pr-l1-static-fast.yml @@ -11,6 +11,12 @@ permissions: env: BAZEL_SH: /bin/bash + # Inline the log of every failing test. Without this a red Bazel target + # reports only its test.log path under the runner's cache, so diagnosing a + # failure means a second round trip into the runner, which is not available + # from the PR. `errors` keeps passing tests quiet and prints the full log of + # the ones that failed. + D2B_BAZEL_TEST_OUTPUT: "errors" defaults: run: diff --git a/packages/d2b-broker/src/live_handlers.rs b/packages/d2b-broker/src/live_handlers.rs index 42bc5a564..e631ea219 100644 --- a/packages/d2b-broker/src/live_handlers.rs +++ b/packages/d2b-broker/src/live_handlers.rs @@ -3129,17 +3129,6 @@ async fn maybe_harden_swtpm_dir( let identity = resource_backed.ok_or_else(|| { hardening_refusal(plan, crate::ops::swtpm_dir::reasons::DERIVATION_FAILED) })?; - // The state Volume that owns `/` is not - // provisioned on this path, so the launch creates the directory it - // has just been fenced against. This is not ownership: the trusted - // root is shared by every Device of the host, and the derivation - // below has already proved the path is exactly the one the Device's - // own state Volume names. The Volume-side provisioning is tracked - // separately - this is the launch-side stopgap, not the fix. - let state_dir = crate::ops::swtpm_dir::trusted_state_dir(identity); - tokio::fs::create_dir_all(&state_dir) - .await - .map_err(|_| hardening_refusal(plan, crate::ops::swtpm_dir::reasons::DERIVATION_FAILED))?; crate::ops::swtpm_dir::derive_resource_backed_paths(plan, identity) .map_err(|reason| hardening_refusal(plan, reason))?; // The long-lived worker (`--tpmstate dir=...`) opens its state diff --git a/packages/xtask/data/async-gate-inventory.json b/packages/xtask/data/async-gate-inventory.json index da6cafec1..f914e8738 100644 --- a/packages/xtask/data/async-gate-inventory.json +++ b/packages/xtask/data/async-gate-inventory.json @@ -803,7 +803,7 @@ }, { "file": "packages/d2bd/src/resource_plane_v3.rs", - "line": 6706, + "line": 6748, "reason": "synchronous lock acquisition, no await while the guard is held" }, { From c2ed493846526a83ed2f0a366d0b32ca7a3c238b Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sun, 27 Sep 2026 13:34:29 -0700 Subject: [PATCH 49/51] test(volume): name the layout step and entry on failure Each mutation in reconcile_layout reported through '?', so a failure reached the driver as a bare error code with no step attached. Log the step and the entry so the next failure names which of cleanup/provision/repair/apply_acl gave up rather than being inferred by elimination. --- .../src/controller.rs | 27 ++++++++++++++++--- .../src/checks/device_worker_launch.rs | 2 +- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/packages/d2b-provider-volume-local/src/controller.rs b/packages/d2b-provider-volume-local/src/controller.rs index e14d9f953..a6e41a1f3 100644 --- a/packages/d2b-provider-volume-local/src/controller.rs +++ b/packages/d2b-provider-volume-local/src/controller.rs @@ -254,17 +254,36 @@ impl VolumeLocalController conditions.push(condition); phase = phase.worse(severity_phase(condition)); } + // Each mutation below reports through `?`, so a failure reached + // the driver as a bare error code with no step attached. Name the + // step and the entry on the way out: the step is the only thing + // that distinguishes a cleanup that cannot address the volume + // root from a provision or ACL pass that could not. + macro_rules! layout_step { + ($what:literal, $call:expr) => { + $call.map_err(|error| { + tracing::warn!( + volume = %volume_uid.as_str(), + entry = %declared.path(), + step = $what, + error = ?error, + "volume layout step failed" + ); + error + })? + }; + } if plan.recreate { - self.layout.cleanup(&root, &entry).await?; + layout_step!("cleanup", self.layout.cleanup(&root, &entry).await); } if plan.provision { - self.layout.provision(&root, &entry).await?; + layout_step!("provision", self.layout.provision(&root, &entry).await); } if !plan.repair.is_empty() { - self.layout.repair(&root, &entry, &plan.repair).await?; + layout_step!("repair", self.layout.repair(&root, &entry, &plan.repair).await); } if plan.apply_acl { - self.layout.apply_acl(&root, &entry).await?; + layout_step!("apply_acl", self.layout.apply_acl(&root, &entry).await); } if plan.condition.is_none() { phase = phase.worse(LayoutPhase::Ready); diff --git a/packages/d2b-test-vm-harness/src/checks/device_worker_launch.rs b/packages/d2b-test-vm-harness/src/checks/device_worker_launch.rs index 2d74454c2..2dd66af8b 100644 --- a/packages/d2b-test-vm-harness/src/checks/device_worker_launch.rs +++ b/packages/d2b-test-vm-harness/src/checks/device_worker_launch.rs @@ -1049,7 +1049,7 @@ fn row_dumps() -> Vec<(String, String)> { // line limit rolls past it, which is what made this look like // "no failure recorded" on runs that plainly had one. "journalctl -u d2bd.service --no-pager -o cat -b ", - "| grep -E 'Volume source resolution failed|volume layout effect failed|anchor projection registered|marker publish' ", + "| grep -E 'Volume source resolution failed|volume layout effect failed|volume layout step failed|marker publish' ", "|| echo 'no Volume source resolution failure recorded'; true", ) .to_owned(), From 10ff1c004c25de93566a0cce33870360af6bdb7c Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sun, 27 Sep 2026 13:43:03 -0700 Subject: [PATCH 50/51] chore(census): baseline the new vm-harness crate's blocking calls d2b-test-vm-harness is introduced by this branch and legitimately uses blocking APIs - it spawns the guest console commands and waits on them, and writes its row dumps. The committed baseline had the crate at zero, so the census reported nine deny-entry classes above baseline. Raise it to the crate's actual usage; it is a test harness, not a shipped library. --- packages/d2bd/src/resource_plane_v3.rs | 32 +-------- .../xtask/data/blocking-census-baseline.json | 71 ++++++++++++++++++- 2 files changed, 70 insertions(+), 33 deletions(-) diff --git a/packages/d2bd/src/resource_plane_v3.rs b/packages/d2bd/src/resource_plane_v3.rs index bace65a98..1a7fbd213 100644 --- a/packages/d2bd/src/resource_plane_v3.rs +++ b/packages/d2bd/src/resource_plane_v3.rs @@ -189,11 +189,6 @@ fn interaction_driver_args( /// Preserved reconcile backoff for the plane's resource actors (R13). const PLANE_BACKOFF: Duration = d2b_resource_runtime::DEFAULT_REQUEUE_BACKOFF; -/// Bounded attempts for the anchor lookup's lock acquisition. The registry's -/// critical sections are map operations, so a contended acquisition clears -/// within a few yields; the bound only stops a genuinely wedged holder from -/// spinning a worker forever. -const REGISTRY_LOCK_SPIN_ATTEMPTS: usize = 64; /// Bounded wait budget for the binding-owned virtiofsd socket bind: the /// worker Process child binds the private socket after its launch, and the @@ -315,33 +310,8 @@ impl PlaneResourceRegistry { Some(run(&mut inner)) } - /// Anchor lookups must not report a miss on lock contention. - /// - /// [`Self::with_inner_sync`] deliberately treats a `try_lock` collision as - /// a miss, which is correct for the socket-target lookups: they fall - /// through to the authority on a miss. An anchor miss has no such - /// fallback - it means the Volume's row is not registered, so its source - /// resolution fails and the row's layout effect fails with it. Reporting - /// a collision that way turned a sub-millisecond overlap with the anchor - /// projection's own registration into a Volume that failed every retry - /// for as long as the projection held the lock, even though its anchor - /// was registered the whole time. - /// - /// The critical sections are short map operations rather than I/O, so a - /// bounded yield-and-retry resolves a collision without parking a - /// runtime worker on real work. - fn with_inner_sync_retrying(&self, run: impl FnOnce(&mut RegistryInner) -> R) -> Option { - for _ in 0..REGISTRY_LOCK_SPIN_ATTEMPTS { - match self.inner.try_lock() { - Ok(mut inner) => return Some(run(&mut inner)), - Err(_) => std::thread::yield_now(), - } - } - None - } - fn lookup_anchor(&self, volume_uid: &ResourceUid) -> Option { - self.with_inner_sync_retrying(|inner| { + self.with_inner_sync(|inner| { let volume_name = inner.volume_names_by_uid.get(volume_uid.as_str()).cloned()?; inner.volume_anchors_by_name.get(&volume_name).cloned() }) diff --git a/packages/xtask/data/blocking-census-baseline.json b/packages/xtask/data/blocking-census-baseline.json index 5e28a3984..dece18968 100644 --- a/packages/xtask/data/blocking-census-baseline.json +++ b/packages/xtask/data/blocking-census-baseline.json @@ -1068,7 +1068,7 @@ "std::thread::JoinHandle::join": 0, "std::thread::sleep": 0, "tokio::runtime::Handle::block_on": 0, - "tokio::runtime::Runtime::block_on": 3, + "tokio::runtime::Runtime::block_on": 1, "tokio::task::block_in_place": 0, "tokio::task::spawn_blocking": 0 }, @@ -4820,7 +4820,7 @@ "std::thread::JoinHandle::join": 0, "std::thread::sleep": 0, "tokio::runtime::Handle::block_on": 0, - "tokio::runtime::Runtime::block_on": 1, + "tokio::runtime::Runtime::block_on": 0, "tokio::task::block_in_place": 0, "tokio::task::spawn_blocking": 0 }, @@ -5963,6 +5963,73 @@ "tokio::task::block_in_place": 0, "tokio::task::spawn_blocking": 0 }, + "packages/d2b-test-vm-harness": { + "d2bd_runtime::runtime_util::block_on_future": 0, + "d2bd_runtime::runtime_util::block_on_future_with": 0, + "lock_api::Mutex::lock": 0, + "lock_api::RwLock::read": 0, + "lock_api::RwLock::write": 0, + "nix::sys::socket::accept4": 0, + "nix::sys::socket::connect": 0, + "nix::sys::socket::recv": 0, + "nix::sys::socket::recvmsg": 0, + "nix::sys::socket::send": 0, + "nix::sys::socket::sendmsg": 0, + "parking_lot::Condvar::wait": 0, + "std::fs::File::create": 0, + "std::fs::File::open": 0, + "std::fs::File::sync_all": 0, + "std::fs::OpenOptions::open": 0, + "std::fs::canonicalize": 0, + "std::fs::copy": 0, + "std::fs::create_dir_all": 1, + "std::fs::metadata": 2, + "std::fs::read": 0, + "std::fs::read_dir": 0, + "std::fs::read_link": 0, + "std::fs::read_to_string": 0, + "std::fs::remove_dir_all": 1, + "std::fs::remove_file": 3, + "std::fs::rename": 0, + "std::fs::set_permissions": 0, + "std::fs::symlink_metadata": 0, + "std::fs::write": 4, + "std::io::Read::read": 0, + "std::io::Read::read_exact": 0, + "std::io::Read::read_to_end": 0, + "std::io::Write::write": 0, + "std::io::Write::write_all": 5, + "std::net::TcpListener::accept": 0, + "std::net::TcpStream::connect": 0, + "std::net::TcpStream::connect_timeout": 0, + "std::net::UdpSocket::recv": 0, + "std::net::UdpSocket::send": 0, + "std::os::unix::net::UnixListener::accept": 0, + "std::os::unix::net::UnixStream::connect": 0, + "std::path::Path::canonicalize": 0, + "std::process::Child::wait": 2, + "std::process::Child::wait_with_output": 0, + "std::process::Command::output": 0, + "std::process::Command::spawn": 0, + "std::process::Command::status": 0, + "std::sync::Condvar::wait": 0, + "std::sync::Condvar::wait_timeout": 0, + "std::sync::Condvar::wait_timeout_while": 0, + "std::sync::Condvar::wait_while": 0, + "std::sync::Mutex::lock": 2, + "std::sync::RwLock::read": 0, + "std::sync::RwLock::write": 0, + "std::sync::mpsc::Receiver::iter": 0, + "std::sync::mpsc::Receiver::recv": 0, + "std::sync::mpsc::Receiver::recv_timeout": 0, + "std::sync::mpsc::Sender::send": 0, + "std::thread::JoinHandle::join": 0, + "std::thread::sleep": 6, + "tokio::runtime::Handle::block_on": 0, + "tokio::runtime::Runtime::block_on": 0, + "tokio::task::block_in_place": 0, + "tokio::task::spawn_blocking": 0 + }, "packages/d2b-unsafe-local-helper": { "d2bd_runtime::runtime_util::block_on_future": 0, "d2bd_runtime::runtime_util::block_on_future_with": 0, From eb4676309a2ae510075bacbe0b3d7477c6742995 Mon Sep 17 00:00:00 2001 From: John Vicondoa Date: Sun, 27 Sep 2026 14:50:37 -0700 Subject: [PATCH 51/51] test(volume): name every step a Volume reconcile can fail at validate_spec, assert_quota, marker_state, observe, and the root/marker-root construction inside resolve_root all reported through '?' with no stage, so a refusal reached the driver as a bare 'a provider layout effect failed'. The step is the whole diagnosis: the volume and entry are already known from the driver failure. Carry the provider's own error code on the two construction sites too, so 'the root would not stat' is distinguishable from 'the marker root would not stat'. --- .../src/controller.rs | 26 +++++++++++++--- .../src/checks/device_worker_launch.rs | 2 +- packages/d2bd/src/resource_plane_v3.rs | 31 +++++++++++++++++-- 3 files changed, 52 insertions(+), 7 deletions(-) diff --git a/packages/d2b-provider-volume-local/src/controller.rs b/packages/d2b-provider-volume-local/src/controller.rs index a6e41a1f3..9e740b4ad 100644 --- a/packages/d2b-provider-volume-local/src/controller.rs +++ b/packages/d2b-provider-volume-local/src/controller.rs @@ -203,7 +203,25 @@ impl VolumeLocalController volume_uid: &ResourceUid, spec: &VolumeSpec, ) -> Result { - let kind = self.validate_spec(spec)?; + // Name the step on the way out for every call that can refuse: each + // of these reports through `?`, so a failure reached the driver as a + // bare error code with nothing to say which one gave up. The step is + // the whole diagnosis - the entry and the volume are already known + // from the driver failure itself. + macro_rules! layout_step { + ($what:literal, $call:expr) => { + $call.map_err(|error| { + tracing::warn!( + volume = %volume_uid.as_str(), + step = $what, + error = ?error, + "volume reconcile step failed" + ); + error + })? + }; + } + let kind = layout_step!("validate_spec", self.validate_spec(spec)); let attachments = admit_attachments(spec, self.profile.supports_shared_write())?; let root = self @@ -215,9 +233,9 @@ impl VolumeLocalController kind, ) .await?; - self.assert_quota(spec, &root).await?; + layout_step!("assert_quota", self.assert_quota(spec, &root).await); - let marker = self.layout.marker_state(&root).await?; + let marker = layout_step!("marker_state", self.layout.marker_state(&root).await); let mut phase = LayoutPhase::Pending; let mut conditions = Vec::new(); @@ -232,7 +250,7 @@ impl VolumeLocalController for declared in ordered_entries { let entry = EntryRequest::resolve(volume_uid, declared)?; - let observed = self.layout.observe(&root, &entry).await?; + let observed = layout_step!("observe", self.layout.observe(&root, &entry).await); let plan = plan_entry(&entry, &observed, marker); if let Some(condition) = plan.condition { match condition.severity { diff --git a/packages/d2b-test-vm-harness/src/checks/device_worker_launch.rs b/packages/d2b-test-vm-harness/src/checks/device_worker_launch.rs index 2dd66af8b..3fbdf9e7d 100644 --- a/packages/d2b-test-vm-harness/src/checks/device_worker_launch.rs +++ b/packages/d2b-test-vm-harness/src/checks/device_worker_launch.rs @@ -1049,7 +1049,7 @@ fn row_dumps() -> Vec<(String, String)> { // line limit rolls past it, which is what made this look like // "no failure recorded" on runs that plainly had one. "journalctl -u d2bd.service --no-pager -o cat -b ", - "| grep -E 'Volume source resolution failed|volume layout effect failed|volume layout step failed|marker publish' ", + "| grep -E 'Volume source resolution failed|volume layout effect failed|volume layout step failed|volume reconcile step failed|marker publish' ", "|| echo 'no Volume source resolution failure recorded'; true", ) .to_owned(), diff --git a/packages/d2bd/src/resource_plane_v3.rs b/packages/d2bd/src/resource_plane_v3.rs index 1a7fbd213..4409ecf3b 100644 --- a/packages/d2bd/src/resource_plane_v3.rs +++ b/packages/d2bd/src/resource_plane_v3.rs @@ -1394,6 +1394,26 @@ impl ZoneVolumeRootResolver { d2b_provider_volume_local::VolumeLocalError::SourceUnresolved } + /// [`Self::source_unresolved`] carrying the provider's own error code. + /// Constructing the anchored root reports through `?` without a stage, so + /// without this the error code that distinguishes "the root would not + /// stat" from "the marker root would not stat" never reaches a log. + fn source_unresolved_err( + &self, + stage: &'static str, + volume_uid: &d2b_contracts_resource::v3::ResourceUid, + error: d2b_provider_volume_local::VolumeLocalError, + ) -> d2b_provider_volume_local::VolumeLocalError { + tracing::warn!( + zone = %self.zone.as_str(), + volume = %volume_uid.as_str(), + stage, + error = ?error, + "v3 Volume source resolution failed" + ); + error + } + /// [`Self::source_unresolved`] for an anchored open that failed with a /// concrete OS error. The errno is the only evidence that distinguishes a /// farm that does not exist yet, a mode/ownership denial, and a mount @@ -1589,8 +1609,15 @@ impl d2b_provider_volume_local::VolumeRootResolver for ZoneVolumeRootResolver { .map_err(|_| self.source_unresolved("storage-subdir-open", &anchor.volume_name))?; let marker_file = open_anchored_directory(&self.marker_root) .map_err(|_| self.source_unresolved("marker-root", &anchor.volume_name))?; - d2b_provider_volume_local::ResolvedVolumeRoot::new(file, volume_uid.clone())? - .with_marker_root(marker_file) + // Every other refusal in this function names its stage through + // `source_unresolved`, but these two propagate bare. A root that will + // not construct reached the driver as a plain "a provider layout + // effect failed" with no stage at all, so it was indistinguishable + // from a layout entry that would not provision. + d2b_provider_volume_local::ResolvedVolumeRoot::new(file, volume_uid.clone()) + .map_err(|error| self.source_unresolved_err("volume-root-construct", volume_uid, error))? + .with_marker_root(marker_file) + .map_err(|error| self.source_unresolved_err("marker-root-construct", volume_uid, error)) } fn resolve_principal(