From f06e96c4e903cef5c4fcf5d22b806b78775c86c9 Mon Sep 17 00:00:00 2001 From: Richard Kiene Date: Thu, 20 Aug 2026 11:55:26 -0700 Subject: [PATCH 01/13] feat: a conformance crate, because gascan-core denies the unwraps a suite needs --- Cargo.lock | 10 ++++ Cargo.toml | 2 +- crates/gascan-conformance/Cargo.toml | 18 ++++++ crates/gascan-conformance/src/lib.rs | 84 ++++++++++++++++++++++++++++ 4 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 crates/gascan-conformance/Cargo.toml create mode 100644 crates/gascan-conformance/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 916e597..88966da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -518,6 +518,16 @@ dependencies = [ "tower 0.4.13", ] +[[package]] +name = "gascan-conformance" +version = "0.1.20" +dependencies = [ + "camino", + "gascan-core", + "tempfile", + "tokio", +] + [[package]] name = "gascan-core" version = "0.1.20" diff --git a/Cargo.toml b/Cargo.toml index 93afbc5..cd3bb98 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["crates/gascan", "crates/gascan-apple", "crates/gascan-arca", "crates/gascan-core", "crates/gascan-e2e", "crates/gascan-engine-proto", "crates/gascan-inherited-fd", "crates/gascan-oci-fixture", "crates/gascan-proto", "crates/gascand"] +members = ["crates/gascan", "crates/gascan-apple", "crates/gascan-arca", "crates/gascan-conformance", "crates/gascan-core", "crates/gascan-e2e", "crates/gascan-engine-proto", "crates/gascan-inherited-fd", "crates/gascan-oci-fixture", "crates/gascan-proto", "crates/gascand"] resolver = "3" [workspace.package] diff --git a/crates/gascan-conformance/Cargo.toml b/crates/gascan-conformance/Cargo.toml new file mode 100644 index 0000000..6f62a2a --- /dev/null +++ b/crates/gascan-conformance/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "gascan-conformance" +version = "0.1.20" +edition.workspace = true +license.workspace = true +rust-version.workspace = true +publish = false + +[dependencies] +camino.workspace = true +gascan-core = { path = "../gascan-core" } +tempfile = "3" + +[dev-dependencies] +tokio = { workspace = true, features = ["macros", "rt", "rt-multi-thread", "sync", "time"] } + +[lints] +workspace = true diff --git a/crates/gascan-conformance/src/lib.rs b/crates/gascan-conformance/src/lib.rs new file mode 100644 index 0000000..e2dec48 --- /dev/null +++ b/crates/gascan-conformance/src/lib.rs @@ -0,0 +1,84 @@ +//! Backend conformance: one contract, run against every `RuntimeBackend`. +//! +//! This crate exists because `gascan-core/src/lib.rs:2` denies +//! `clippy::unwrap_used`, and a conformance suite is built from unwrapping +//! assertions. It is a dev-dependency of its consumers and ships nowhere. + +use camino::Utf8Path; +use gascan_core::manifest::Manifest; +use gascan_core::policy::PolicyCompiler; +use gascan_core::runtime::{CreateRequest, NetworkIsolation, RuntimeCapabilities, RuntimeVersion}; +use gascan_core::sandbox::SandboxSpec; +use std::ops::Deref; + +pub struct CreateRequestFixture { + _root: tempfile::TempDir, + request: CreateRequest, +} + +impl CreateRequestFixture { + /// A request against the approved workspace image. + /// + /// Correct for the fake and for apple. **Wrong for a live engine**, whose + /// store holds only what the tier seeded -- use [`Self::for_image`] there. + pub fn pinned(name: &str, network: &str) -> Self { + assert!(matches!(network, "offline" | "networked")); + Self::build(name, &format!("version = 1\nnetwork = '{network}'\n"), None) + } + + /// A request against `image`, for a backend whose store was seeded with it. + /// + /// The manifest is the only knob, matching `policy_request_from_manifest` + /// in arca's live tier: the guest user and any ports are manifest facts, + /// and a caller reaching around them would build a request gascan itself + /// cannot produce. + pub fn for_image(name: &str, image: &str, manifest: &str) -> Self { + Self::build(name, manifest, Some(image)) + } + + pub fn request(&self) -> CreateRequest { + self.request.clone() + } + + fn build(name: &str, manifest_text: &str, image: Option<&str>) -> Self { + let temp = tempfile::tempdir().expect("temporary backend-contract root"); + let root = Utf8Path::from_path(temp.path()).expect("UTF-8 temporary path"); + std::fs::write(root.join("gascan.toml"), manifest_text) + .expect("write backend-contract manifest"); + let manifest = Manifest::load(root).expect("load backend-contract manifest"); + let spec = SandboxSpec::from_root(name, root, manifest).expect("build sealed sandbox spec"); + let request = match image { + None => PolicyCompiler::compile(spec, &capabilities()), + Some(image) => PolicyCompiler::compile_for_image(spec, &capabilities(), image), + } + .expect("compile backend-contract policy"); + Self { + _root: temp, + request, + } + } +} + +impl Deref for CreateRequestFixture { + type Target = CreateRequest; + + fn deref(&self) -> &Self::Target { + &self.request + } +} + +/// Every flag true. The compiler gates on what a runtime CLAIMS, and the +/// contract only needs a well-formed request; what is under test is the +/// backend's behaviour, not the compiler's gating. +pub fn capabilities() -> RuntimeCapabilities { + RuntimeCapabilities { + version: RuntimeVersion::new(1, 1, 0), + bind_mounts: true, + named_volumes: true, + tty: true, + signals: true, + loopback_publish: true, + resource_limits: true, + offline: NetworkIsolation::Proven, + } +} From 9dcca6af7dae7fbe01e3a162f0dd9012ee44d372 Mon Sep 17 00:00:00 2001 From: Richard Kiene Date: Thu, 20 Aug 2026 12:34:45 -0700 Subject: [PATCH 02/13] feat: the backend contract moves to where every backend can reach it `backend_contract` lived in `crates/gascan-core/tests/backend_contract.rs`, a test target rather than a library, so no other crate could import it. It moves to `gascan-conformance`, which apple and arca can both depend on. The body is the original at `crates/gascan-core/tests/backend_contract.rs:149-179` unchanged but for three things: the signature takes `fixture: &CreateRequestFixture`, the `create_request("contract")` line goes away because the request is now passed in, and one fully-qualified `gascan_core::runtime::ContainerState::Stopped` shortens against the new import. `diff` of the two bodies shows those three and nothing else. The fixture is a parameter because apple and arca pin different images: `PolicyCompiler::compile` pins the approved workspace image, which a live engine's seeded store does not hold. The old copy in `gascan-core` is deliberately left in place; a later commit removes it. `tests/fake.rs` instantiates the contract against `FakeRuntime` through a `&dyn RuntimeBackend`. It binds the fixture to a local before borrowing it, so the fixture's `TempDir` outlives the request that points into it. Verified at this tree: - `cargo test -p gascan-conformance --test fake` before the move: exit 101, E0432 unresolved import `gascan_conformance::backend_contract`. - After the move: exit 0, 1 passed. - Inverting the final assertion to `assert!(backend.inspect(&id).await.unwrap().is_some())` and re-running: exit 101, panic at `crates/gascan-conformance/src/lib.rs:124:5`. Reverted and re-run: exit 0, and `cmp` against the pre-edit copy exits 0, so the revert is byte-identical. - `cargo fmt --all --check`: exit 0. - `cargo clippy --workspace --all-targets -- -D warnings`: exit 0. `gascan-core` is untouched: `git diff --quiet HEAD -- crates/gascan-core/` exits 0. Its lint gate at `crates/gascan-core/src/lib.rs:2` is unchanged, and `grep -rn "allow(clippy::\(expect_used\|panic\|unwrap_used\))" crates/` finds no matches. --- crates/gascan-conformance/src/lib.rs | 41 +++++++++++++++++++++++++ crates/gascan-conformance/tests/fake.rs | 10 ++++++ 2 files changed, 51 insertions(+) create mode 100644 crates/gascan-conformance/tests/fake.rs diff --git a/crates/gascan-conformance/src/lib.rs b/crates/gascan-conformance/src/lib.rs index e2dec48..866d416 100644 --- a/crates/gascan-conformance/src/lib.rs +++ b/crates/gascan-conformance/src/lib.rs @@ -82,3 +82,44 @@ pub fn capabilities() -> RuntimeCapabilities { offline: NetworkIsolation::Proven, } } + +use gascan_core::runtime::{ + ContainerState, ExecInput, ExecOutput, ExecRequest, RemoveRequest, ResourceKind, RuntimeBackend, +}; + +/// The contract every `RuntimeBackend` owes, whatever it is implemented over. +/// +/// `fixture` is a parameter and not built here because `PolicyCompiler::compile` +/// pins the approved workspace image, which a live engine's seeded store does +/// not hold -- see `CreateRequestFixture::for_image`. +pub async fn backend_contract(backend: &dyn RuntimeBackend, fixture: &CreateRequestFixture) { + let id = fixture.id().clone(); + assert_eq!(backend.inspect(&id).await.unwrap(), None); + let created = backend.create(fixture.request()).await.unwrap(); + assert!( + created + .created() + .iter() + .any(|resource| resource.kind() == ResourceKind::Container) + ); + assert_eq!( + backend.inspect(&id).await.unwrap().unwrap().state, + ContainerState::Stopped + ); + backend.start(&id).await.unwrap(); + let mut session = backend + .exec(ExecRequest::fixture(id.clone(), ["true"])) + .await + .unwrap(); + session.send(ExecInput::Close).await.unwrap(); + assert_eq!( + session.next().await.unwrap().unwrap(), + ExecOutput::Exit { code: 0, signal: 0 } + ); + backend.stop(&id).await.unwrap(); + backend + .remove(RemoveRequest::from_resources(created.created().to_vec()).unwrap()) + .await + .unwrap(); + assert_eq!(backend.inspect(&id).await.unwrap(), None); +} diff --git a/crates/gascan-conformance/tests/fake.rs b/crates/gascan-conformance/tests/fake.rs new file mode 100644 index 0000000..83fe0fb --- /dev/null +++ b/crates/gascan-conformance/tests/fake.rs @@ -0,0 +1,10 @@ +use gascan_conformance::{CreateRequestFixture, backend_contract, capabilities}; +use gascan_core::fake_runtime::FakeRuntime; +use gascan_core::runtime::RuntimeBackend; + +#[tokio::test] +async fn fake_runtime_satisfies_the_backend_contract() { + let backend: Box = Box::new(FakeRuntime::new(capabilities())); + let fixture = CreateRequestFixture::pinned("contract", "offline"); + backend_contract(backend.as_ref(), &fixture).await; +} From daa687bd57dbd43e5f306849f369337099647bf3 Mon Sep 17 00:00:00 2001 From: Richard Kiene Date: Thu, 20 Aug 2026 13:25:10 -0700 Subject: [PATCH 03/13] refactor: gascan-core stops owning a contract every backend needs `crates/gascan-conformance` has held `backend_contract` since 9dcca6a, and until now `gascan-core` held a second copy of it. Delete the original. Removed from `crates/gascan-core/tests/backend_contract.rs`, line numbers as they stood at 9dcca6a: - `:149-181`, `pub async fn backend_contract(backend: &dyn RuntimeBackend)`. - `:653-658`, `fake_runtime_satisfies_backend_contract_through_trait_object`, its only caller. `grep -rn "backend_contract(" crates/` at 9dcca6a found exactly that one call site; `crates/gascan-conformance/tests/fake.rs` replaces both. `crates/gascan-core/tests/common/mod.rs` is deliberately unchanged. It is compiled into one target only -- `grep -rn "mod common;" crates/gascan-core/tests/` returns a single hit, `backend_contract.rs:1` -- and every fixture it exports still has callers among the fake-only tests that remain: 22 uses of `capabilities`, 26 of `create_request`, 10 of `create_request_with_network`. Nothing there became unused, so nothing there was deleted. No import in `backend_contract.rs` became unused either; `RuntimeBackend` now appears only on its `use` line but is still required to bring the trait's methods into scope. Also merges the two `use gascan_core::runtime::{...}` groups in `crates/gascan-conformance/src/lib.rs` into the one at `:10`. The second, at `:86-88`, was an artifact of keeping 9dcca6a append-only so its review could see the relocation was a relocation; that reason is spent. Verified: - `cargo test -p gascan-core` before the deletion: exit 0, 175 passed, 0 failed, 0 ignored. - `cargo test -p gascan-core` after: exit 0, 174 passed, 0 failed, 0 ignored. Delta is exactly 1, the removed trait-object test. No other test was deleted, weakened, or ignored, and none failed. - `cargo clippy -p gascan-core --all-targets -- -D warnings`: exit 0. - `cargo clippy -p gascan-conformance --all-targets -- -D warnings`: exit 0. - `cargo test -p gascan-conformance --test fake`: exit 0, 1 passed. - `cargo fmt --all --check`: exit 0. --- crates/gascan-conformance/src/lib.rs | 9 ++--- crates/gascan-core/tests/backend_contract.rs | 39 -------------------- 2 files changed, 4 insertions(+), 44 deletions(-) diff --git a/crates/gascan-conformance/src/lib.rs b/crates/gascan-conformance/src/lib.rs index 866d416..535e296 100644 --- a/crates/gascan-conformance/src/lib.rs +++ b/crates/gascan-conformance/src/lib.rs @@ -7,7 +7,10 @@ use camino::Utf8Path; use gascan_core::manifest::Manifest; use gascan_core::policy::PolicyCompiler; -use gascan_core::runtime::{CreateRequest, NetworkIsolation, RuntimeCapabilities, RuntimeVersion}; +use gascan_core::runtime::{ + ContainerState, CreateRequest, ExecInput, ExecOutput, ExecRequest, NetworkIsolation, + RemoveRequest, ResourceKind, RuntimeBackend, RuntimeCapabilities, RuntimeVersion, +}; use gascan_core::sandbox::SandboxSpec; use std::ops::Deref; @@ -83,10 +86,6 @@ pub fn capabilities() -> RuntimeCapabilities { } } -use gascan_core::runtime::{ - ContainerState, ExecInput, ExecOutput, ExecRequest, RemoveRequest, ResourceKind, RuntimeBackend, -}; - /// The contract every `RuntimeBackend` owes, whatever it is implemented over. /// /// `fixture` is a parameter and not built here because `PolicyCompiler::compile` diff --git a/crates/gascan-core/tests/backend_contract.rs b/crates/gascan-core/tests/backend_contract.rs index a86198c..cb256ea 100644 --- a/crates/gascan-core/tests/backend_contract.rs +++ b/crates/gascan-core/tests/backend_contract.rs @@ -146,39 +146,6 @@ async fn persistent_logs_are_isolated_by_exact_sandbox_id() { ); } -pub async fn backend_contract(backend: &dyn RuntimeBackend) { - let fixture = create_request("contract"); - let id = fixture.id().clone(); - assert_eq!(backend.inspect(&id).await.unwrap(), None); - let created = backend.create(fixture.request()).await.unwrap(); - assert!( - created - .created() - .iter() - .any(|resource| resource.kind() == ResourceKind::Container) - ); - assert_eq!( - backend.inspect(&id).await.unwrap().unwrap().state, - gascan_core::runtime::ContainerState::Stopped - ); - backend.start(&id).await.unwrap(); - let mut session = backend - .exec(ExecRequest::fixture(id.clone(), ["true"])) - .await - .unwrap(); - session.send(ExecInput::Close).await.unwrap(); - assert_eq!( - session.next().await.unwrap().unwrap(), - ExecOutput::Exit { code: 0, signal: 0 } - ); - backend.stop(&id).await.unwrap(); - backend - .remove(RemoveRequest::from_resources(created.created().to_vec()).unwrap()) - .await - .unwrap(); - assert_eq!(backend.inspect(&id).await.unwrap(), None); -} - #[tokio::test] async fn inventory_reports_owned_foreign_and_mismatched_resources() { let backend = FakeRuntime::new(capabilities()); @@ -650,12 +617,6 @@ async fn injected_post_mutation_create_failure_reports_partial_resources() { assert_eq!(failure.created().len(), 2); } -#[tokio::test] -async fn fake_runtime_satisfies_backend_contract_through_trait_object() { - let backend: Box = Box::new(FakeRuntime::new(capabilities())); - backend_contract(backend.as_ref()).await; -} - #[test] fn validated_fixture_keeps_its_canonical_bind_source_alive() { let fixture = create_request("live-root"); From 0e1f3fbbb5d7e44918b3eab37349b756a2d0aa84 Mon Sep 17 00:00:00 2001 From: Richard Kiene Date: Thu, 20 Aug 2026 13:45:11 -0700 Subject: [PATCH 04/13] refactor: apple runs the shared contract instead of its own copy of it crates/gascan-apple/tests/live/backend_contract.rs walked the same ground as the shared suite in 65 hand-rolled lines. It now calls gascan_conformance::backend_contract, keeping the one assertion the shared contract does not make: that list_resources() reports nothing named for the sandbox once it has been removed. gascan-conformance is a [dev-dependencies] entry only, so it is compiled into no shipped artifact. The test's name changed, so tests/ci/expected-ignored-tests.txt line 8 becomes backend_contract::backend_contract_holds_on_apple. Before that edit ./scripts/ci-check-ignored-tests.sh exited 1, naming exactly that one removal and one addition; after it, exit 0, "49 ignored test(s), matching the baseline". THE LIVE RUN FAILED. On host newcombe (Darwin 25.6.0 arm64, container CLI 1.1.0, service running) on 2026-08-20, cargo test -p gascan-apple --test live -- --ignored backend_contract_holds_on_apple exited 101: panicked at crates/gascan-conformance/src/lib.rs:104:5: assertion `left == right` failed left: Running right: Stopped That is the shared contract's post-create state assertion. It is not a regression from the extraction: the same assertion stands byte-identical at daa687b^:crates/gascan-core/tests/backend_contract.rs:160-163, where it only ever ran against FakeRuntime. Apple's create emits `container run` (crates/gascan-apple/src/translate.rs:100), which starts the container, so apple leaves the sandbox Running where the fake leaves it Stopped. Apple's old file asserted nothing about state after create, so the divergence was never covered. It is left unfixed here, and the contract is not softened to accommodate it. Because the run stops at that assertion, whether apple satisfies the rest of the shared contract -- the exec walk in particular -- is still unmeasured. The old file called start twice and stop twice to assert idempotence; the shared contract calls each once. Apple's idempotence is therefore asserted nowhere until a later task in this plan promotes it into the shared contract. Residue from the failed run -- container gascan-live-backend-92391-1787258495344035000-2e7e3b521ca5 and its three gascan-{cache,mise,config}- volumes -- was removed by exact name; `container list --all`, `container volume list` and `container network list` afterwards match their pre-run output. cargo test -p gascan-apple --no-run, cargo fmt --all --check and cargo clippy --workspace --all-targets -- -D warnings each exited 0. --- Cargo.lock | 1 + crates/gascan-apple/Cargo.toml | 1 + .../tests/live/backend_contract.rs | 52 +++---------------- tests/ci/expected-ignored-tests.txt | 2 +- 4 files changed, 9 insertions(+), 47 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 88966da..0537087 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -486,6 +486,7 @@ dependencies = [ "async-trait", "base64", "camino", + "gascan-conformance", "gascan-core", "libc", "rustix", diff --git a/crates/gascan-apple/Cargo.toml b/crates/gascan-apple/Cargo.toml index 2c735ca..eb5fa55 100644 --- a/crates/gascan-apple/Cargo.toml +++ b/crates/gascan-apple/Cargo.toml @@ -17,6 +17,7 @@ tokio.workspace = true thiserror.workspace = true [dev-dependencies] +gascan-conformance = { path = "../gascan-conformance" } libc = "0.2" tempfile = "3" tokio = { workspace = true, features = ["macros", "rt", "time"] } diff --git a/crates/gascan-apple/tests/live/backend_contract.rs b/crates/gascan-apple/tests/live/backend_contract.rs index b45c2b4..b2d10fc 100644 --- a/crates/gascan-apple/tests/live/backend_contract.rs +++ b/crates/gascan-apple/tests/live/backend_contract.rs @@ -1,59 +1,19 @@ -use std::time::{SystemTime, UNIX_EPOCH}; - -use camino::Utf8Path; use gascan_apple::{AppleBackend, ProcessRunner}; -use gascan_core::{ - manifest::Manifest, - policy::PolicyCompiler, - runtime::{ - NetworkIsolation, RemoveRequest, RuntimeBackend, RuntimeCapabilities, RuntimeVersion, - }, - sandbox::SandboxSpec, -}; +use gascan_conformance::{CreateRequestFixture, backend_contract}; +use gascan_core::runtime::RuntimeBackend; +use std::time::{SystemTime, UNIX_EPOCH}; #[tokio::test] #[ignore = "requires Apple silicon macOS 26+ with container service and locked workspace image"] -async fn backend_contract() { +async fn backend_contract_holds_on_apple() { let nonce = SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap() .as_nanos(); let name = format!("gascan-live-backend-{}-{nonce}", std::process::id()); - let root = tempfile::tempdir().unwrap(); - let path = Utf8Path::from_path(root.path()).unwrap(); - std::fs::write( - path.join("gascan.toml"), - "version = 1\nnetwork = 'offline'\n", - ) - .unwrap(); - let spec = SandboxSpec::from_root(&name, path, Manifest::load(path).unwrap()).unwrap(); - let request = PolicyCompiler::compile( - spec, - &RuntimeCapabilities { - version: RuntimeVersion::new(1, 1, 0), - bind_mounts: true, - named_volumes: true, - tty: true, - signals: true, - loopback_publish: true, - resource_limits: true, - offline: NetworkIsolation::Proven, - }, - ) - .unwrap(); - let id = request.id().clone(); + let fixture = CreateRequestFixture::pinned(&name, "offline"); let backend = AppleBackend::new(ProcessRunner); - assert!(backend.inspect(&id).await.unwrap().is_none()); - let created = backend.create(request).await.unwrap(); - backend.start(&id).await.unwrap(); - backend.start(&id).await.unwrap(); - backend.stop(&id).await.unwrap(); - backend.stop(&id).await.unwrap(); - backend - .remove(RemoveRequest::from_resources(created.created().to_vec()).unwrap()) - .await - .unwrap(); - assert!(backend.inspect(&id).await.unwrap().is_none()); + backend_contract(&backend, &fixture).await; assert!( !backend .list_resources() diff --git a/tests/ci/expected-ignored-tests.txt b/tests/ci/expected-ignored-tests.txt index d086f64..bf86b37 100644 --- a/tests/ci/expected-ignored-tests.txt +++ b/tests/ci/expected-ignored-tests.txt @@ -5,7 +5,7 @@ attach::attach_preserves_binary_streams_and_exact_exit_codes attach::attached_process_forwards_sigint_and_closes_stdin attach::attached_process_reports_resize_signal_and_exit attach::unsupported_signal_matrix_returns_promptly -backend_contract::backend_contract +backend_contract::backend_contract_holds_on_apple changed_setup_is_reported_but_not_run_by_up_or_shell cli_lifecycle_survives_daemon_and_host_state_changes cli_recovers_from_stale_daemon_metadata_and_runtime_truth From 049b4ba1c619f8495fe17882e0fc17fc6bb39148 Mon Sep 17 00:00:00 2001 From: Richard Kiene Date: Thu, 20 Aug 2026 14:13:14 -0700 Subject: [PATCH 05/13] test: arca meets the shared backend contract, and here is what it did It fails, at the same assertion apple fails, with a third answer. Run on host `newcombe`, 2026-08-20, against the engine `engine/arca-pin.json` pins: revision c545612b056e028d5885968a7b9f586d694f994c, tag gascan-engine-m4. GASCAN_ARCA_ENGINE_BIN=.../arca-engine \ GASCAN_ARCA_KERNEL_PATH="$HOME/Library/Application Support/dev.gascan/engine/vmlinux" \ GASCAN_ARCA_VMINIT_LAYOUT="$HOME/Library/Application Support/dev.gascan/engine/vminit" \ GASCAN_ARCA_BASE_OCI_LAYOUT=/tmp/alpine-oci \ cargo test -p gascan-arca --test live -- --ignored --test-threads=1 \ --nocapture backend_contract_holds_on_arca Exit 101. Full output: running 1 test test conformance::backend_contract_holds_on_arca ... thread 'conformance::backend_contract_holds_on_arca' (14162575) panicked at crates/gascan-conformance/src/lib.rs:104:5: assertion `left == right` failed left: Creating right: Stopped note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace FAILED failures: conformance::backend_contract_holds_on_arca test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 28 filtered out; finished in 1.16s The failing assertion is the post-`create` state assertion at `crates/gascan-conformance/src/lib.rs:104` -- the same one apple fails (`0e1f3fb`). Everything the contract does after it -- `start`, `exec`, `stop`, `remove`, and the final absent-`inspect` -- was NOT REACHED. Reproducible: 4 runs on this host on 2026-08-20, all four `Creating` vs `Stopped` at `lib.rs:104` (1.30s, 1.13s, 1.00s, 1.16s). **`Creating` is terminal here, not a transient the contract read too early.** MEASURED with a throwaway probe (created, then polled `inspect` every 200ms for 30s, printing only on change; not committed): the sole transition printed was the first read, `PROBE 0.00s state=Some(Creating)`, and `PROBE final after 30s: Some(Creating)`. The state never moved. So the failure is not a missing wait. That makes three backends and three different post-`create` states for the same compiled request: `FakeRuntime` reports `Stopped`, apple reports `Running` because its `create` emits `container run` (`gascan-apple/src/translate.rs:96`), and arca reports `Creating`. The assertion encodes the fake's answer and neither real backend agrees with it, nor with the other. Recorded as an observation; the contract is deliberately left unchanged, because a suite edited until it passes measures nothing. The test's own image is `engine.image(TAG)` and not the bare tag the plan's sketch used. MEASURED on this host, with the bare tag: the run panicked at `gascan-conformance/src/lib.rs:57` with `compile backend-contract policy: InvalidWorkspaceImage` in 0.66s, before any call reached the backend -- `PolicyCompiler::compile_for_image` refuses a mutable reference (`gascan-core/src/policy.rs:179`). `LiveEngine::image` supplies the store's `repository@sha256:...`, which is what the tier's own `policy_request_from_manifest` passes. `tests/ci/expected-ignored-tests.txt` goes 49 -> 50 lines. The guard was run before the baseline edit and FAILED exit 1 naming the added entry, then run after it and passed: `ci-check-ignored-tests: 50 ignored test(s), matching the baseline`, exit 0. --- Cargo.lock | 1 + crates/gascan-arca/Cargo.toml | 1 + crates/gascan-arca/tests/live.rs | 2 + crates/gascan-arca/tests/live/conformance.rs | 44 ++++++++++++++++++++ tests/ci/expected-ignored-tests.txt | 1 + 5 files changed, 49 insertions(+) create mode 100644 crates/gascan-arca/tests/live/conformance.rs diff --git a/Cargo.lock b/Cargo.lock index 0537087..a2b7c93 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -503,6 +503,7 @@ version = "0.1.0" dependencies = [ "async-trait", "camino", + "gascan-conformance", "gascan-core", "gascan-engine-proto", "gascan-oci-fixture", diff --git a/crates/gascan-arca/Cargo.toml b/crates/gascan-arca/Cargo.toml index 20049c9..7a82fa1 100644 --- a/crates/gascan-arca/Cargo.toml +++ b/crates/gascan-arca/Cargo.toml @@ -19,6 +19,7 @@ tower.workspace = true [dev-dependencies] camino.workspace = true +gascan-conformance = { path = "../gascan-conformance" } gascan-oci-fixture.workspace = true rustix.workspace = true serde_json.workspace = true diff --git a/crates/gascan-arca/tests/live.rs b/crates/gascan-arca/tests/live.rs index edbdb57..11eca86 100644 --- a/crates/gascan-arca/tests/live.rs +++ b/crates/gascan-arca/tests/live.rs @@ -1,5 +1,7 @@ #[path = "live/common/mod.rs"] mod common; +#[path = "live/conformance.rs"] +mod conformance; #[path = "live/connect.rs"] mod connect; #[path = "live/exec.rs"] diff --git a/crates/gascan-arca/tests/live/conformance.rs b/crates/gascan-arca/tests/live/conformance.rs new file mode 100644 index 0000000..e65e0f9 --- /dev/null +++ b/crates/gascan-arca/tests/live/conformance.rs @@ -0,0 +1,44 @@ +use crate::common::{LiveEngine, base_oci_layout, layout_running}; +use camino::Utf8Path; +use gascan_arca::ArcaBackend; +use gascan_conformance::{CreateRequestFixture, backend_contract}; + +/// The tag the derived layout is loaded under. +const TAG: &str = "gascan-conformance:latest"; + +/// `user = 'root'` because the base layout is a stock alpine with no +/// `workspace` account -- see `lifecycle.rs`'s note on the same constant. +/// +/// `network = 'networked'` and not `'offline'`: offline is the one capability +/// this engine is known NOT to honour +/// (`docs/evidence/2026-08-18-arca-engine-offline.md`), so an offline request +/// would test the refuted property by accident. +const MANIFEST: &str = "version = 1\nnetwork = 'networked'\nuser = 'root'\n"; + +/// The shared backend contract, run against a real `arca-engine`. +/// +/// **The image is `engine.image(TAG)` and not `TAG`**, which is why the engine +/// is started before the fixture is built. `PolicyCompiler::compile_for_image` +/// refuses a mutable reference outright (`gascan-core/src/policy.rs:179`), and +/// `LiveEngine::image` is what turns the seeded tag into the store's own +/// `repository@sha256:...`. MEASURED, on `newcombe` 2026-08-20 with the bare +/// tag: the test panicked at `gascan-conformance/src/lib.rs:57` with +/// `compile backend-contract policy: InvalidWorkspaceImage`, in 0.66s -- before +/// a single call reached the backend. A fixture that cannot be built measures +/// nothing about arca. +#[tokio::test] +#[ignore = "requires a built arca-engine, a kernel, a vminit layout and a base OCI layout"] +async fn backend_contract_holds_on_arca() { + let temp = tempfile::tempdir().expect("a temporary layout root"); + let destination = Utf8Path::from_path(temp.path()).expect("a utf-8 temporary path"); + let layout = layout_running( + &base_oci_layout(), + destination, + TAG, + &["sh", "-c", "while :; do sleep 1; done"], + ); + let engine = LiveEngine::start_with_images(&[layout.as_path()]).await; + let backend = ArcaBackend::new(engine.transport().await); + let fixture = CreateRequestFixture::for_image("conformance", &engine.image(TAG), MANIFEST); + backend_contract(&backend, &fixture).await; +} diff --git a/tests/ci/expected-ignored-tests.txt b/tests/ci/expected-ignored-tests.txt index bf86b37..d418326 100644 --- a/tests/ci/expected-ignored-tests.txt +++ b/tests/ci/expected-ignored-tests.txt @@ -9,6 +9,7 @@ backend_contract::backend_contract_holds_on_apple changed_setup_is_reported_but_not_run_by_up_or_shell cli_lifecycle_survives_daemon_and_host_state_changes cli_recovers_from_stale_daemon_metadata_and_runtime_truth +conformance::backend_contract_holds_on_arca connect::a_call_against_a_killed_engine_fails_rather_than_hanging connect::a_real_engine_accepts_the_placeholder_authority developer_configuration_persists_across_restart_and_image_replacement From 766eb6eac8fc70ccb4ff94e8a76edd3ca0d155ac Mon Sep 17 00:00:00 2001 From: Richard Kiene Date: Thu, 20 Aug 2026 14:29:44 -0700 Subject: [PATCH 06/13] test: correct two claims in 049b4ba, and stop excepting this test from kill() Fix round 1 on Task 5's review. `049b4ba` is not amended: it is cited by the review document and the plan's ledger, and a correcting commit is the honest evidence that the correction happened. No live test was re-run in this round; the measurement recorded in `049b4ba` stands unchanged. CORRECTION 1. `049b4ba` says the three backends give three post-`create` states "for the same compiled request". **That is false.** The requests differ in four inputs, each forced by the backend's own store and base image: apple's is `CreateRequestFixture::pinned(&name, "offline")` (`crates/gascan-apple/tests/live/backend_contract.rs:14`) -- pinned approved workspace image, `network = offline`, default `workspace` user, nonce-suffixed name. Arca's is `for_image("conformance", &engine.image(TAG), MANIFEST)` (`crates/gascan-arca/tests/live/conformance.rs:48`) -- store digest reference, `network = networked`, `user = root`, fixed name. What is held constant is the CONTRACT (`crates/gascan-conformance/src/lib.rs:94-124`), not the request. The finding survives; the sentence claimed a controlled comparison that was not one, and would have had the next person rule the request out as a variable. CORRECTION 2. `049b4ba` says "`Creating` is terminal here" and that arca's `create` leaves the sandbox in `Creating` "indefinitely". **Both overclaim.** `Creating` is not terminal -- `start` leaves it in under two seconds, which is what the positive control does whenever it passes (`finished in 1.99s`, exit 0, `.superpowers/sdd/2026-08-20-backend-conformance-suite/arca-positive-control.log`), and "indefinitely" extrapolated past the 30s observed. The defensible claim is narrower and carries the whole finding: **`create` performs no autonomous state transition; the sandbox sits in the engine's `created` status until something starts it.** That claim is now anchored to the pinned engine's own source rather than to a deleted probe. Verified here: `git -C .artifacts/arca-engine/arca rev-parse HEAD` -> c545612b056e028d5885968a7b9f586d694f994c, matching `engine/arca-pin.json:6`, with only the generated `Sources/ContainerBridge/BuildInfo.generated.swift` modified. At that revision `Sources/ArcaEngine/EngineTranslation.swift:127-134` maps `case "created": return .creating` (`:129`). It is a total function of the current status string with no time term, so no wait changes what it returns -- decisive by construction. The probe is demoted to corroboration: it was never committed and its source is quoted nowhere, so its single line of output is indistinguishable from what a broken probe would print. CODE CHANGE. `engine.kill().await` is now the last statement of the test, with a comment explaining it, following `connect.rs:96-103`'s shape for the tier's one other exception. The earlier rationale for omitting it -- that `kill()` adds an unrelated failure mode -- is withdrawn; it is exactly the argument `crates/gascan-arca/tests/live/common/mod.rs:473-477` rejects ("This assertion is what stops that regressing, and it is here rather than only in `shutdown.rs` because every test in this tier stops an engine"), guarding an abort measured at 6 crashes in 192 runs before the engine fix (`:462-471`). **The added line does not execute today and changes no recorded result.** `backend_contract` panics at `crates/gascan-conformance/src/lib.rs:104`, so control never reaches the next statement, and no statement before the contract call is touched by this diff. That is an argument from unreachability, not a re-measurement: the live test was not re-run. Also corrected, a wrong anchor in `049b4ba`: apple's `create` emits `container run` at `crates/gascan-apple/src/translate.rs:100`, inside the `create` that opens at `:94`. `:96`, which `049b4ba` cites, is `validate_view(&view)?;`. The claim was true; the pointer was not. Also added: a comment on the staying-up `Cmd` (the one deliberate difference from apple that carried none), pointing at `lifecycle.rs:33-53`. Verified after the edit on `newcombe`, 2026-08-20: `cargo test -p gascan-arca --test live --no-run` exit 0; `cargo fmt --all --check` exit 0; `cargo clippy --workspace --all-targets -- -D warnings` exit 0. The ignored-test baseline is unchanged at 50 lines; this round adds and removes no test, and the guard was not re-run. --- crates/gascan-arca/tests/live/conformance.rs | 28 ++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/crates/gascan-arca/tests/live/conformance.rs b/crates/gascan-arca/tests/live/conformance.rs index e65e0f9..2faedc8 100644 --- a/crates/gascan-arca/tests/live/conformance.rs +++ b/crates/gascan-arca/tests/live/conformance.rs @@ -31,6 +31,12 @@ const MANIFEST: &str = "version = 1\nnetwork = 'networked'\nuser = 'root'\n"; async fn backend_contract_holds_on_arca() { let temp = tempfile::tempdir().expect("a temporary layout root"); let destination = Utf8Path::from_path(temp.path()).expect("a utf-8 temporary path"); + // `sh -c 'while :; do sleep 1; done'` and not the base image's own `Cmd`: + // alpine's is `/bin/sh`, which exits immediately with no tty attached, and + // the contract does start -> exec -> stop, so the container has to still be + // there. `lifecycle.rs:33-53` carries the measured note on this exact `Cmd`. + // It does not matter yet -- the contract fails before `start` -- and it + // starts mattering the day it gets that far. let layout = layout_running( &base_oci_layout(), destination, @@ -41,4 +47,26 @@ async fn backend_contract_holds_on_arca() { let backend = ArcaBackend::new(engine.transport().await); let fixture = CreateRequestFixture::for_image("conformance", &engine.image(TAG), MANIFEST); backend_contract(&backend, &fixture).await; + + // `kill()` and not a bare drop, matching every other terminating test in + // this tier, because its exit-status assertion is deliberately spread + // across all of them: "This assertion is what stops that regressing, and it + // is here rather than only in `shutdown.rs` because every test in this tier + // stops an engine" (`common/mod.rs:473-477`), guarding an abort that ran at + // 6 crashes in 192 runs before the engine fix (`:462-471`). + // + // **It does not execute today**, and that is not a reason to leave it out. + // `backend_contract` panics at `gascan-conformance/src/lib.rs:104`, so this + // line is unreachable until arca's post-`create` state stops being + // `Creating`. What it buys is that the day the contract gets past that + // assertion, this test is already inside the tier's shutdown guard rather + // than a silent exception to it -- and `kill()` is also the only thing that + // prints the engine's own drained stdout/stderr (`exit.diagnostics`, + // `common/mod.rs:493`). + // + // The cost, stated plainly: while the contract fails where it does, this + // run discards the engine's account of the `create` it is measuring. + // Recovering it on the red path would mean catching the panic around the + // contract call, which is a restructuring this test does not justify. + engine.kill().await; } From 22dfee82636c8a4a821a6af6d70bd02be2f18a4f Mon Sep 17 00:00:00 2001 From: Richard Kiene Date: Thu, 20 Aug 2026 15:50:34 -0700 Subject: [PATCH 07/13] test: start and stop are idempotent on every backend, not just the fake MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `duplicate_create_is_rejected_and_start_stop_are_idempotent` in crates/gascan-core/tests/backend_contract.rs held three assertions that any `RuntimeBackend` owes, stated against `FakeRuntime` alone. All three move into `backend_contract`: the doubled `start`, the doubled `stop`, and a second `create` of a held id failing with `resource_conflict`. Duplicate-create promoted too, against the brief's default of leaving it behind, because it clears the design's §4 bar. It reaches no fake-only machinery -- no `calls()`, `outcomes()`, `seed_*`, `FailureBoundary`, or `persistent` -- and each backend detects the collision its own way: the fake by map lookup (crates/gascan-core/src/fake_runtime.rs:695), apple by a pre-flight inventory scan (crates/gascan-apple/src/backend.rs:238), arca engine-side, its wire code mapped at crates/gascan-arca/src/error.rs:49. Only the stable code is portable, so only the stable code is asserted. Because both halves promoted, the whole test function goes rather than half of it: `cargo test -p gascan-core` was 174 passed / 15 suites before and is 173 passed / 15 suites after, both exit 0. These assertions are exercised by `FakeRuntime` only today. Apple and arca still fail the contract at the post-`create` state assertion (crates/gascan-conformance/src/lib.rs:104), which precedes every line added here, so neither reaches a doubled `start`, a doubled `stop`, or the second `create`. Task 4 recorded apple's idempotence as asserted nowhere until a later task promoted it; this is that task, and the assertion now exists in the shared contract for all three backends -- but no idempotence or duplicate-create behaviour has been measured on apple or arca. No live test was run. Each promoted assertion was mutation-checked against `FakeRuntime`, the file `touch`ed first so the run recompiled rather than serving a cached artifact (each failing run's output carries `Compiling gascan-core v0.1.20`): - `start` rejecting a second call -> exit 101, panic at lib.rs:116:30, the second `backend.start(&id)`. - `stop` rejecting a second call -> exit 101, panic at lib.rs:127:29, the second `backend.stop(&id)`. - the fake's duplicate-id conflict check deleted -> exit 101, panic at lib.rs:109:49, `unwrap_err()` on an `Ok` value. Every mutation was reverted and crates/gascan-core/src/fake_runtime.rs is byte-identical to its pre-mutation state -- sha256 14c289767b0440c99c9b0ab0e4ff82a315dba73f6dc05473123fb4b24186e827 before and after, `cmp` against a pristine copy clean, and the file is absent from this commit. `cargo test -p gascan-conformance --test fake` exits 0 after the revert. `cargo fmt --all --check` exit 0, `cargo clippy --workspace --all-targets -- -D warnings` exit 0, `./scripts/ci-check-ignored-tests.sh` exit 0 at 50 ignored tests, unchanged. --- crates/gascan-conformance/src/lib.rs | 13 +++++++++++++ crates/gascan-core/tests/backend_contract.rs | 15 --------------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/crates/gascan-conformance/src/lib.rs b/crates/gascan-conformance/src/lib.rs index 535e296..796d41b 100644 --- a/crates/gascan-conformance/src/lib.rs +++ b/crates/gascan-conformance/src/lib.rs @@ -105,6 +105,18 @@ pub async fn backend_contract(backend: &dyn RuntimeBackend, fixture: &CreateRequ backend.inspect(&id).await.unwrap().unwrap().state, ContainerState::Stopped ); + // A second `create` of an id the backend already holds is a conflict, not a + // silent re-create. Each backend detects this its own way -- the fake by + // map lookup, apple by pre-flight inventory scan, arca engine-side -- so the + // stable code is the only portable thing to assert. + assert_eq!( + backend.create(fixture.request()).await.unwrap_err().code(), + "resource_conflict" + ); + // Doubled deliberately: `start` and `stop` are idempotent, so the second + // call of each must succeed and not report the sandbox's current state as + // an error. Collapsing either pair deletes the assertion. + backend.start(&id).await.unwrap(); backend.start(&id).await.unwrap(); let mut session = backend .exec(ExecRequest::fixture(id.clone(), ["true"])) @@ -116,6 +128,7 @@ pub async fn backend_contract(backend: &dyn RuntimeBackend, fixture: &CreateRequ ExecOutput::Exit { code: 0, signal: 0 } ); backend.stop(&id).await.unwrap(); + backend.stop(&id).await.unwrap(); backend .remove(RemoveRequest::from_resources(created.created().to_vec()).unwrap()) .await diff --git a/crates/gascan-core/tests/backend_contract.rs b/crates/gascan-core/tests/backend_contract.rs index cb256ea..472143b 100644 --- a/crates/gascan-core/tests/backend_contract.rs +++ b/crates/gascan-core/tests/backend_contract.rs @@ -624,21 +624,6 @@ fn validated_fixture_keeps_its_canonical_bind_source_alive() { assert!(fixture.bind_mounts()[0].source.exists()); } -#[tokio::test] -async fn duplicate_create_is_rejected_and_start_stop_are_idempotent() { - let backend = FakeRuntime::new(capabilities()); - let fixture = create_request("lifecycle"); - let id = fixture.id().clone(); - backend.create(fixture.request()).await.unwrap(); - let error = backend.create(fixture.request()).await.unwrap_err(); - assert_eq!(error.code(), "resource_conflict"); - - backend.start(&id).await.unwrap(); - backend.start(&id).await.unwrap(); - backend.stop(&id).await.unwrap(); - backend.stop(&id).await.unwrap(); -} - #[tokio::test] async fn inventory_keeps_unowned_resources_observable() { let backend = FakeRuntime::new(capabilities()); From 9f2c0bffd476d176e2b5d494ef7e393ca9f15107 Mon Sep 17 00:00:00 2001 From: Richard Kiene Date: Thu, 20 Aug 2026 16:07:35 -0700 Subject: [PATCH 08/13] test: clean up a rejected create's residue, and correct 22dfee8's line anchors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two review fixes on top of 22dfee8. No assertion was weakened, added, or reordered; the post-`create` state assertion at crates/gascan-conformance/src/lib.rs:104-107 is untouched, and crates/gascan-core/src/fake_runtime.rs is still sha256 14c289767b0440c99c9b0ab0e4ff82a315dba73f6dc05473123fb4b24186e827. I-1, a record correction. Every line anchor in 22dfee8's message is four lower than the tree that commit ships, because the four-line comment block at lib.rs:108-111 was added after the mutation cycles ran and the numbers were recorded during them. The runs and their panics are exactly as reported; only the pointers were stale. Anchored three ways: assertion during the runs in 22dfee8 in this commit second `start` 116:30 120:30 136 second `stop` 127:29 131:29 147 `unwrap_err()` 109:49 113:49 112 Verified by `git show 22dfee8:crates/gascan-conformance/src/lib.rs | grep -n`, which puts `unwrap_err` at 113 and the doubled calls at 119/120 and 130/131. The "in this commit" column names the call sites only -- no mutation was run against this tree, so no panic is claimed at those lines. `git show 22dfee8:crates/gascan-conformance/src/lib.rs` is the durable way to read the middle column. I-2, a latent leak closed. The promoted duplicate-`create` assertion kept only `.code()` and dropped the `CreateFailure`, which carries whatever resources the rejected create had already built; the walk's `remove` covers the first create's resources only. Nothing was observed leaking -- this prevents a leak rather than fixing one. It is a no-op on all three backends today: the fake's map lookup (crates/gascan-core/src/fake_runtime.rs:695) and apple's pre-flight inventory scan (crates/gascan-apple/src/backend.rs:238-252) both fire before a single resource is made, and arca never reaches the line. It exists for live engines, where a conflicting create has been measured reporting the three volumes it had made (crates/gascan-arca/tests/live/lifecycle.rs:275-283). The failure is now bound and its `created()` removed when non-empty. Nothing is asserted about that list. `is_empty()` would be a new portability claim, and arca's own live suite predicts arca would fail it -- asserting something we expect to fail and have not measured is the opposite of what this branch is for. Because removal is cleanup and not a claim, there is no assertion here for a spec §6 mutation to falsify, so no mutation proof accompanies this change; the three proofs covering the promoted assertions stand unchanged from 22dfee8. `cargo test -p gascan-conformance --test fake` exit 0, 1 passed. `cargo test -p gascan-core` exit 0, 173 passed across 15 suites, and its backend_contract suite 33 passed -- both unchanged from 22dfee8, as expected for a change confined to gascan-conformance. `cargo fmt --all --check` exit 0. `cargo clippy --workspace --all-targets -- -D warnings` exit 0. No `#[ignore]` added or removed: `git diff | grep -c ignore` is 0, so the 50-line baseline is untouched. No live test was run. --- crates/gascan-conformance/src/lib.rs | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/crates/gascan-conformance/src/lib.rs b/crates/gascan-conformance/src/lib.rs index 796d41b..982d3f2 100644 --- a/crates/gascan-conformance/src/lib.rs +++ b/crates/gascan-conformance/src/lib.rs @@ -109,10 +109,26 @@ pub async fn backend_contract(backend: &dyn RuntimeBackend, fixture: &CreateRequ // silent re-create. Each backend detects this its own way -- the fake by // map lookup, apple by pre-flight inventory scan, arca engine-side -- so the // stable code is the only portable thing to assert. - assert_eq!( - backend.create(fixture.request()).await.unwrap_err().code(), - "resource_conflict" - ); + let conflict = backend.create(fixture.request()).await.unwrap_err(); + assert_eq!(conflict.code(), "resource_conflict"); + // Cleanup, not a claim. A rejected `create` may still have built resources + // before it hit the collision, and the `remove` at the end of this walk + // knows only about the first create's. Nothing is asserted about what the + // list holds: `is_empty()` would be a portability claim, and arca's own live + // suite predicts arca would fail it. + // + // No-op on every backend today -- the fake's map lookup + // (`gascan-core/src/fake_runtime.rs:695`) and apple's pre-flight inventory + // scan (`gascan-apple/src/backend.rs:238-252`) both fire before a single + // resource is made, and arca does not reach this line. It is here for live + // engines, where a conflicting create has been measured reporting the three + // volumes it had made (`gascan-arca/tests/live/lifecycle.rs:275-283`). + if !conflict.created().is_empty() { + backend + .remove(RemoveRequest::from_resources(conflict.created().to_vec()).unwrap()) + .await + .unwrap(); + } // Doubled deliberately: `start` and `stop` are idempotent, so the second // call of each must succeed and not report the sandbox's current state as // an error. Collapsing either pair deletes the assertion. From e7e55e4c5cd7a7740c05d84f730c30fedef1c399 Mon Sep 17 00:00:00 2001 From: Richard Kiene Date: Thu, 20 Aug 2026 16:18:08 -0700 Subject: [PATCH 09/13] test: drop 9f2c0bf's residue cleanup, which would have torn down the sandbox Reverses the `if !conflict.created().is_empty() { ... remove ... }` block added in 9f2c0bf. The review finding it answered, and the ruling that directed it, both rested on a false premise: that `conflict.created()` names orphaned residue. It cannot. Both creates in this walk pass the same `fixture.request()` -- crates/gascan-conformance/src/lib.rs:97 and :112 -- so the same sandbox id and the same resource names. Anything the rejected create reported would therefore carry the names of the live sandbox the walk still has to `start`, `exec`, `stop` and `remove`, and removing it would have torn that sandbox down mid-walk and double-removed its container and managed network at the closing `RemoveRequest::from_resources(created.created().to_vec())`. The live measurement the premise leaned on does not transfer. crates/gascan-arca/tests/live/lifecycle.rs:259-278 removes the container and the three volumes first, keeping only the network name held, and only then does the conflicting create rebuild those three volumes and fail -- so there they are genuinely orphaned. Here "residue" and "the sandbox under test" are the same names, so what is cleanup in that scenario is a teardown in this one. Nothing was observed misbehaving. The branch never fired on any backend: the fake's map lookup and apple's pre-flight inventory scan both reject the duplicate before building anything, and apple and arca do not reach this line at all. This removes latent wrong code rather than fixing an observed failure. Two latent surfaces go with it -- the mid-walk teardown and double-remove above, and the `.unwrap()` on that removal, which on a live engine reporting residue it had already rolled back would have unwrapped a remove of something absent, whose error status is backend-defined. No narrower cleanup replaces it. Separating "residue" from "the sandbox under test" would need a distinction nobody has measured on any live backend. In its place the code records the open question with its anchor, at lib.rs:114-123. The assertion keeps 9f2c0bf's bound form -- `let conflict = ...; assert_eq!(conflict.code(), "resource_conflict");` -- rather than folding back to the chained expression 22dfee8 shipped. It is two lines instead of four, and the comment that replaces the removed block is about `conflict.created()`, so the value it discusses should be named in the code beside it. 9f2c0bf's I-1 anchor corrections stand unchanged. Its comment cited `gascan-apple/src/backend.rs:238-252` for apple's pre-flight scan; that text is deleted here, so the correction is moot, but for the record `:252` is the closing brace of the conflict block and `:253` is `let mut created = Vec::new();` -- verified by numbering the file -- so the range as written ended correctly and it is `:253`, not `:252`, that is the first line after the scan. `cargo test -p gascan-conformance --test fake` exit 0, 1 passed. `cargo test -p gascan-core` exit 0, 173 passed across 15 suites, backend_contract 33 passed -- both unchanged from 22dfee8 and 9f2c0bf. `cargo fmt --all --check` exit 0. `cargo clippy --workspace --all-targets -- -D warnings` exit 0. crates/gascan-core/src/fake_runtime.rs is still sha256 14c289767b0440c99c9b0ab0e4ff82a315dba73f6dc05473123fb4b24186e827 and absent from this commit. No `#[ignore]` added or removed. No live test was run. This round removes code and adds no assertion, so no mutation proof is owed. --- crates/gascan-conformance/src/lib.rs | 29 +++++++++++----------------- 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/crates/gascan-conformance/src/lib.rs b/crates/gascan-conformance/src/lib.rs index 982d3f2..de332f1 100644 --- a/crates/gascan-conformance/src/lib.rs +++ b/crates/gascan-conformance/src/lib.rs @@ -111,24 +111,17 @@ pub async fn backend_contract(backend: &dyn RuntimeBackend, fixture: &CreateRequ // stable code is the only portable thing to assert. let conflict = backend.create(fixture.request()).await.unwrap_err(); assert_eq!(conflict.code(), "resource_conflict"); - // Cleanup, not a claim. A rejected `create` may still have built resources - // before it hit the collision, and the `remove` at the end of this walk - // knows only about the first create's. Nothing is asserted about what the - // list holds: `is_empty()` would be a portability claim, and arca's own live - // suite predicts arca would fail it. - // - // No-op on every backend today -- the fake's map lookup - // (`gascan-core/src/fake_runtime.rs:695`) and apple's pre-flight inventory - // scan (`gascan-apple/src/backend.rs:238-252`) both fire before a single - // resource is made, and arca does not reach this line. It is here for live - // engines, where a conflicting create has been measured reporting the three - // volumes it had made (`gascan-arca/tests/live/lifecycle.rs:275-283`). - if !conflict.created().is_empty() { - backend - .remove(RemoveRequest::from_resources(conflict.created().to_vec()).unwrap()) - .await - .unwrap(); - } + // `conflict.created()` is deliberately neither removed nor asserted about. + // A rejected `create` may report resources it built before the collision, + // but both creates here use the same request, so those names are the live + // sandbox's own -- removing them would tear down the sandbox this walk still + // has to start, exec, stop and remove. The one live measurement of a + // conflicting create reporting what it made + // (`gascan-arca/tests/live/lifecycle.rs:259-278`) does not settle this case: + // there the container and volumes had been removed first and only the + // network name was still held, so its three volumes were genuinely + // orphaned. What a same-request collision reports is unmeasured everywhere. + // Doubled deliberately: `start` and `stop` are idempotent, so the second // call of each must succeed and not report the sandbox's current state as // an error. Collapsing either pair deletes the assertion. From a32a29e802f9b673efba509ae53de78eee1006bd Mon Sep 17 00:00:00 2001 From: Richard Kiene Date: Thu, 20 Aug 2026 17:33:55 -0700 Subject: [PATCH 10/13] test: an exec session's terminal Exit ends the stream, on every backend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Promotes one assertion out of `exec_session_is_live_bidirectional_and_emits_one_exit` in crates/gascan-core/tests/backend_contract.rs into `backend_contract`: after the terminal `ExecOutput::Exit`, the session yields nothing further. It is now crates/gascan-conformance/src/lib.rs:148 and the duplicated half is gone from gascan-core. It clears the design's §4 bar on both halves. Each backend closes the stream in its own code -- the fake's spawned task returns after the `Exit` send so `outputs` drops (crates/gascan-core/src/fake_runtime.rs:1123-1127), apple breaks on `terminal` after mapping `AttachOutput::Exit` (crates/gascan-apple/src/backend.rs:604 and :614-616), arca breaks on the same flag over engine frames (crates/gascan-arca/src/backend.rs:368-374 and :387-389). And it is load-bearing rather than incidental: gascand drains to the end of the stream in two places, crates/gascand/src/service.rs:2336 and crates/gascand/src/ssh/manager.rs:700, each with a second drain after `session.cancel()` (:2341 and :707). A backend that kept the stream open after `Exit` hangs all four forever and nothing else in the suite would notice. It reaches no fake-only machinery: no `calls()`, `outcomes()`, `seed_*`, `FailureBoundary`, or `FakeRuntime::persistent`. The rest of that test does not promote and stays. The stdin echo needs the fake's own `fake-echo-stdin` (fake_runtime.rs:596), which no container image has, and `Exit { code: 143, signal: 15 }` is the fake's `128 + signal` arithmetic (fake_runtime.rs:1118-1122) -- apple hardcodes `signal: 0` on every exit (gascan-apple/src/backend.rs:604) and arca passes the engine's `exit.signal` through (gascan-arca/src/backend.rs:370-373), so that is three behaviours rather than one contract. The test is renamed `exec_session_echoes_stdin_and_maps_a_signal_to_its_exit_code` because "emits_one_exit" was the name of the assertion that left, and a doc comment says where it went. This assertion is exercised by `FakeRuntime` only today. Apple and arca still fail the contract at the post-`create` state assertion (crates/gascan-conformance/src/lib.rs:104) -- apple reports `Running`, arca `Creating` -- which precedes the exec entirely, so neither backend reaches it and no stream-termination behaviour has been measured on either. The three-backend evidence above is code read statically, not a measurement. No live test was run. Mutation-checked against `FakeRuntime`, the file `touch`ed first so the run recompiled rather than serving a cached artifact. Mutation: a second `send_fake_exec_output(.., ExecOutput::Exit { code, signal })` appended after the terminal one, so the stream no longer ends there. `cargo test -p gascan-conformance` exited 101, its output carrying `Compiling gascan-core v0.1.20`, and it panicked at crates/gascan-conformance/src/lib.rs:148:5, `assertion failed: session.next().await.is_none()`. The comment block above the assertion was written before the cycle ran, not after, so that anchor is the line this commit ships. The mutation was reverted and crates/gascan-core/src/fake_runtime.rs is byte-identical -- sha256 14c289767b0440c99c9b0ab0e4ff82a315dba73f6dc05473123fb4b24186e827 before and after, `cmp` against a pristine copy exit 0, and the file is absent from this commit. `cargo test -p gascan-conformance` exit 0, 1 passed. `cargo test -p gascan-core` exit 0, 173 passed across 15 suites -- unchanged from e7e55e4, because this moves an assertion rather than a test function. `cargo fmt --all --check` exit 0. No `#[ignore]` added or removed: the diff contains no `#[ignore]` line in either direction, and the renamed test carries none and appears nowhere in tests/ci/expected-ignored-tests.txt. --- crates/gascan-conformance/src/lib.rs | 10 ++++++++++ crates/gascan-core/tests/backend_contract.rs | 9 +++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/crates/gascan-conformance/src/lib.rs b/crates/gascan-conformance/src/lib.rs index de332f1..a67e26e 100644 --- a/crates/gascan-conformance/src/lib.rs +++ b/crates/gascan-conformance/src/lib.rs @@ -136,6 +136,16 @@ pub async fn backend_contract(backend: &dyn RuntimeBackend, fixture: &CreateRequ session.next().await.unwrap().unwrap(), ExecOutput::Exit { code: 0, signal: 0 } ); + // `Exit` is terminal: the stream ends there, and every consumer that drains + // to completion depends on it -- `gascand/src/service.rs:2336` and + // `gascand/src/ssh/manager.rs:700` both loop `while let Some(..) = + // session.next().await` and would hang forever against a backend that kept + // the stream open. Each backend closes it in its own code: the fake by its + // spawned task returning after the `Exit` send + // (`gascan-core/src/fake_runtime.rs:1123`), apple by breaking on `terminal` + // so the sender drops (`gascan-apple/src/backend.rs:614`), arca by the same + // break over engine frames (`gascan-arca/src/backend.rs:387`). + assert!(session.next().await.is_none()); backend.stop(&id).await.unwrap(); backend.stop(&id).await.unwrap(); backend diff --git a/crates/gascan-core/tests/backend_contract.rs b/crates/gascan-core/tests/backend_contract.rs index 472143b..420a030 100644 --- a/crates/gascan-core/tests/backend_contract.rs +++ b/crates/gascan-core/tests/backend_contract.rs @@ -55,8 +55,14 @@ async fn fake_runtime_records_cancelled_exec_before_terminal_status() { assert_eq!(backend.exec_cancellations().await, 1); } +/// The stdin/resize/signal half of exec, which needs the fake's own +/// `fake-echo-stdin` command and its signal-to-exit-code arithmetic. +/// +/// "the stream ends after the terminal `Exit`" used to be asserted here too and +/// is now in `gascan_conformance::backend_contract`, which every backend runs; +/// the name follows the assertions that are left. #[tokio::test] -async fn exec_session_is_live_bidirectional_and_emits_one_exit() { +async fn exec_session_echoes_stdin_and_maps_a_signal_to_its_exit_code() { let backend = FakeRuntime::new(capabilities()); let fixture = create_request("live-exec"); let id = fixture.id().clone(); @@ -90,7 +96,6 @@ async fn exec_session_is_live_bidirectional_and_emits_one_exit() { signal: 15 } ); - assert!(session.next().await.is_none()); } #[tokio::test] From ba458c9b92706bdb89bb3d28b0c3b68ebbc522ab Mon Sep 17 00:00:00 2001 From: Richard Kiene Date: Thu, 20 Aug 2026 17:34:42 -0700 Subject: [PATCH 11/13] docs: what promoted, what did not, and the machinery that decided each MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit §3 loses its "Estimated 6-8 will promote cleanly" and gains the measured outcome. Across Tasks 6 and 7, FOUR assertions promoted: three in 22dfee8 (the doubled `start`, the doubled `stop`, a second `create` of a held id failing with `resource_conflict`) and one in a32a29e (an exec session's terminal `Exit` ends the stream). Counted by candidate rather than by assertion, one of the seven named tests promoted whole, one promoted in part, and five promoted nothing. Both countings fall below 6. Nothing was promoted to move the number; the estimate is corrected instead, beside the note that an earlier draft said 13. A fake-only table now names each remaining candidate and the machinery that decided it: `set_exec_result` and `set_logs`; `seed_volume`; `FakeRuntime::persistent`. Two entries are not blocked by machinery at all but by the contract's shape -- it is one walk over one fixture, and arca's must be `network = 'networked'`, so `offline_fake_create_has_no_managed_network` and the network element of the ordering assertion would each need the contract to branch on the fixture. That is a design change and this task does not make one. Two corrections carried in, both re-derived at a32a29e rather than repeated: - The plan's candidate table says `networked_fake_create_reports_network_then_volumes_then_container` asserts its ordering "through the fake's call recorder". It does not. It reads `outcome.created()` at crates/gascan-core/tests/backend_contract.rs:509-517 and touches neither `calls()` nor `outcomes()`, and `CreateOutcome::new` stores the vec verbatim (crates/gascan-core/src/runtime.rs:778-784). The verdict is unchanged, so the wrong reason would have survived unnoticed. The reasons that do hold are recorded in its place. - §2 now states that `crates/gascan-core/tests/common/mod.rs` does NOT go away and the duplication with `gascan-conformance` is permanent, not "deliberate and short-lived" pending a "Task 9". The plan has eight tasks and there is no Task 9; the false sentence is at .superpowers/sdd/2026-08-20-backend-conformance-suite/task-3-brief.md:21 and was repeated in that task's report, never in this design -- so it is stated correctly here rather than rewritten. MEASURED at a32a29e: `mod common;` appears in exactly one file, backend_contract.rs:1 (policy.rs defines its own `capabilities()` at :19), and that one target still calls `capabilities()` on 20 lines, `create_request(` on 24 and `create_request_with_network(` on 9, so every export survives. Task 3's Step 2 deferred this deletion check to the nonexistent Task 9; it was run here -- `grep -rn "create_request\|capabilities()\|CreateRequestFixture" crates/gascan-core/tests/` returns 63 lines, exit 0 -- and nothing is unreferenced, so common/mod.rs is unchanged. §3's opening count is re-anchored too: 23 `#[tokio::test]` functions was true at 10e3342 and is 21 at a32a29e. Documentation only; no code changes. `cargo fmt --all --check` exit 0. --- ...-08-20-backend-conformance-suite-design.md | 51 +++++++++++++++++-- 1 file changed, 46 insertions(+), 5 deletions(-) diff --git a/docs/superpowers/specs/2026-08-20-backend-conformance-suite-design.md b/docs/superpowers/specs/2026-08-20-backend-conformance-suite-design.md index b6390e1..df7ee44 100644 --- a/docs/superpowers/specs/2026-08-20-backend-conformance-suite-design.md +++ b/docs/superpowers/specs/2026-08-20-backend-conformance-suite-design.md @@ -82,9 +82,23 @@ The fixtures move too. `crates/gascan-core/tests/common/mod.rs` is **61 lines** the contract needs — `capabilities()` (`:27`), `create_request()` (`:40`), `create_request_with_network()` (`:44`). +**`common/mod.rs` does not go away, and the duplication with `gascan-conformance` is permanent.** +Re-derived at HEAD after Task 7: `mod common;` appears in exactly one file, +`crates/gascan-core/tests/backend_contract.rs:1` — `policy.rs` has its own `capabilities()` at +`:19` and does not include it — and that one target still calls `capabilities()` on **20** lines, +`create_request(` on **24** and `create_request_with_network(` on **9**. Every export is still +referenced, so no task in this plan deletes any of them. The accepted cost is two copies of a +three-function fixture; the alternative is the one this section already rejected, since pointing +`gascan-core/tests` at `gascan-conformance` would mint a `gascan-core` dev-dependency on a crate +that depends on `gascan-core`. Drafts of the task briefs called this duplication "deliberate and +short-lived" and deferred it to a "Task 9" — the plan has eight tasks, there is no Task 9, and the +duplication is not short-lived. + ## 3. What gets promoted, and what stays fake-only -`backend_contract.rs` holds **23** `#[tokio::test]` functions. They are not one population. +`backend_contract.rs` held **23** `#[tokio::test]` functions at `10e3342`. They are not one +population. (It holds **21** after Task 7: the fake instantiation moved to `gascan-conformance` and +Task 6 consumed `duplicate_create_is_rejected_and_start_stop_are_idempotent` whole.) **Promote — assertions any backend must satisfy.** The lifecycle walk already in `backend_contract()`, plus `duplicate_create_is_rejected_and_start_stop_are_idempotent`, @@ -93,10 +107,37 @@ the contract needs — `capabilities()` (`:27`), `create_request()` (`:40`), `create_collision_reports_resources_created_before_the_collision`, `offline_fake_create_has_no_managed_network`, `networked_fake_create_reports_network_then_volumes_then_container`, -`persistent_logs_are_isolated_by_exact_sandbox_id`. **Estimated 6–8 will promote cleanly**, and the -estimate is deliberately a range: each one has to be re-read to separate what any backend owes from -what this double happens to do. **An earlier draft of this analysis said 13. That was wrong** — see -§4 for the class of error it made. +`persistent_logs_are_isolated_by_exact_sandbox_id`. Each one had to be re-read to separate what any +backend owes from what this double happens to do. **An earlier draft of this analysis said 13. That +was wrong** — see §4 for the class of error it made. **This section then estimated 6–8. That was +wrong too, and the measured outcome below replaces it** rather than standing beside it. + +### Triage outcome, measured across Tasks 6 and 7 + +**Four assertions promoted, not 6–8.** Counted the other way — by candidate rather than by +assertion — one of the seven named tests promoted whole, one promoted in part, and five promoted +nothing. Both countings are below the estimate, and nothing was promoted to close the gap. + +| where | what promoted | +|---|---| +| Task 6, `22dfee8` | from `duplicate_create_is_rejected_and_start_stop_are_idempotent`: the doubled `start`, the doubled `stop`, and a second `create` of a held id failing with `resource_conflict`. Whole test consumed. | +| Task 7 | from `exec_session_is_live_bidirectional_and_emits_one_exit`: the exec stream ends at the terminal `Exit`. The rest of that test is fake-only and stays, renamed `exec_session_echoes_stdin_and_maps_a_signal_to_its_exit_code` so the name matches the assertions left in it. | + +**Every promoted assertion is exercised by `FakeRuntime` alone today.** Apple and arca both fail the +contract at the post-`create` state assertion (`crates/gascan-conformance/src/lib.rs:104`) — apple +reports `Running`, arca `Creating` — which precedes every line promoted, so neither backend has +been measured against any of it. + +**Stays fake-only — the remaining candidates, and the machinery that decided each.** + +| test | why it stays | +|---|---| +| `exec_and_logs_preserve_binary_bytes_and_exact_exit_code` | `set_exec_result` and `set_logs`, both fake-only. Asserting the property portably needs a command that emits known bytes on stdout *and* stderr and exits non-zero; the fake's vocabulary for that is `fake-stdout` / `fake-stderr` / `fake-exit` (`crates/gascan-core/src/fake_runtime.rs:588-636`), which no container image has, and the fake maps the portable spelling to nothing at all — `Some("true") \| Some("sh") => (Vec::new(), Vec::new(), 0)` at `:633`. Giving the contract a per-backend command means parameterising it, a design change. The exit code the walk *can* portably assert is already asserted. The log half is worse than unportable: `since` is not the same quantity across backends — apple passes `--since {n}ms` to the CLI, a duration ago (`crates/gascan-apple/src/backend.rs:630-632`), arca sends `since_unix_millis`, an absolute instant (`crates/gascan-arca/src/backend.rs:411-414`). | +| `exec_session_is_live_bidirectional_and_emits_one_exit` | **Promoted in part**, see above. What stays needs `fake-echo-stdin` to get stdin back, and its `Exit { code: 143, signal: 15 }` is the fake's own `128 + signal` arithmetic (`crates/gascan-core/src/fake_runtime.rs:1118-1122`), not something a backend owes. | +| `create_collision_reports_resources_created_before_the_collision` | `seed_volume`, fake-only. The assertion's entire content is that a failure reports exactly the resources built before a **planted** collision at a chosen index. Creating twice does produce a collision on a real backend, but with the same request — so the reported names would be the live sandbox's own, and what a same-request collision reports is unmeasured on every backend. `crates/gascan-conformance/src/lib.rs:114-123` records that open question in the contract itself. | +| `offline_fake_create_has_no_managed_network` | Not machinery — fixture shape. The contract is one walk over one fixture, and arca's must be `network = 'networked'`: offline is the capability the pinned engine is proven not to honour (`docs/evidence/2026-08-18-arca-engine-offline.md`). An unconditional "no managed network" assertion fails for a networked fixture on *every* backend, so promoting it needs the contract to branch on the fixture's network. That is a design change, and it is not made here. | +| `networked_fake_create_reports_network_then_volumes_then_container` | Same fixture-conditionality — the network element exists only for a networked fixture — and, separately, **nothing owes the ordering**. `RemoveRequest::from_resources` does not reorder (`crates/gascan-core/src/runtime.rs:1001-1017`), yet the fake's recorded removal comes out container / volume / network, so re-ordering is the backend's job and no consumer reads `created()` positionally. Arca's list is in whatever order the engine's `CreateResponse` carried (`crates/gascan-arca/src/backend.rs:80-108`) — an unmeasured property of a pinned external binary. **Correction to the plan's candidate table**, which says this ordering "is asserted through the fake's call recorder": it is not. The test reads `outcome.created()` (`crates/gascan-core/tests/backend_contract.rs:509-517`) and touches neither `calls()` nor `outcomes()`. The verdict is unchanged; the stated reason was wrong. | +| `persistent_logs_are_isolated_by_exact_sandbox_id` | `FakeRuntime::persistent`, named fake-only machinery, plus `fake-stdout` to get a marker into the log. Isolation-by-id also needs two live sandboxes and `backend_contract` takes one fixture, so promoting it would mean a second design change on top of the machinery. | **Stays fake-only — tests of the double's controllability, not of the contract.** `named_failure_is_injected_once_at_the_call_boundary`, From 99f1449db16b0748e094cf7dd0a5400c35d0bb89 Mon Sep 17 00:00:00 2001 From: Richard Kiene Date: Thu, 20 Aug 2026 18:16:33 -0700 Subject: [PATCH 12/13] docs: what the conformance suite measured on each backend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All three backends report a different state after `create`, and only the test double satisfies the contract. `FakeRuntime` reports `Stopped`, apple `Running` (its `create` compiles to `container run`, translate.rs:100), arca `Creating` (the pinned engine maps status "created" -> .creating). Apple and arca both panic at gascan-conformance/src/lib.rs:104, the walk's third assertion -- so `start`, `exec`, `stop`, `remove` and the closing absent `inspect` were NOT REACHED on either real backend. They are unmeasured, not passing and not failing. docs/evidence/2026-08-20-backend-conformance.md is the durable record: the command and exit code for each backend, the host and date, the engine revision, the four arca reproductions, the positive control run BEFORE the conformance test existed, and the apple residue check. It also carries what this branch surfaced and deliberately did not answer -- what a same-request duplicate `create` reports in `created()`, unmeasured on every backend, with the cleanup that assumed otherwise written in 9f2c0bf and reversed in e7e55e4 -- and the nearest precedent a reader will find for that mistake, apple's tests/live/storage.rs:22-37, which is correct where it is. CORRECTS THE DESIGN'S §6, which claimed the arca instantiation "inherits CI coverage free". CI's live-tier step sets one variable and the tier needs four; `base_oci_layout()` panics rather than skips; apple's tier runs in no CI job at all, line 178 being the file's only `--ignored`. Both real-backend measurements are therefore local-only, on a named machine on a named date, and that is the only evidence for them that will ever exist. The new §6 states this as a derivation and says in terms that no CI run of the arca test has been observed. Also retitles §3's "Promote -- assertions any backend must satisfy" to "Candidates considered": five of the seven listed promoted nothing, and the measured table below it already corrected the substance. START-HERE records P5.3 as executed, points at the evidence, and opens item 10 for the two failures -- naming the assertion, both mechanisms, the three live design candidates, and the instruction not to close it by widening lib.rs:104. It does NOT claim P5's exit is met: the second clause, gascan-e2e on arca, is untouched and was out of scope. VERIFIED at this tree, each command run alone with the machine clear of other cargo jobs: `cargo fmt --all --check` exit 0; `cargo clippy --workspace --all-targets -- -D warnings` exit 0; `cargo test --workspace` exit 0, zero occurrences of FAILED / "error: test failed" / panicked across the whole log, 50 ignored; `./scripts/ci-check-ignored-tests.sh` exit 0, "50 ignored test(s), matching the baseline". `cargo tree -p gascan --edges normal` and the same for gascand return no gascan-conformance, so the crate is reachable from neither shipped binary. An earlier run of the same list failed `gascan --lib` on daemon::tests::inherited_startup_diagnostic_survives_path_replacement, which START-HERE:3232 names as 7 of the 12 failures in 43 runs of that command. Alone: `test result: ok. 1 passed; 0 failed; 323 filtered out; finished in 0.21s`, exit 0, against the 60s bound it had blown. No file under crates/gascan is touched by this branch. No live test was run here. No assertion edited, no #[ignore] added or removed. --- .../2026-08-20-backend-conformance.md | 295 ++++++++++++++++++ docs/status/START-HERE.md | 105 +++++-- ...-08-20-backend-conformance-suite-design.md | 26 +- 3 files changed, 392 insertions(+), 34 deletions(-) create mode 100644 docs/evidence/2026-08-20-backend-conformance.md diff --git a/docs/evidence/2026-08-20-backend-conformance.md b/docs/evidence/2026-08-20-backend-conformance.md new file mode 100644 index 0000000..a1d30fd --- /dev/null +++ b/docs/evidence/2026-08-20-backend-conformance.md @@ -0,0 +1,295 @@ +# Backend conformance across three backends — MEASURED, 2026-08-20 + +**The shared `RuntimeBackend` contract was run against all three backends for the +first time. All three report a *different* state after `create`, and only the +test double satisfies the contract's assertion.** + +| backend | state after `create` | mechanism | +|---|---|---| +| `FakeRuntime` | `Stopped` | what the double models | +| apple | `Running` | `create` translates to `container run` — `crates/gascan-apple/src/translate.rs:100`, inside the `create` opening at `:94` | +| arca | `Creating` | the pinned engine maps container status `"created"` → `.creating` | + +Both real backends fail at `crates/gascan-conformance/src/lib.rs:104`, the +walk's third assertion, reached after `inspect`-absent and `create`. +**Everything after it — the duplicate +`create`, the doubled `start`, the exec session, the doubled `stop`, `remove`, +and the final absent `inspect` — was NOT REACHED on either backend.** Nobody may +write that apple or arca passed or failed the exec walk. It was not run. + +This document is what P5.3's acceptance criterion 8 required: arca's result is a +*finding*, not a pass criterion. The deliverable is the measurement, and the +failing tests are committed asserting the contract as written. **No assertion was +weakened to produce a green tier.** + +## What was under test + +| | | +|---|---| +| Gas Can revision | branch `feat/backend-conformance-suite`. The eleven commits under "How this branch got here" are the work; the commit that adds this document sits on top of them, at `ba458c9`'s child. Re-derive with `git log --oneline main..feat/backend-conformance-suite`. | +| Contract | `crates/gascan-conformance/src/lib.rs`, `pub async fn backend_contract(&dyn RuntimeBackend, &CreateRequestFixture)` | +| Instantiations | `crates/gascan-conformance/tests/fake.rs`, `crates/gascan-apple/tests/live/backend_contract.rs`, `crates/gascan-arca/tests/live/conformance.rs` | +| Host | `newcombe`, Darwin 25.6.0 arm64 | +| Date | 2026-08-20 | +| Apple's runtime | `container` CLI 1.1.0, service running | +| Arca revision | `c545612b056e028d5885968a7b9f586d694f994c`, the revision `engine/arca-pin.json` names under tag `gascan-engine-m4`. `git -C .artifacts/arca-engine/arca rev-parse HEAD` returns it. | + +**The three backends were NOT given the same request, and the contract does not +require them to be.** What is constant is the contract; the fixture is a +parameter. The fake and apple use `CreateRequestFixture::pinned` — the pinned +workspace image, `network = 'offline'` — and arca uses `for_image` over a stock +alpine layout seeded into the live engine's store, with `network = 'networked'` +and `user = 'root'`, both forced and both explained in `conformance.rs`'s header +comment. The names differ too. Any claim that one compiled request was fed to +three backends is false; see "Two claims corrected" below. + +## The commands, and what each returned + +### Fake — PASSES, and CI runs it every push + +``` +cargo test -p gascan-conformance --test fake +``` + +Exit **0**, 1 passed. It is not `#[ignore]`d, so it runs inside +`cargo test --workspace`, which is exactly `.github/workflows/ci.yml`'s `rust` +job. This is the only one of the three with continuous coverage. + +### Apple — FAILS at the post-`create` state + +``` +cargo test -p gascan-apple --test live -- --ignored backend_contract_holds_on_apple +``` + +Exit **101**: + +``` +panicked at crates/gascan-conformance/src/lib.rs:104:5: +assertion `left == right` failed + left: Running + right: Stopped +``` + +`create` for apple compiles to `container run` +(`crates/gascan-apple/src/translate.rs:100`), so the container is started by the +same command that creates it. There is no window in which apple's `create` has +produced a `Stopped` container. + +### Arca — FAILS at the same assertion, for a different reason + +``` +GASCAN_ARCA_ENGINE_BIN=... GASCAN_ARCA_KERNEL_PATH=... \ +GASCAN_ARCA_VMINIT_LAYOUT=... GASCAN_ARCA_BASE_OCI_LAYOUT=... \ + cargo test -p gascan-arca --test live -- --ignored --test-threads=1 --nocapture \ + backend_contract_holds_on_arca +``` + +Exit **101**, `left: Creating`, `right: Stopped`, +`test result: FAILED. 0 passed; 1 failed; 28 filtered out; finished in 1.16s`. +Reproduced **four times** — 1.30s, 1.13s, 1.00s, 1.16s — same assertion, same +values, every time. + +## Why arca's `Creating` is not the contract reading too early + +This is the obvious alternative explanation and it is closed against the +engine's own source, not against a probe. At revision `c545612b`, +`Sources/ArcaEngine/EngineTranslation.swift:127-134` is a bare `switch` over the +status string: + +```swift +public func sandboxState(fromStatus status: String) -> Arca_Engine_V1_SandboxState { + switch status { + case "created": return .creating + ... +``` + +No clock, no retry, no stored history. **`create` performs no autonomous state +transition; the sandbox sits in the engine's `created` status until something +starts it.** Waiting cannot change what `inspect` returns, so a poll loop in the +contract would not have found `Stopped`. + +Note what this does *not* say: `Creating` is not a state the sandbox is stuck +in, and this document deliberately avoids the words "terminal" and +"indefinitely". `start` leaves it in under two seconds — the positive control +below drives `create` → `start` → `inspect` → `stop` → `remove` end to end in +1.99s. + +## The positive control, run BEFORE the conformance test existed + +Deliberately ordered that way, so a failure would be attributable to arca rather +than to the machine or the environment: + +``` +lifecycle::create_start_inspect_stop_and_remove_drive_a_real_container +test result: ok. 1 passed; 0 failed; 27 filtered out; finished in 1.99s +``` + +Exit **0**. Same host, same day, same four `GASCAN_ARCA_*` variables, same +engine. A real container was created, started, inspected, stopped and removed. +The engine, the kernel, the vminit layout and the base OCI layout were all +working when the conformance test was run against them. + +The captured log is at +`.superpowers/sdd/2026-08-20-backend-conformance-suite/arca-positive-control.log`, +which is **git-ignored and exists only on this machine** — `.gitignore:1`. The +result quoted above is the durable copy. + +## Neither real-backend measurement is reproducible in CI, and the design said otherwise + +**Correcting §6 of `docs/superpowers/specs/2026-08-20-backend-conformance-suite-design.md`, +which claimed the arca instantiation "inherits CI coverage free".** That is false +in effect, and the design has been corrected as part of this work. + +- **Arca.** `.github/workflows/ci.yml:170-178` runs the live tier, but sets + **one** variable, `GASCAN_ARCA_ENGINE_BIN`. The tier needs four. + `backend_contract_holds_on_arca` calls `base_oci_layout()` + (`crates/gascan-arca/tests/live/common/mod.rs:156-159`), whose absence is a + `panic!` and never a skip — deliberately, per the rule at `:137-140`. The + step's own comment records, as a measurement at this revision, that 20 of the + 25 tests `--ignored` selects fail in 0.00s on exactly that missing variable, + and have done so since milestone 2. **The new test can only join them.** That + is a derivation from the panic-not-skip rule and the step's recorded + measurement; no CI run of this test has been observed. +- **Apple.** No CI job runs its tier at all. `--ignored` appears in exactly one + place in `.github/workflows/ci.yml` — line 178, arca's step. + +**So both real-backend measurements are local-only, on a named machine, on a +named date, and this document is the only evidence for them that will ever +exist** until someone puts a kernel, a vminit layout and a base OCI layout on a +runner. The three instantiations do stay wired in: +`scripts/ci-check-ignored-tests.sh` diffs the whole `#[ignore]` set against +`tests/ci/expected-ignored-tests.txt` and fails in both directions, so a test +that vanishes is caught. That guard proves the tests exist. It does not prove +they ran. + +## Apple's residue was cleaned up, and here is the anchor + +The failing apple run creates a real container before it panics. Measured +afterwards on `newcombe`, 2026-08-20: + +| command | what it listed | +|---|---| +| `container list --all` | `buildkit`, `code-3fd063e3b68e` | +| `container volume list` | `gascan-cache-code-3fd063e3b68e`, `containerization-linux-build`, `gascan-config-code-3fd063e3b68e`, `gascan-mise-code-3fd063e3b68e` | +| `container network list` | `default`, `gascan-network-code-3fd063e3b68e` | + +Grepping all three for the failed run's container, +`gascan-live-backend-92391-1787258495344035000-2e7e3b521ca5`, returns **0**. No +`gascan-live-backend-*` resource survives. The remaining entries all belong to +the user's own `code` sandbox and to the containerization build image. + +## An open question this branch surfaced and deliberately did not answer + +**What a *same-request* duplicate `create` reports in `created()` is unmeasured +on every backend.** + +The contract's walk issues its second `create` with the same request as the +first (`crates/gascan-conformance/src/lib.rs:97` and `:112`) and asserts only +that the failure's code is `resource_conflict`. `conflict.created()` is neither +inspected nor removed, and the reason is recorded in the contract itself: a +rejected `create` may report resources it built before the collision, but with +an identical request those names are the **live sandbox's own**. Removing them +would tear down the sandbox the walk still has to start, exec, stop and remove. + +That is not a hypothetical. A cleanup of exactly that shape was written +(`9f2c0bf`) and **reversed** (`e7e55e4`) before it left the branch, because it +would have destroyed the sandbox under test and then double-removed the +container. + +The one live measurement that exists does not settle it. +`crates/gascan-arca/tests/live/lifecycle.rs:259-278` observes a conflicting +create reporting what it made — but there the container and volumes had been +removed first and only the network name was still held, so the three volumes it +reported were genuinely orphaned. **The place to measure the same-request case is +arca's live tier, mirroring `lifecycle.rs:264` without the preparatory remove.** + +### A trap left in the tree, named here because nothing else warns about it + +`crates/gascan-apple/tests/live/storage.rs:22-37` holds +`create_with_partial_cleanup`, which does precisely the +`if !failure.created().is_empty() { remove }` shape that was reversed above. **It +is correct there** — its callers use it on creates expected to fail against +independently-seeded state, not on a duplicate of a live sandbox. But it is the +nearest precedent a future reader will find, and copying it into the conformance +walk would reintroduce `9f2c0bf`'s defect. + +## What promoted, and the estimate it came in under + +**Four assertions across Tasks 6 and 7, against an estimated 6-8.** Counted by +candidate rather than by assertion: of seven candidate tests, **one promoted +whole**, **one promoted in part**, and **five promoted nothing**. + +| where | what promoted | +|---|---| +| `22dfee8` | the doubled `start`, the doubled `stop`, and a second `create` of a held id failing with `resource_conflict` | +| `a32a29e` | an exec session's stream ends at the terminal `Exit` | + +**The spec was corrected, not the work.** Nothing was promoted to close the gap +to the estimate, and §3 of the design now carries the measured outcome in place +of the estimate, with the reason each of the five non-promoting candidates was +left where it is. Every promoted assertion is exercised by `FakeRuntime` alone +today, because all four sit *after* `lib.rs:104`. + +## Two claims corrected + +**`049b4ba`'s commit message states two things that are wrong, and `766eb6e` +corrects both.** A reader who runs `git log 049b4ba` alone gets the uncorrected +text, so it is repeated here: + +1. It says the three backends were given **"the same compiled request"**. They + were not — the requests differ in image, network, user and name. What is + constant across the three instantiations is the *contract*, not the request. +2. It calls arca's `Creating` **"terminal"**. It is not. `start` leaves it in + under two seconds. The accurate statement is the one this document uses: + `create` performs no autonomous state transition. + +## How this branch got here + +| commit | what | +|---|---| +| `f06e96c` | created `crates/gascan-conformance`, dev-dependency only, with `CreateRequestFixture` and `capabilities()` | +| `9dcca6a` | moved the contract in from `gascan-core/tests/`, verified byte-identical apart from three sanctioned changes | +| `daa687b` | deleted the duplicated original from `gascan-core`; test count 175 → 174 | +| `0e1f3fb` | apple instantiation, replacing a 65-line hand-rolled duplicate | +| `049b4ba` | arca instantiation — the measurement the plan exists for | +| `766eb6e` | corrected the two claims above | +| `22dfee8` | promoted 3 assertions | +| `9f2c0bf` | corrected `22dfee8`'s line anchors; added a residue cleanup | +| `e7e55e4` | **reversed** that cleanup — it would have torn down the sandbox under test | +| `a32a29e` | promoted 1 assertion | +| `ba458c9` | design §3 updated with the measured triage outcome | + +## What this does not say + +- **Nothing about whether apple or arca satisfies the rest of the contract.** + Neither reached `start`. `Running` and `Creating` are the only real-backend + facts here. +- **Nothing about which backend is wrong.** Three implementations disagree about + what `create` means, and this document records the disagreement. Whether the + contract should assert `Stopped`, or assert a set, or make the post-`create` + state a fixture-declared expectation, is a design question that has not been + answered and must not be answered by editing the assertion to fit whatever a + backend happens to do. +- **Nothing about a user seeing this.** The contract is a test-tier instrument + and the failures are in the test tier. **Whether any production path reads + `inspect` immediately after `create` and depends on `Stopped` was NOT + surveyed.** Do not read this document as saying the product is unaffected; it + says only that the question was not asked. +- **Nothing about offline.** Arca's fixture is `networked` on purpose — + `docs/evidence/2026-08-18-arca-engine-offline.md` is the reason, and it stands + unchanged. + +## What follows + +1. **The two failures stay in the tree asserting the contract as written.** They + fail today, on a real backend, for a real reason. Weakening `lib.rs:104` to + accept three states would make the suite green and would make it worthless — + that is the outcome acceptance criterion 8 exists to forbid. +2. **Deciding what a backend owes after `create` is the next piece of work**, and + it is a design decision with three live candidates in front of it. It is not + in P5.3's scope. +3. **P5's exit is not met.** Its first clause — extract the conformance suite and + run it against apple and arca — is done, and its result is above. Its second + clause, `gascan-e2e` on arca, is untouched and was explicitly out of scope. +4. **A same-request `create` collision should be measured in arca's live tier** + before anything asserts what `created()` holds on that path. diff --git a/docs/status/START-HERE.md b/docs/status/START-HERE.md index 4039f01..708a05f 100644 --- a/docs/status/START-HERE.md +++ b/docs/status/START-HERE.md @@ -7,8 +7,9 @@ Rewritten 2026-08-18 after **MILESTONE 4 MERGED**, and updated the same day afte instance record's publish race was fixed and merged (PR #80, #81). Updated again 2026-08-19, from branch `fix/daemon-reader-retryable-verdict`, after open item 1's residual — the reader's retryable verdict — was implemented there and opened as PR #87. **Updated 2026-08-20: PR #87 is -merged, open item 1 is closed entire, and the assignment is now P5.3 — the backend conformance -suite, specced and planned in the block below and not started.** Everything above the `Where the work is` heading is current; everything below it is +merged, open item 1 is closed entire, and P5.3 — the backend conformance suite — has been +EXECUTED on branch `feat/backend-conformance-suite`, all eight tasks, and it found two backends +failing the contract. See the block below and open item 10.** Everything above the `Where the work is` heading is current; everything below it is history, **with seven exceptions that are current**: the sections headed `THE FOURTH MECHANISM` through `THE TENTH MECHANISM`, which describe live CI flakes — the ninth is about the *local* suite, so read it before trusting a green local run, and the tenth (added 2026-08-20) is why @@ -18,31 +19,44 @@ suite, so read it before trusting a green local run, and the tenth (added 2026-0 ## IF YOU READ NOTHING ELSE, READ THIS BLOCK -**THE ASSIGNMENT IS P5.3, THE BACKEND CONFORMANCE SUITE. IT IS SPECCED, PLANNED, AND NOT STARTED.** -Chosen by the maintainer on 2026-08-20 after PR #87 merged, on the instruction to follow roadmap -order. Read these two, in this order, and nothing else is needed to begin: +**P5.3, THE BACKEND CONFORMANCE SUITE, IS EXECUTED — ALL EIGHT TASKS — ON BRANCH +`feat/backend-conformance-suite`, WHICH IS NOT MERGED.** As of this edit it had no PR — the +maintainer was holding it for a whole-branch review first. Check with `gh pr list` and re-derive its +commits with `git log --oneline main..feat/backend-conformance-suite` rather than trusting either +fact as written here. + +**THE FINDING, AND IT IS THE POINT OF THE WHOLE EXERCISE: ALL THREE BACKENDS REPORT A DIFFERENT +STATE AFTER `create`, AND ONLY THE TEST DOUBLE SATISFIES THE CONTRACT.** `FakeRuntime` reports +`Stopped`, apple reports `Running` (its `create` compiles to `container run`), arca reports +`Creating` (the pinned engine maps status `"created"` → `.creating`). Apple and arca both panic at +`crates/gascan-conformance/src/lib.rs:104`, the walk's third assertion — so **`start`, `exec`, +`stop`, `remove` and the closing absent `inspect` were NOT REACHED on either real backend. Do not +write that apple or arca passed or failed the exec walk; it was not run.** | | | |---|---| -| Design | `docs/superpowers/specs/2026-08-20-backend-conformance-suite-design.md` | -| Plan | `docs/superpowers/plans/2026-08-20-backend-conformance-suite.md` — eight tasks, execute with `superpowers:subagent-driven-development` | - -**The one thing that will mislead you if you skip the design.** The roadmap says *"extract the -conformance suite from `fake_runtime.rs`"*, which reads as though a suite must be written. It must -not. `crates/gascan-core/tests/backend_contract.rs:149` is **already** -`pub async fn backend_contract(backend: &dyn RuntimeBackend)`. It cannot be reached from -`gascan-apple` or `gascan-arca` because it lives in a `tests/` target, which is not a library — -which is why `crates/gascan-apple/tests/live/backend_contract.rs` hand-rolls 65 lines over the same -ground. The task is relocation and instantiation, not authorship. - -**The measurement the whole task exists for is Task 5 step 6: run the contract against arca for the -first time.** The plan deliberately writes no expected result for it. If arca fails, the deliverable -is the failing test committed with the failure quoted — **not** a weakened assertion. That is -acceptance criterion 8 in the design, and it is the one that will be under pressure. - -**What is NOT in it, so nobody widens it mid-flight:** the product-level `gascan-e2e`-on-arca work -(P5's *second* exit clause), P5.4/U5, and anything about offline. Each is named under the design's -"Out of scope" with a reason. +| **The evidence** | `docs/evidence/2026-08-20-backend-conformance.md` — **read this before saying anything about what conformance measured.** Commands, exit codes, host, date, engine revision, the positive control, and what was not reached | +| Design | `docs/superpowers/specs/2026-08-20-backend-conformance-suite-design.md` — §3 and §6 both carry corrections made after the measurement | +| Plan | `docs/superpowers/plans/2026-08-20-backend-conformance-suite.md` — the eight tasks as executed | + +**Do not "fix" this by editing the assertion.** Acceptance criterion 8 says arca's result is a +finding, not a pass criterion; forcing green by widening `lib.rs:104` to accept three states is the +one outcome that makes the suite worthless. Deciding what a backend owes after `create` is separate +work and is open item 10. + +**NEITHER REAL-BACKEND MEASUREMENT IS REPRODUCIBLE IN CI, and the design used to claim otherwise.** +CI's live-tier step sets one variable and the tier needs four. `backend_contract_holds_on_arca` +calls `base_oci_layout()`, whose absence is a `panic!` and never a skip, so it can only join the +~20 live tests the step's own comment records as failing in 0.00s on that missing variable since +milestone 2 — that is the derivation, not an observed CI run. Apple's tier runs in **no** CI job at +all; line 178 of `.github/workflows/ci.yml` is the file's only `--ignored`. Both results are +local-only, from `newcombe` on 2026-08-20, and the evidence document is the only record of them +that will ever exist. `scripts/ci-check-ignored-tests.sh` proves the tests still exist; it proves +nothing about their having run. + +**P5'S EXIT IS NOT MET.** P5.3 covered its *first* clause only — extract the suite, run it against +apple and arca — and the answer was two failures. The **second** clause, `gascan-e2e` on arca, is +untouched and was explicitly out of scope, as were P5.4/U5 and anything about offline. --- @@ -110,8 +124,9 @@ eleven of its last twelve runs, since before this branch existed. insertions, markdown**. There is no code in it. A red `rust` there is a flake, exonerated by diff alone. Run `git diff --name-only` before you spend an hour on a red run. -**THE QUEUE BELOW IS WHAT COMES AFTER P5.3, NOT INSTEAD OF IT.** It is a queue, not a menu, and -the maintainer chooses from it once the conformance suite is done: +**THE QUEUE BELOW IS WHAT COMES AFTER P5.3.** P5.3's implementation is done and its branch is +awaiting review; what it *found* is open item 10, which joins this queue rather than blocking it. +It is a queue, not a menu, and the maintainer chooses from it: 1. **The seven unfixed flake mechanisms this file names** — the empty pid-file read, the `reconcile` phase matrix that is red on `main` itself, the `lifecycle` ephemeral-port @@ -684,6 +699,44 @@ unbuilt**: an unstarted implementation with a specified shape, not an open quest grounds that a literal in a test outside the implementing crates is the external contract rather than a copy. Both judgements were merged on the maintainer's standing merge-on-green authorization without a separate review round, and both remain reversible. +10. **APPLE AND ARCA BOTH FAIL THE BACKEND CONTRACT AT + `crates/gascan-conformance/src/lib.rs:104`, AND NOBODY HAS DECIDED WHAT THE RIGHT ANSWER IS.** + Opened 2026-08-20 by P5.3. The assertion is + `assert_eq!(backend.inspect(&id).await.unwrap().unwrap().state, ContainerState::Stopped)`, + immediately after `create`. **All three backends disagree**: `FakeRuntime` `Stopped`, apple + `Running`, arca `Creating`. Full record, with commands, exit codes, host, date and engine + revision, in **`docs/evidence/2026-08-20-backend-conformance.md`** — read it before saying + anything about what conformance measured. + + **Both failures are real, not instrument error.** Apple's `create` compiles to `container run` + (`crates/gascan-apple/src/translate.rs:100`), so it has no `Stopped` window at all. Arca's + engine, at the pinned revision `c545612b`, maps status `"created"` → `.creating` in a bare + `switch` with no clock and no retry (`Sources/ArcaEngine/EngineTranslation.swift:127-134`): + `create` performs no autonomous state transition, so waiting longer would not have helped. + A positive control — `lifecycle::create_start_inspect_stop_and_remove_drive_a_real_container`, + run on the same host and engine **before** the conformance test existed — passed, exit 0. + + **`start`, `exec`, `stop`, `remove` and the closing absent `inspect` were NOT REACHED on either + real backend.** They are unmeasured, not passing and not failing. + + **What is open is a design decision, and it has three live candidates**: assert a set of + acceptable post-`create` states, make the expected state a fixture-declared fact, or change a + backend. **Do not close it by widening `lib.rs:104` to accept whatever the backends do** — that + is the outcome the design's acceptance criterion 8 exists to forbid. The two tests stay in the + tree failing, on the same principle as `network.rs`'s offline test. + + **Neither result can be reproduced by CI**, so re-measuring means a real Mac with the four + `GASCAN_ARCA_*` variables, or `container` running for apple. The reason is in the evidence + document and in the design's §6, which was corrected here — it previously claimed the arca + instantiation "inherits CI coverage free". + + **A second, smaller thing this branch surfaced and did not answer:** what a *same-request* + duplicate `create` reports in `created()` is unmeasured on every backend. A cleanup that + assumed it was orphaned residue was written (`9f2c0bf`) and reversed (`e7e55e4`) because it + would have torn down the sandbox under test. The nearest precedent a reader will find, + `crates/gascan-apple/tests/live/storage.rs:22-37`, is correct *where it is* and must not be + copied into the conformance walk. The place to measure it is arca's live tier, mirroring + `crates/gascan-arca/tests/live/lifecycle.rs:264` without the preparatory remove. ### WHAT WAS RUN, AND WHAT CI DOES WITH IT diff --git a/docs/superpowers/specs/2026-08-20-backend-conformance-suite-design.md b/docs/superpowers/specs/2026-08-20-backend-conformance-suite-design.md index df7ee44..10cae5f 100644 --- a/docs/superpowers/specs/2026-08-20-backend-conformance-suite-design.md +++ b/docs/superpowers/specs/2026-08-20-backend-conformance-suite-design.md @@ -100,7 +100,8 @@ duplication is not short-lived. population. (It holds **21** after Task 7: the fake instantiation moved to `gascan-conformance` and Task 6 consumed `duplicate_create_is_rejected_and_start_stop_are_idempotent` whole.) -**Promote — assertions any backend must satisfy.** The lifecycle walk already in +**Candidates considered.** Five of the seven tests named in this paragraph promoted nothing; the +measured outcome is the table under the next heading, not this list. The lifecycle walk already in `backend_contract()`, plus `duplicate_create_is_rejected_and_start_stop_are_idempotent`, `exec_and_logs_preserve_binary_bytes_and_exact_exit_code`, `exec_session_is_live_bidirectional_and_emits_one_exit`, @@ -221,13 +222,22 @@ has stopped running."* The same sentence is repeated at `:333-335` on `LiveEngin instantiation uses both and inherits the rule. **Where each instantiation actually runs, which is asymmetric and worth knowing before relying on -it:** the fake instantiation runs in `cargo test --workspace`, so CI covers it every push. The -**arca** instantiation lands in the live tier CI already executes — -`cargo test -p gascan-arca --test live --no-fail-fast -- --ignored`, `.github/workflows/ci.yml:178` -— so it inherits CI coverage free. The **apple** instantiation runs **nowhere in CI**; no workflow -step passes `--ignored` for `gascan-apple` or `gascan-e2e`. It is a local, manual tier. This design -does not change that, and no claim that "apple passes conformance" should be made without saying -which machine it was run on and when. +it.** The fake instantiation runs in `cargo test --workspace`, so CI covers it every push. **Neither +real backend's instantiation is executed by CI in any usable sense — CORRECTING what this section +said before**, which was that the arca instantiation "inherits CI coverage free". The **arca** +instantiation does land in the live tier CI executes +(`cargo test -p gascan-arca --test live --no-fail-fast -- --ignored`, +`.github/workflows/ci.yml:178`), but that step sets **one** variable, `GASCAN_ARCA_ENGINE_BIN`, and +the tier needs four. `backend_contract_holds_on_arca` calls `base_oci_layout()`, whose absence is a +`panic!` and never a skip, and the step's own comment records as a measurement that 20 of the 25 +tests `--ignored` selects fail in 0.00s on exactly that missing variable and have done so since +milestone 2 — so the new test can only join them. That is a derivation, not an observed CI run. +The **apple** instantiation runs **nowhere in CI**; no workflow step +passes `--ignored` for `gascan-apple` or `gascan-e2e`, and line 178 is the only `--ignored` in the +file. Both real-backend results are therefore local-only, and no claim that a real backend passes or +fails conformance should be made without naming the machine and the date. What CI *does* hold is +that the tests are still wired in, via `scripts/ci-check-ignored-tests.sh`; that is existence, not +execution. `docs/evidence/2026-08-20-backend-conformance.md` is where the local measurements live. **Proving the extraction is behaviour-preserving.** The fake instantiation must pass before and after the move with **no edit to any assertion body**. If an assertion had to change, the move was From ac365a68e9f76e8292bcd65311fcf82e2a9592c2 Mon Sep 17 00:00:00 2001 From: Richard Kiene Date: Thu, 20 Aug 2026 18:43:39 -0700 Subject: [PATCH 13/13] docs: the assertion two backends fail now says so where they fail MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Final whole-branch review, one Important and eight Minors, all comment or prose. No assertion, no #[allow], no #[ignore], no baseline change. Proof rather than assertion: `git diff -U0 -- crates/ | grep "^[+-]" | grep -v "^[+-][+-]" | grep -v "^\s*[+-]\s*//" | grep -v "^[+-]$"` returns nothing -- every added and removed line under crates/ is a comment or a blank. expected-ignored-tests.txt is untouched at 50 lines and fake_runtime.rs is not in the diff. I-1. The post-`create` state assertion, the one line two of three backends fail and acceptance criterion 8 forbids weakening, carried no comment; apple's instantiation carried none at all, against its arca sibling's fifteen lines on an unreachable kill(). Someone arriving from `panicked at ... left: Running, right: Stopped` had nothing at either the panic site or the test file saying it is a recorded result, and the cheapest path from there is to "fix" it. Three places now say so: the assertion itself, apple's test file, and lib.rs's module doc, which said why the crate exists but not what it currently measures. THAT MOVED THE ASSERTION OFF LINE 104. It is lib.rs:139 now. The recorded panic text stays verbatim -- it is a measurement, not a pointer -- and every prose citation of :104 was re-derived, with the reconciliation written in four places. `grep -rn "lib.rs:104" docs/` returns exactly one hit, the quoted panic. Two other shifted anchors corrected the same way: the two create calls :97/:112 -> :117/:147, the collision comment :114-123 -> :149-164. M-1. Apple's own list_resources tail (backend_contract.rs:37-44, re-derived after this wave) is unreached like everything after the panic and was missing from the evidence document's enumeration, two sections from a residue check run by hand against the CLI. Both are named now, and the difference stated. M-2. The collision comment warned against a shape without naming the precedent. It now names gascan-apple/tests/live/storage.rs:22-37 and says why that helper is correct where it is: its callers face independently-seeded state, not a duplicate of a live sandbox. M-3. The two permanent fixture copies now reference each other, citing the design's §2 for the permanence and the dev-dependency cycle for the reason. The (1,1,0)/(1,0,0) divergence is written down with its inertness argument rather than left to be re-derived: `grep -n "\.version" gascan-core/src/policy.rs` returns :422 constructing PolicyError::OfflineUnsupported and :561-563 formatting it, and :417-425 shows :422 is reachable only through NetworkIsolation::Unsupported, which neither copy sets. M-4. START-HERE declared P5.3 executed a few thousand lines above a line still assigning it the sandbox_id-claim sharing. Settled at both ends: struck through at the line itself with the reason (a production change, excluded wholesale by §5 -- `git diff --name-only main...feat/backend-conformance-suite | grep "/src/"` returns only crates/gascan-conformance/src/lib.rs), and restated in item 10, which points at the history by its text rather than by a line number. M-5. "the only `--ignored` in the file" was overstated in three artifacts. `grep -n -- "--ignored" .github/workflows/ci.yml` returns :139 and :162 inside comments and :178 executed. All three now say the only EXECUTED one. M-6. 0e1f3fb's message says the three `container ...` listings "afterwards match their pre-run output" and no pre-run listing was ever captured. The evidence document did that service for 049b4ba and not for this. It does now, quoting the sentence and saying what was actually measured and what it is not enough to claim. M-7. The CI comment's "20 of the 25" is from an earlier revision and both the evidence document and design §6 propagated it as current. Re-derived: `grep -rc '#\[ignore' crates/gascan-arca/tests/live/*.rs` sums to 29, corroborated by the arca run's own "28 filtered out" beside its 1 failure. Both documents now say so. M-8. The idempotence comment sat 22 lines from the stop pair it also governs; a one-line marker closes it. M-9 and M-10 left, as the review recorded them: seen and not worth the churn. AUDIT-4, correcting an anchor in 99f1449 rather than amending it, since the branch's rule is that a cited commit is not rewritten. 99f1449 cites START-HERE:3232 for "7 of the 12 failures in 43 runs". :3232 is a row of the supporting table; the sentence naming inherited_startup_diagnostic_survives_path_replacement as the dominant flake is at :3236 and the 43-runs total at :3235. VERIFIED at this tree, each run alone with exit codes from plain $? on unpiped commands: `cargo fmt --all --check` exit 0; `cargo clippy --workspace --all-targets -- -D warnings` exit 0, "No issues found". `cargo test --workspace` deliberately NOT run -- nothing here changes executable code, which the comment-only proof above establishes, and clippy --all-targets compiles every target touched. No live test was run. --- .../tests/live/backend_contract.rs | 20 ++++++ crates/gascan-conformance/src/lib.rs | 42 ++++++++++++ crates/gascan-core/tests/common/mod.rs | 10 +++ .../2026-08-20-backend-conformance.md | 67 ++++++++++++++----- docs/status/START-HERE.md | 43 ++++++++---- ...-08-20-backend-conformance-suite-design.md | 16 +++-- 6 files changed, 161 insertions(+), 37 deletions(-) diff --git a/crates/gascan-apple/tests/live/backend_contract.rs b/crates/gascan-apple/tests/live/backend_contract.rs index b2d10fc..859c005 100644 --- a/crates/gascan-apple/tests/live/backend_contract.rs +++ b/crates/gascan-apple/tests/live/backend_contract.rs @@ -3,6 +3,26 @@ use gascan_conformance::{CreateRequestFixture, backend_contract}; use gascan_core::runtime::RuntimeBackend; use std::time::{SystemTime, UNIX_EPOCH}; +/// The shared backend contract, run against a real `container` CLI. +/// +/// **THIS TEST FAILS TODAY AND THE FAILURE IS THE RECORDED RESULT, NOT A +/// REGRESSION.** MEASURED on `newcombe` 2026-08-20 with `container` 1.1.0: +/// `panicked at gascan-conformance/src/lib.rs:104:5 ... left: Running, right: +/// Stopped` -- apple's `create` compiles to `container run` +/// (`gascan-apple/src/translate.rs:100`), so there is no window in which it has +/// produced a `Stopped` container. Everything after that assertion, including +/// the `list_resources` tail below, was NOT REACHED. **Do not weaken the +/// assertion to make this green**; see +/// `docs/evidence/2026-08-20-backend-conformance.md` and +/// `docs/status/START-HERE.md` open item 10. +/// +/// The panic quoted above says `104` because that is where the assertion sat +/// when it was measured; the comment now standing over it moved the assertion +/// down. Re-derive the line rather than trusting either number. +/// +/// **No CI job runs this tier**, so that measurement is the only evidence that +/// will exist until someone runs it again by hand on a machine with the +/// `container` service. #[tokio::test] #[ignore = "requires Apple silicon macOS 26+ with container service and locked workspace image"] async fn backend_contract_holds_on_apple() { diff --git a/crates/gascan-conformance/src/lib.rs b/crates/gascan-conformance/src/lib.rs index a67e26e..84c738f 100644 --- a/crates/gascan-conformance/src/lib.rs +++ b/crates/gascan-conformance/src/lib.rs @@ -3,6 +3,12 @@ //! This crate exists because `gascan-core/src/lib.rs:2` denies //! `clippy::unwrap_used`, and a conformance suite is built from unwrapping //! assertions. It is a dev-dependency of its consumers and ships nowhere. +//! +//! **What it measures today: `FakeRuntime` satisfies the contract and the two +//! real backends do not.** Both fail at the post-`create` state assertion in +//! [`backend_contract`], which carries the detail. That is the recorded +//! deliverable of P5.3, not an outstanding bug -- +//! `docs/evidence/2026-08-20-backend-conformance.md`. use camino::Utf8Path; use gascan_core::manifest::Manifest; @@ -14,6 +20,20 @@ use gascan_core::runtime::{ use gascan_core::sandbox::SandboxSpec; use std::ops::Deref; +/// A compiled `CreateRequest` and the temporary root it was compiled from. +/// +/// **A near-copy of this fixture lives at `gascan-core/tests/common/mod.rs`, and +/// the duplication is deliberate and permanent** -- see the design's §2 +/// (`docs/superpowers/specs/2026-08-20-backend-conformance-suite-design.md`). +/// Pointing `gascan-core/tests` at this crate would mint a `gascan-core` +/// dev-dependency on a crate that depends on `gascan-core`. The copies have +/// already diverged on `capabilities().version` -- `(1, 1, 0)` here against +/// `(1, 0, 0)` there. That is inert by a two-step argument, so it is written +/// down rather than left to be re-derived: the field is read in exactly two +/// places, `gascan-core/src/policy.rs:422` constructing +/// `PolicyError::OfflineUnsupported` and `:561-563` formatting it, and both are +/// reachable only through `NetworkIsolation::Unsupported`, which neither copy +/// sets. Anything else that diverges needs its own such argument. pub struct CreateRequestFixture { _root: tempfile::TempDir, request: CreateRequest, @@ -101,6 +121,21 @@ pub async fn backend_contract(backend: &dyn RuntimeBackend, fixture: &CreateRequ .iter() .any(|resource| resource.kind() == ResourceKind::Container) ); + // **TWO OF THE THREE BACKENDS FAIL HERE, AND THAT IS A RECORDED FINDING, NOT + // A BUG IN THIS ASSERTION.** MEASURED on `newcombe` 2026-08-20: apple + // reports `Running` (its `create` compiles to `container run`, + // `gascan-apple/src/translate.rs:100`) and arca reports `Creating` (the + // pinned engine maps status "created" -> `.creating`). Only `FakeRuntime` + // reports `Stopped`. Both failures, with their commands, exit codes and the + // positive control, are in `docs/evidence/2026-08-20-backend-conformance.md`. + // Those two panics name `lib.rs:104:5`, which is where this `assert_eq!` sat + // when they were taken; adding this comment moved it down. The quoted panic + // text is left as measured -- it is an observation, not a pointer. + // + // **Do not widen this to accept three states.** The design's acceptance + // criterion 8 names that as the one outcome that makes the whole exercise + // worthless. Deciding what a backend owes after `create` is separate work -- + // `docs/status/START-HERE.md` open item 10. assert_eq!( backend.inspect(&id).await.unwrap().unwrap().state, ContainerState::Stopped @@ -121,6 +156,12 @@ pub async fn backend_contract(backend: &dyn RuntimeBackend, fixture: &CreateRequ // there the container and volumes had been removed first and only the // network name was still held, so its three volumes were genuinely // orphaned. What a same-request collision reports is unmeasured everywhere. + // + // **The precedent a reader will find is `gascan-apple/tests/live/storage.rs` + // `create_with_partial_cleanup` (`:22-37`), and it must not be copied here.** + // It is correct where it is -- its callers hand it creates expected to fail + // against independently-seeded state, so what those failures report really + // is orphaned. Here the collision is a duplicate of the live sandbox. // Doubled deliberately: `start` and `stop` are idempotent, so the second // call of each must succeed and not report the sandbox's current state as @@ -146,6 +187,7 @@ pub async fn backend_contract(backend: &dyn RuntimeBackend, fixture: &CreateRequ // so the sender drops (`gascan-apple/src/backend.rs:614`), arca by the same // break over engine frames (`gascan-arca/src/backend.rs:387`). assert!(session.next().await.is_none()); + // Doubled deliberately -- the `stop` half of the idempotence pair above. backend.stop(&id).await.unwrap(); backend.stop(&id).await.unwrap(); backend diff --git a/crates/gascan-core/tests/common/mod.rs b/crates/gascan-core/tests/common/mod.rs index e5e4d0e..b0b9e24 100644 --- a/crates/gascan-core/tests/common/mod.rs +++ b/crates/gascan-core/tests/common/mod.rs @@ -5,6 +5,16 @@ use gascan_core::runtime::{CreateRequest, NetworkIsolation, RuntimeCapabilities, use gascan_core::sandbox::SandboxSpec; use std::ops::Deref; +/// A compiled `CreateRequest` and the temporary root it was compiled from. +/// +/// **A near-copy of this fixture lives at `gascan-conformance/src/lib.rs`, and +/// the duplication is deliberate and permanent** -- see the design's §2 +/// (`docs/superpowers/specs/2026-08-20-backend-conformance-suite-design.md`). +/// Pointing this target at `gascan-conformance` would mint a `gascan-core` +/// dev-dependency on a crate that depends on `gascan-core`. The copies have +/// already diverged on `capabilities().version` -- `(1, 0, 0)` here against +/// `(1, 1, 0)` there -- which the conformance copy's own comment argues is +/// inert. Do not let anything else diverge without making that argument again. pub struct CreateRequestFixture { _root: tempfile::TempDir, request: CreateRequest, diff --git a/docs/evidence/2026-08-20-backend-conformance.md b/docs/evidence/2026-08-20-backend-conformance.md index a1d30fd..dda9587 100644 --- a/docs/evidence/2026-08-20-backend-conformance.md +++ b/docs/evidence/2026-08-20-backend-conformance.md @@ -10,12 +10,25 @@ test double satisfies the contract's assertion.** | apple | `Running` | `create` translates to `container run` — `crates/gascan-apple/src/translate.rs:100`, inside the `create` opening at `:94` | | arca | `Creating` | the pinned engine maps container status `"created"` → `.creating` | -Both real backends fail at `crates/gascan-conformance/src/lib.rs:104`, the -walk's third assertion, reached after `inspect`-absent and `create`. -**Everything after it — the duplicate -`create`, the doubled `start`, the exec session, the doubled `stop`, `remove`, -and the final absent `inspect` — was NOT REACHED on either backend.** Nobody may -write that apple or arca passed or failed the exec walk. It was not run. +Both real backends fail at the post-`create` state assertion in +`crates/gascan-conformance/src/lib.rs` — the walk's third assertion, reached +after `inspect`-absent and `create`. **It is at `:139` as this document is +written; the panics quoted below say `:104`, which is where it sat when they +were taken, and the explanatory comment added over it afterwards moved it down. +The panic text is left as measured. Re-derive the line; do not trust either +number.** + +**Everything after that assertion — the duplicate `create`, the doubled `start`, +the exec session, the doubled `stop`, `remove`, and the final absent `inspect` — +was NOT REACHED on either backend.** Nor was apple's own tail: the +`list_resources` assertion at +`crates/gascan-apple/tests/live/backend_contract.rs:37-44`, which checks that no +resource named for the run survives, is downstream of the panic and did not +execute either. **Do not confuse it with the residue check +recorded further down this document, which was run by hand against the +`container` CLI afterwards and is a different instrument measuring a narrower +property.** Nobody may write that apple or arca passed or failed the exec walk, +or that apple's own residue assertion passed. None of it was run. This document is what P5.3's acceptance criterion 8 required: arca's result is a *finding*, not a pass criterion. The deliverable is the measurement, and the @@ -26,7 +39,7 @@ weakened to produce a green tier.** | | | |---|---| -| Gas Can revision | branch `feat/backend-conformance-suite`. The eleven commits under "How this branch got here" are the work; the commit that adds this document sits on top of them, at `ba458c9`'s child. Re-derive with `git log --oneline main..feat/backend-conformance-suite`. | +| Gas Can revision | branch `feat/backend-conformance-suite`. The commits under "How this branch got here" are the work; this document and the review fixes sit on top of them. Re-derive the full list with `git log --oneline main..feat/backend-conformance-suite` rather than counting from here. | | Contract | `crates/gascan-conformance/src/lib.rs`, `pub async fn backend_contract(&dyn RuntimeBackend, &CreateRequestFixture)` | | Instantiations | `crates/gascan-conformance/tests/fake.rs`, `crates/gascan-apple/tests/live/backend_contract.rs`, `crates/gascan-arca/tests/live/conformance.rs` | | Host | `newcombe`, Darwin 25.6.0 arm64 | @@ -145,13 +158,17 @@ in effect, and the design has been corrected as part of this work. `backend_contract_holds_on_arca` calls `base_oci_layout()` (`crates/gascan-arca/tests/live/common/mod.rs:156-159`), whose absence is a `panic!` and never a skip — deliberately, per the rule at `:137-140`. The - step's own comment records, as a measurement at this revision, that 20 of the - 25 tests `--ignored` selects fail in 0.00s on exactly that missing variable, - and have done so since milestone 2. **The new test can only join them.** That - is a derivation from the panic-not-skip rule and the step's recorded - measurement; no CI run of this test has been observed. -- **Apple.** No CI job runs its tier at all. `--ignored` appears in exactly one - place in `.github/workflows/ci.yml` — line 178, arca's step. + step's own comment records, as a measurement, that 20 of the 25 tests + `--ignored` selects fail in 0.00s on exactly that missing variable, and have + done so since milestone 2. **That 25 is from an earlier revision.** Re-derived + at this one: `grep -rc '#\[ignore' crates/gascan-arca/tests/live/*.rs`, summed, + is **29**, one of them added by this branch — which is also why the arca run + below reports 28 filtered out beside its 1 failure. **The new test can only + join the failing majority.** That is a derivation from the panic-not-skip rule + and the step's recorded measurement; no CI run of this test has been observed. +- **Apple.** No CI job runs its tier at all. `--ignored` appears three times in + `.github/workflows/ci.yml` — `:139` and `:162`, both inside comments, and + `:178`, arca's step, **the only executed one**. **So both real-backend measurements are local-only, on a named machine, on a named date, and this document is the only evidence for them that will ever @@ -178,13 +195,22 @@ Grepping all three for the failed run's container, `gascan-live-backend-*` resource survives. The remaining entries all belong to the user's own `code` sandbox and to the containerization build image. +**`0e1f3fb`'s commit message claims more than this, and it is the second +uncorrected claim on the branch.** It says the three `container …` listings +"afterwards match their pre-run output" — **no pre-run listing was ever +captured**, so that comparison was never available to make. What was measured is +the narrower property above, and only afterwards: that nothing named for the run +survives. That is enough to say the run left no residue; it is not enough to say +the host is in the state it started in. Unlike `049b4ba`'s two claims, this one +has no correcting commit, which is why the correction is here. + ## An open question this branch surfaced and deliberately did not answer **What a *same-request* duplicate `create` reports in `created()` is unmeasured on every backend.** The contract's walk issues its second `create` with the same request as the -first (`crates/gascan-conformance/src/lib.rs:97` and `:112`) and asserts only +first (`crates/gascan-conformance/src/lib.rs:117` and `:147`) and asserts only that the failure's code is `resource_conflict`. `conflict.created()` is neither inspected nor removed, and the reason is recorded in the contract itself: a rejected `create` may report resources it built before the collision, but with @@ -228,7 +254,7 @@ whole**, **one promoted in part**, and **five promoted nothing**. to the estimate, and §3 of the design now carries the measured outcome in place of the estimate, with the reason each of the five non-promoting candidates was left where it is. Every promoted assertion is exercised by `FakeRuntime` alone -today, because all four sit *after* `lib.rs:104`. +today, because all four sit *after* the post-`create` state assertion. ## Two claims corrected @@ -258,6 +284,10 @@ text, so it is repeated here: | `e7e55e4` | **reversed** that cleanup — it would have torn down the sandbox under test | | `a32a29e` | promoted 1 assertion | | `ba458c9` | design §3 updated with the measured triage outcome | +| `99f1449` | this document, `START-HERE` item 10, and the design §6 correction below | + +The final whole-branch review's fixes land after `99f1449` — comment-only, no +assertion touched. They are why the assertion's line number moved off `104`. ## What this does not say @@ -282,8 +312,9 @@ text, so it is repeated here: ## What follows 1. **The two failures stay in the tree asserting the contract as written.** They - fail today, on a real backend, for a real reason. Weakening `lib.rs:104` to - accept three states would make the suite green and would make it worthless — + fail today, on a real backend, for a real reason. Weakening the post-`create` + state assertion to accept three states would make the suite green and would + make it worthless — that is the outcome acceptance criterion 8 exists to forbid. 2. **Deciding what a backend owes after `create` is the next piece of work**, and it is a design decision with three live candidates in front of it. It is not diff --git a/docs/status/START-HERE.md b/docs/status/START-HERE.md index 708a05f..de05f63 100644 --- a/docs/status/START-HERE.md +++ b/docs/status/START-HERE.md @@ -29,9 +29,11 @@ fact as written here. STATE AFTER `create`, AND ONLY THE TEST DOUBLE SATISFIES THE CONTRACT.** `FakeRuntime` reports `Stopped`, apple reports `Running` (its `create` compiles to `container run`), arca reports `Creating` (the pinned engine maps status `"created"` → `.creating`). Apple and arca both panic at -`crates/gascan-conformance/src/lib.rs:104`, the walk's third assertion — so **`start`, `exec`, -`stop`, `remove` and the closing absent `inspect` were NOT REACHED on either real backend. Do not -write that apple or arca passed or failed the exec walk; it was not run.** +the post-`create` state assertion in `crates/gascan-conformance/src/lib.rs` — the walk's third, +`:139` today and `:104` in the recorded panic text, because the comment now standing over it moved +it. **`start`, `exec`, `stop`, `remove`, the closing absent `inspect`, and apple's own +`list_resources` tail were NOT REACHED on either real backend. Do not write that apple or arca +passed or failed the exec walk; it was not run.** | | | |---|---| @@ -40,16 +42,18 @@ write that apple or arca passed or failed the exec walk; it was not run.** | Plan | `docs/superpowers/plans/2026-08-20-backend-conformance-suite.md` — the eight tasks as executed | **Do not "fix" this by editing the assertion.** Acceptance criterion 8 says arca's result is a -finding, not a pass criterion; forcing green by widening `lib.rs:104` to accept three states is the -one outcome that makes the suite worthless. Deciding what a backend owes after `create` is separate -work and is open item 10. +finding, not a pass criterion; forcing green by widening that assertion to accept three states is +the one outcome that makes the suite worthless. Deciding what a backend owes after `create` is +separate work and is open item 10. **The assertion now carries a comment saying all of this**, so +a reader arriving from a panic message is not left to guess. **NEITHER REAL-BACKEND MEASUREMENT IS REPRODUCIBLE IN CI, and the design used to claim otherwise.** CI's live-tier step sets one variable and the tier needs four. `backend_contract_holds_on_arca` calls `base_oci_layout()`, whose absence is a `panic!` and never a skip, so it can only join the ~20 live tests the step's own comment records as failing in 0.00s on that missing variable since milestone 2 — that is the derivation, not an observed CI run. Apple's tier runs in **no** CI job at -all; line 178 of `.github/workflows/ci.yml` is the file's only `--ignored`. Both results are +all; `.github/workflows/ci.yml` mentions `--ignored` three times and `:178`, arca's step, is the +only **executed** one. Both results are local-only, from `newcombe` on 2026-08-20, and the evidence document is the only record of them that will ever exist. `scripts/ci-check-ignored-tests.sh` proves the tests still exist; it proves nothing about their having run. @@ -699,8 +703,9 @@ unbuilt**: an unstarted implementation with a specified shape, not an open quest grounds that a literal in a test outside the implementing crates is the external contract rather than a copy. Both judgements were merged on the maintainer's standing merge-on-green authorization without a separate review round, and both remain reversible. -10. **APPLE AND ARCA BOTH FAIL THE BACKEND CONTRACT AT - `crates/gascan-conformance/src/lib.rs:104`, AND NOBODY HAS DECIDED WHAT THE RIGHT ANSWER IS.** +10. **APPLE AND ARCA BOTH FAIL THE BACKEND CONTRACT AT ITS POST-`create` STATE ASSERTION + (`crates/gascan-conformance/src/lib.rs`, `:139` today, `:104` in the recorded panic text — the + comment now over it moved the line), AND NOBODY HAS DECIDED WHAT THE RIGHT ANSWER IS.** Opened 2026-08-20 by P5.3. The assertion is `assert_eq!(backend.inspect(&id).await.unwrap().unwrap().state, ContainerState::Stopped)`, immediately after `create`. **All three backends disagree**: `FakeRuntime` `Stopped`, apple @@ -721,9 +726,17 @@ unbuilt**: an unstarted implementation with a specified shape, not an open quest **What is open is a design decision, and it has three live candidates**: assert a set of acceptable post-`create` states, make the expected state a fixture-declared fact, or change a - backend. **Do not close it by widening `lib.rs:104` to accept whatever the backends do** — that - is the outcome the design's acceptance criterion 8 exists to forbid. The two tests stay in the - tree failing, on the same principle as `network.rs`'s offline test. + backend. **Do not close it by widening that assertion to accept whatever the backends do** — + that is the outcome the design's acceptance criterion 8 exists to forbid. The two tests stay in + the tree failing, on the same principle as `network.rs`'s offline test, and both the assertion + and apple's test file now carry comments saying so. + + **P5.3 did NOT take the `sandbox_id`-claim sharing** that the history below once assigned to it + (grep `Sharing it belongs to`), and that was right: it is a production change to + `gascan-arca/src/translate.rs` and + `gascan-apple/src/inspect.rs`, and the design's §5 excludes production changes wholesale — the + branch touches exactly one `/src/` file, the new conformance crate's own. It is unassigned work + now, not P5.3's residue. **Neither result can be reproduced by CI**, so re-measuring means a real Mac with the four `GASCAN_ARCA_*` variables, or `container` running for apple. The reason is in the evidence @@ -3378,4 +3391,8 @@ engine.** Do not add a test double for it. The `sandbox_id`-claim rule is still duplicated verbatim between `gascan-arca/src/translate.rs` and `gascan-apple/src/inspect.rs`, each with its own test and -a comment warning they must not diverge. Sharing it belongs to P5.3. +a comment warning they must not diverge. ~~Sharing it belongs to P5.3.~~ **P5.3 did not take it, +deliberately: it is a production change and the design's §5 excludes those wholesale — the whole +branch touches one `/src/` file, `git diff --name-only main...feat/backend-conformance-suite | grep +"/src/"` returning only `crates/gascan-conformance/src/lib.rs`. It is unassigned now.** See open +item 10. diff --git a/docs/superpowers/specs/2026-08-20-backend-conformance-suite-design.md b/docs/superpowers/specs/2026-08-20-backend-conformance-suite-design.md index 10cae5f..79916fa 100644 --- a/docs/superpowers/specs/2026-08-20-backend-conformance-suite-design.md +++ b/docs/superpowers/specs/2026-08-20-backend-conformance-suite-design.md @@ -125,7 +125,8 @@ nothing. Both countings are below the estimate, and nothing was promoted to clos | Task 7 | from `exec_session_is_live_bidirectional_and_emits_one_exit`: the exec stream ends at the terminal `Exit`. The rest of that test is fake-only and stays, renamed `exec_session_echoes_stdin_and_maps_a_signal_to_its_exit_code` so the name matches the assertions left in it. | **Every promoted assertion is exercised by `FakeRuntime` alone today.** Apple and arca both fail the -contract at the post-`create` state assertion (`crates/gascan-conformance/src/lib.rs:104`) — apple +contract at the post-`create` state assertion (`crates/gascan-conformance/src/lib.rs:139`, `:104` +in the recorded panic text — the comment now over it moved the line) — apple reports `Running`, arca `Creating` — which precedes every line promoted, so neither backend has been measured against any of it. @@ -135,7 +136,7 @@ been measured against any of it. |---|---| | `exec_and_logs_preserve_binary_bytes_and_exact_exit_code` | `set_exec_result` and `set_logs`, both fake-only. Asserting the property portably needs a command that emits known bytes on stdout *and* stderr and exits non-zero; the fake's vocabulary for that is `fake-stdout` / `fake-stderr` / `fake-exit` (`crates/gascan-core/src/fake_runtime.rs:588-636`), which no container image has, and the fake maps the portable spelling to nothing at all — `Some("true") \| Some("sh") => (Vec::new(), Vec::new(), 0)` at `:633`. Giving the contract a per-backend command means parameterising it, a design change. The exit code the walk *can* portably assert is already asserted. The log half is worse than unportable: `since` is not the same quantity across backends — apple passes `--since {n}ms` to the CLI, a duration ago (`crates/gascan-apple/src/backend.rs:630-632`), arca sends `since_unix_millis`, an absolute instant (`crates/gascan-arca/src/backend.rs:411-414`). | | `exec_session_is_live_bidirectional_and_emits_one_exit` | **Promoted in part**, see above. What stays needs `fake-echo-stdin` to get stdin back, and its `Exit { code: 143, signal: 15 }` is the fake's own `128 + signal` arithmetic (`crates/gascan-core/src/fake_runtime.rs:1118-1122`), not something a backend owes. | -| `create_collision_reports_resources_created_before_the_collision` | `seed_volume`, fake-only. The assertion's entire content is that a failure reports exactly the resources built before a **planted** collision at a chosen index. Creating twice does produce a collision on a real backend, but with the same request — so the reported names would be the live sandbox's own, and what a same-request collision reports is unmeasured on every backend. `crates/gascan-conformance/src/lib.rs:114-123` records that open question in the contract itself. | +| `create_collision_reports_resources_created_before_the_collision` | `seed_volume`, fake-only. The assertion's entire content is that a failure reports exactly the resources built before a **planted** collision at a chosen index. Creating twice does produce a collision on a real backend, but with the same request — so the reported names would be the live sandbox's own, and what a same-request collision reports is unmeasured on every backend. `crates/gascan-conformance/src/lib.rs:149-164` records that open question in the contract itself, and names `gascan-apple/tests/live/storage.rs:22-37` as the precedent not to copy. | | `offline_fake_create_has_no_managed_network` | Not machinery — fixture shape. The contract is one walk over one fixture, and arca's must be `network = 'networked'`: offline is the capability the pinned engine is proven not to honour (`docs/evidence/2026-08-18-arca-engine-offline.md`). An unconditional "no managed network" assertion fails for a networked fixture on *every* backend, so promoting it needs the contract to branch on the fixture's network. That is a design change, and it is not made here. | | `networked_fake_create_reports_network_then_volumes_then_container` | Same fixture-conditionality — the network element exists only for a networked fixture — and, separately, **nothing owes the ordering**. `RemoveRequest::from_resources` does not reorder (`crates/gascan-core/src/runtime.rs:1001-1017`), yet the fake's recorded removal comes out container / volume / network, so re-ordering is the backend's job and no consumer reads `created()` positionally. Arca's list is in whatever order the engine's `CreateResponse` carried (`crates/gascan-arca/src/backend.rs:80-108`) — an unmeasured property of a pinned external binary. **Correction to the plan's candidate table**, which says this ordering "is asserted through the fake's call recorder": it is not. The test reads `outcome.created()` (`crates/gascan-core/tests/backend_contract.rs:509-517`) and touches neither `calls()` nor `outcomes()`. The verdict is unchanged; the stated reason was wrong. | | `persistent_logs_are_isolated_by_exact_sandbox_id` | `FakeRuntime::persistent`, named fake-only machinery, plus `fake-stdout` to get a marker into the log. Isolation-by-id also needs two live sandboxes and `backend_contract` takes one fixture, so promoting it would mean a second design change on top of the machinery. | @@ -231,10 +232,13 @@ instantiation does land in the live tier CI executes the tier needs four. `backend_contract_holds_on_arca` calls `base_oci_layout()`, whose absence is a `panic!` and never a skip, and the step's own comment records as a measurement that 20 of the 25 tests `--ignored` selects fail in 0.00s on exactly that missing variable and have done so since -milestone 2 — so the new test can only join them. That is a derivation, not an observed CI run. -The **apple** instantiation runs **nowhere in CI**; no workflow step -passes `--ignored` for `gascan-apple` or `gascan-e2e`, and line 178 is the only `--ignored` in the -file. Both real-backend results are therefore local-only, and no claim that a real backend passes or +milestone 2 — so the new test can only join the failing majority. That is a derivation, not an +observed CI run. **That `25` is from an earlier revision**: re-derived at HEAD, +`grep -rc '#\[ignore' crates/gascan-arca/tests/live/*.rs` sums to **29**, one of them added by this +plan. The **apple** instantiation runs **nowhere in CI**; no workflow step +passes `--ignored` for `gascan-apple` or `gascan-e2e`, and of the three `--ignored` occurrences in +`.github/workflows/ci.yml` — `:139` and `:162` inside comments, `:178` in arca's step — only `:178` +is executed. Both real-backend results are therefore local-only, and no claim that a real backend passes or fails conformance should be made without naming the machine and the date. What CI *does* hold is that the tests are still wired in, via `scripts/ci-check-ignored-tests.sh`; that is existence, not execution. `docs/evidence/2026-08-20-backend-conformance.md` is where the local measurements live.