P5.3: the backend conformance suite, and what it measured on each backend

Cargo.toml

@@ -1,5 +1,5 @@
 [workspace]
-members = ["crates/gascan", "crates/gascan-apple", "crates/gascan-arca", "crates/gascan-core", "crates/gascan-e2e", "crates/gascan-engine-proto", "crates/gascan-inherited-fd", "crates/gascan-oci-fixture", "crates/gascan-proto", "crates/gascand"]
+members = ["crates/gascan", "crates/gascan-apple", "crates/gascan-arca", "crates/gascan-conformance", "crates/gascan-core", "crates/gascan-e2e", "crates/gascan-engine-proto", "crates/gascan-inherited-fd", "crates/gascan-oci-fixture", "crates/gascan-proto", "crates/gascand"]
 resolver = "3"
 
 [workspace.package]

crates/gascan-apple/Cargo.toml

@@ -17,6 +17,7 @@ tokio.workspace = true
 thiserror.workspace = true
 
 [dev-dependencies]
+gascan-conformance = { path = "../gascan-conformance" }
 libc = "0.2"
 tempfile = "3"
 tokio = { workspace = true, features = ["macros", "rt", "time"] }

crates/gascan-apple/tests/live/backend_contract.rs

@@ -1,59 +1,39 @@
-use std::time::{SystemTime, UNIX_EPOCH};
-
-use camino::Utf8Path;
 use gascan_apple::{AppleBackend, ProcessRunner};
-use gascan_core::{
-    manifest::Manifest,
-    policy::PolicyCompiler,
-    runtime::{
-        NetworkIsolation, RemoveRequest, RuntimeBackend, RuntimeCapabilities, RuntimeVersion,
-    },
-    sandbox::SandboxSpec,
-};
+use gascan_conformance::{CreateRequestFixture, backend_contract};
+use gascan_core::runtime::RuntimeBackend;
+use std::time::{SystemTime, UNIX_EPOCH};
 
+/// The shared backend contract, run against a real `container` CLI.
+///
+/// **THIS TEST FAILS TODAY AND THE FAILURE IS THE RECORDED RESULT, NOT A
+/// REGRESSION.** MEASURED on `newcombe` 2026-08-20 with `container` 1.1.0:
+/// `panicked at gascan-conformance/src/lib.rs:104:5 ... left: Running, right:
+/// Stopped` -- apple's `create` compiles to `container run`
+/// (`gascan-apple/src/translate.rs:100`), so there is no window in which it has
+/// produced a `Stopped` container. Everything after that assertion, including
+/// the `list_resources` tail below, was NOT REACHED. **Do not weaken the
+/// assertion to make this green**; see
+/// `docs/evidence/2026-08-20-backend-conformance.md` and
+/// `docs/status/START-HERE.md` open item 10.
+///
+/// The panic quoted above says `104` because that is where the assertion sat
+/// when it was measured; the comment now standing over it moved the assertion
+/// down. Re-derive the line rather than trusting either number.
+///
+/// **No CI job runs this tier**, so that measurement is the only evidence that
+/// will exist until someone runs it again by hand on a machine with the
+/// `container` service.
 #[tokio::test]
 #[ignore = "requires Apple silicon macOS 26+ with container service and locked workspace image"]
-async fn backend_contract() {
+async fn backend_contract_holds_on_apple() {
     let nonce = SystemTime::now()
         .duration_since(UNIX_EPOCH)
         .unwrap()
         .as_nanos();
     let name = format!("gascan-live-backend-{}-{nonce}", std::process::id());
-    let root = tempfile::tempdir().unwrap();
-    let path = Utf8Path::from_path(root.path()).unwrap();
-    std::fs::write(
-        path.join("gascan.toml"),
-        "version = 1\nnetwork = 'offline'\n",
-    )
-    .unwrap();
-    let spec = SandboxSpec::from_root(&name, path, Manifest::load(path).unwrap()).unwrap();
-    let request = PolicyCompiler::compile(
-        spec,
-        &RuntimeCapabilities {
-            version: RuntimeVersion::new(1, 1, 0),
-            bind_mounts: true,
-            named_volumes: true,
-            tty: true,
-            signals: true,
-            loopback_publish: true,
-            resource_limits: true,
-            offline: NetworkIsolation::Proven,
-        },
-    )
-    .unwrap();
-    let id = request.id().clone();
+    let fixture = CreateRequestFixture::pinned(&name, "offline");
     let backend = AppleBackend::new(ProcessRunner);
-    assert!(backend.inspect(&id).await.unwrap().is_none());
-    let created = backend.create(request).await.unwrap();
-    backend.start(&id).await.unwrap();
-    backend.start(&id).await.unwrap();
-    backend.stop(&id).await.unwrap();
-    backend.stop(&id).await.unwrap();
-    backend
-        .remove(RemoveRequest::from_resources(created.created().to_vec()).unwrap())
-        .await
-        .unwrap();
-    assert!(backend.inspect(&id).await.unwrap().is_none());
+    backend_contract(&backend, &fixture).await;
     assert!(
         !backend
             .list_resources()

crates/gascan-arca/Cargo.toml

@@ -19,6 +19,7 @@ tower.workspace = true
 
 [dev-dependencies]
 camino.workspace = true
+gascan-conformance = { path = "../gascan-conformance" }
 gascan-oci-fixture.workspace = true
 rustix.workspace = true
 serde_json.workspace = true

crates/gascan-arca/tests/live.rs

@@ -1,5 +1,7 @@
 #[path = "live/common/mod.rs"]
 mod common;
+#[path = "live/conformance.rs"]
+mod conformance;
 #[path = "live/connect.rs"]
 mod connect;
 #[path = "live/exec.rs"]

crates/gascan-arca/tests/live/conformance.rs

@@ -0,0 +1,72 @@
+use crate::common::{LiveEngine, base_oci_layout, layout_running};
+use camino::Utf8Path;
+use gascan_arca::ArcaBackend;
+use gascan_conformance::{CreateRequestFixture, backend_contract};
+
+/// The tag the derived layout is loaded under.
+const TAG: &str = "gascan-conformance:latest";
+
+/// `user = 'root'` because the base layout is a stock alpine with no
+/// `workspace` account -- see `lifecycle.rs`'s note on the same constant.
+///
+/// `network = 'networked'` and not `'offline'`: offline is the one capability
+/// this engine is known NOT to honour
+/// (`docs/evidence/2026-08-18-arca-engine-offline.md`), so an offline request
+/// would test the refuted property by accident.
+const MANIFEST: &str = "version = 1\nnetwork = 'networked'\nuser = 'root'\n";
+
+/// The shared backend contract, run against a real `arca-engine`.
+///
+/// **The image is `engine.image(TAG)` and not `TAG`**, which is why the engine
+/// is started before the fixture is built. `PolicyCompiler::compile_for_image`
+/// refuses a mutable reference outright (`gascan-core/src/policy.rs:179`), and
+/// `LiveEngine::image` is what turns the seeded tag into the store's own
+/// `repository@sha256:...`. MEASURED, on `newcombe` 2026-08-20 with the bare
+/// tag: the test panicked at `gascan-conformance/src/lib.rs:57` with
+/// `compile backend-contract policy: InvalidWorkspaceImage`, in 0.66s -- before
+/// a single call reached the backend. A fixture that cannot be built measures
+/// nothing about arca.
+#[tokio::test]
+#[ignore = "requires a built arca-engine, a kernel, a vminit layout and a base OCI layout"]
+async fn backend_contract_holds_on_arca() {
+    let temp = tempfile::tempdir().expect("a temporary layout root");
+    let destination = Utf8Path::from_path(temp.path()).expect("a utf-8 temporary path");
+    // `sh -c 'while :; do sleep 1; done'` and not the base image's own `Cmd`:
+    // alpine's is `/bin/sh`, which exits immediately with no tty attached, and
+    // the contract does start -> exec -> stop, so the container has to still be
+    // there. `lifecycle.rs:33-53` carries the measured note on this exact `Cmd`.
+    // It does not matter yet -- the contract fails before `start` -- and it
+    // starts mattering the day it gets that far.
+    let layout = layout_running(
+        &base_oci_layout(),
+        destination,
+        TAG,
+        &["sh", "-c", "while :; do sleep 1; done"],
+    );
+    let engine = LiveEngine::start_with_images(&[layout.as_path()]).await;
+    let backend = ArcaBackend::new(engine.transport().await);
+    let fixture = CreateRequestFixture::for_image("conformance", &engine.image(TAG), MANIFEST);
+    backend_contract(&backend, &fixture).await;
+
+    // `kill()` and not a bare drop, matching every other terminating test in
+    // this tier, because its exit-status assertion is deliberately spread
+    // across all of them: "This assertion is what stops that regressing, and it
+    // is here rather than only in `shutdown.rs` because every test in this tier
+    // stops an engine" (`common/mod.rs:473-477`), guarding an abort that ran at
+    // 6 crashes in 192 runs before the engine fix (`:462-471`).
+    //
+    // **It does not execute today**, and that is not a reason to leave it out.
+    // `backend_contract` panics at `gascan-conformance/src/lib.rs:104`, so this
+    // line is unreachable until arca's post-`create` state stops being
+    // `Creating`. What it buys is that the day the contract gets past that
+    // assertion, this test is already inside the tier's shutdown guard rather
+    // than a silent exception to it -- and `kill()` is also the only thing that
+    // prints the engine's own drained stdout/stderr (`exit.diagnostics`,
+    // `common/mod.rs:493`).
+    //
+    // The cost, stated plainly: while the contract fails where it does, this
+    // run discards the engine's account of the `create` it is measuring.
+    // Recovering it on the red path would mean catching the panic around the
+    // contract call, which is a restructuring this test does not justify.
+    engine.kill().await;
+}

crates/gascan-conformance/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "gascan-conformance"
+version = "0.1.20"
+edition.workspace = true
+license.workspace = true
+rust-version.workspace = true
+publish = false
+
+[dependencies]
+camino.workspace = true
+gascan-core = { path = "../gascan-core" }
+tempfile = "3"
+
+[dev-dependencies]
+tokio = { workspace = true, features = ["macros", "rt", "rt-multi-thread", "sync", "time"] }
+
+[lints]
+workspace = true