diff --git a/docs/bwrap-support/bubblewrap-backend.md b/docs/bwrap-support/bubblewrap-backend.md index 744e5b0d9..c293f9075 100644 --- a/docs/bwrap-support/bubblewrap-backend.md +++ b/docs/bwrap-support/bubblewrap-backend.md @@ -673,6 +673,41 @@ request fails if its private namespace cannot be configured. the `allowedHosts` / `blockedHosts` lists. Non-cooperating clients are not merely unrouted — their traffic is dropped by the egress chain. +### Losing the network provider mid-run + +`slirp4netns` carries the sandbox's only route, so a slirp that dies under a +running workload leaves the sandbox running against a dead network: every +connection fails with a generic transport error, and the run is attributed to +whatever the workload reported. Two checks close that, and both apply to +firewall-enforcement mode as well, since it stands up the same supervisor and +the same slirp. + +**Before the workload starts.** Readiness is latched, not revoked — it says +slirp *came up*, not that it is still up — so it can already be stale by the +time the startup gate opens. The supervisor is re-checked immediately before +the gate is released. Any exit fails the run, including a successful one: +slirp's exit code says nothing about whether the sandbox still has a route. + +**For the lifetime of the workload.** The supervisor inherits the write end of +a pipe nothing ever writes to, and slirp inherits it in turn; the runner keeps +only the read end. That descriptor reaches EOF when *both* have exited, which +is what separates a dead network from an orphaned slirp still carrying traffic +after its supervisor was killed. A monitor thread in the executor watches the +descriptor, terminates the sandbox when it closes, and fails the run naming the +supervisor's exit status and a bounded tail of its stderr: + +```text +wait failed: Bubblewrap: the sandbox lost its network provider while the +workload was running; the proxy network supervisor exited with exit status: 137 +(stderr: sent tapfd=7 for tap0 +received tapfd=7 +Killed) +``` + +The monitor is disarmed *before* teardown stops the supervisor, so an ordinary +shutdown — which closes the same descriptor — is never reported as a loss. Only +an exit is detected; see [Limitations](#limitations). + ### Example: builtin test proxy with allowlist ```json @@ -932,3 +967,6 @@ Test configs are in `tests/configs/bubblewrap_*.json`. retained for compatibility but does not filter unprivileged. - **No state-aware lifecycle** — Bubblewrap implements `ScriptRunner` only (one-shot), not `StatefulSandboxBackend` +- **Provider-loss detection is exit-based** — in the private-namespace modes a + `slirp4netns` that exits mid-run fails the run; one that is alive but wedged + is not detected, and reaches the workload as an unreachable network diff --git a/src/backends/bubblewrap/common/src/bwrap_runner.rs b/src/backends/bubblewrap/common/src/bwrap_runner.rs index 9a9d69d03..2eb6a37d6 100644 --- a/src/backends/bubblewrap/common/src/bwrap_runner.rs +++ b/src/backends/bubblewrap/common/src/bwrap_runner.rs @@ -30,8 +30,9 @@ use std::fmt::Write as FmtWrite; use std::os::fd::AsFd; use std::os::unix::process::CommandExt; use std::path::{Component, Path, PathBuf}; -use std::process::{Child, ChildStdin, Command, Stdio}; -use std::time::Duration; +use std::process::{Child, ChildStdin, Command, ExitStatus, Stdio}; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; +use std::time::{Duration, Instant}; use lxc_common::network_iptables::{EgressHookPoint, NetworkIptablesManager}; use wxc_common::interruptible_reader::{wrap_pipe, InterruptibleReader, ReadCanceller}; @@ -39,8 +40,8 @@ use wxc_common::logger::Logger; use wxc_common::models::{ExecutionRequest, ScriptResponse}; use wxc_common::sandbox_process::{ boxed_closer, cancel_and_join_discard, duplicate_and_take_native_stdio, group_kill, - spawn_discard, take_boxed_read, take_boxed_write, wait_with_timeout, NativeStdio, - SandboxBackend, SandboxProcess, StdioMode, StreamCloser, WaitError, + spawn_discard, take_boxed_read, take_boxed_write, NativeStdio, SandboxBackend, SandboxProcess, + StdioMode, StreamCloser, }; use wxc_common::unix_proxy_coordinator::UnixProxyCoordinator; use wxc_common::validator::{ @@ -49,7 +50,9 @@ use wxc_common::validator::{ use crate::{ bwrap_command::{self, ResolvedNetworkMode}, - bwrap_version, network_rules, proxy_network, + bwrap_version, network_rules, + provider_monitor::ProviderMonitor, + proxy_network, }; /// Bubblewrap sandbox runner. Uses only shared `ContainerPolicy` fields — @@ -640,13 +643,11 @@ impl BubblewrapScriptRunner { let startup_result = startup .child_pid(&mut child) .and_then(|child_pid| { - proxy_network - .as_mut() - .ok_or_else(|| { - "Bubblewrap: proxy network lifecycle disappeared during startup" - .to_string() - })? - .attach(child_pid, logger) + let network = proxy_network.as_mut().ok_or_else(|| { + "Bubblewrap: proxy network lifecycle disappeared during startup".to_string() + })?; + network.attach(child_pid, logger)?; + network.check_alive() }) .and_then(|()| startup.release()); if let Err(error) = startup_result { @@ -693,6 +694,14 @@ impl BubblewrapScriptRunner { Some(Duration::from_millis(u64::from(request.script_timeout))) }; + let child = Arc::new(Mutex::new(child)); + // Armed only now: until the gate was released a dead provider surfaced + // as a startup failure instead. + let monitor = proxy_network + .as_mut() + .and_then(|network| network.take_liveness_watch()) + .map(|watch| ProviderMonitor::watch(watch, Arc::clone(&child), group)); + Ok(BwrapChild { child, stdin, @@ -704,6 +713,7 @@ impl BubblewrapScriptRunner { proxy, proxy_network, fw_manager, + monitor, timeout, }) } @@ -712,7 +722,9 @@ impl BubblewrapScriptRunner { /// A spawned `bwrap` sandbox: the child process, its parent-side pipe ends, /// and the per-run network proxy / iptables state torn down once it exits. struct BwrapChild { - child: Child, + /// The lock is what keeps [`ProviderMonitor`]'s thread from signalling a + /// pid this side has already reaped. + child: Arc>, stdin: Option, stdout: Option, stderr: Option, @@ -727,13 +739,22 @@ struct BwrapChild { proxy: UnixProxyCoordinator, proxy_network: Option, fw_manager: Option, + monitor: Option, timeout: Option, } impl BwrapChild { + fn lock_child(&self) -> MutexGuard<'_, Child> { + self.child.lock().unwrap_or_else(PoisonError::into_inner) + } + /// Tear down per-run network state (iptables rules + proxy). Idempotent at /// the manager level. fn cleanup(&mut self, logger: &mut Logger) { + // Stopping the supervisor closes the descriptor the monitor watches, so + // disarming first is what keeps a normal teardown from reading as a + // provider that died. + self.monitor.take(); cleanup_iptables(&mut self.fw_manager, logger); if let Some(mut network) = self.proxy_network.take() { network.stop(logger); @@ -766,6 +787,66 @@ impl BubblewrapSandboxProcess { let mut logger = Logger::new(wxc_common::logger::Mode::Buffer); self.inner.cleanup(&mut logger); } + + fn provider_lost(&self) -> bool { + self.inner + .monitor + .as_ref() + .is_some_and(ProviderMonitor::provider_lost) + } + + /// Wait for the sandbox to exit, its deadline to pass, or its network + /// provider to disappear. + fn await_outcome(&mut self) -> BwrapOutcome { + const MIN_POLL: Duration = Duration::from_millis(1); + const MAX_POLL: Duration = Duration::from_millis(50); + + let deadline = self.inner.timeout.map(|timeout| Instant::now() + timeout); + let mut interval = MIN_POLL; + loop { + let exited = match self.inner.lock_child().try_wait() { + Ok(exited) => exited, + Err(error) => return BwrapOutcome::Io(error), + }; + // Checked after the exit rather than before it: the monitor kills + // the sandbox, so a provider death shows up here as an ordinary + // exit that would otherwise be reported as the workload's own. + if self.provider_lost() { + return BwrapOutcome::ProviderLost; + } + if let Some(status) = exited { + return BwrapOutcome::Exited(status); + } + + let now = Instant::now(); + let mut nap = interval; + if let Some(deadline) = deadline { + if now >= deadline { + return BwrapOutcome::Timeout; + } + nap = nap.min(deadline - now); + } + std::thread::sleep(nap); + interval = (interval * 2).min(MAX_POLL); + } + } + + fn lost_provider_message(&mut self) -> String { + match self.inner.proxy_network.as_mut() { + Some(network) => network.lost_provider_detail(), + None => "Bubblewrap: the sandbox lost its network provider while the workload \ + was running" + .to_string(), + } + } +} + +/// How a `bwrap` run ended. +enum BwrapOutcome { + Exited(ExitStatus), + Timeout, + ProviderLost, + Io(std::io::Error), } impl SandboxProcess for BubblewrapSandboxProcess { @@ -808,32 +889,33 @@ impl SandboxProcess for BubblewrapSandboxProcess { fn try_wait(&mut self) -> std::io::Result> { Ok(self .inner - .child + .lock_child() .try_wait()? .map(|status| status.code().unwrap_or(-1))) } fn id(&self) -> u32 { - self.inner.child.id() + self.inner.lock_child().id() } fn kill(&mut self) -> std::io::Result<()> { // No-op once the child has exited and been reaped: its pid/pgid can be // recycled, so signaling it could hit an unrelated process (group). A // reaped `Child` returns its cached status here without a syscall. - if self.inner.child.try_wait()?.is_some() { + let mut child = self.inner.lock_child(); + if child.try_wait()?.is_some() { return Ok(()); } if self.inner.group { // Pipes mode: bwrap leads its own process group — tree-kill it. - group_kill(&mut self.inner.child) + group_kill(&mut child) } else { // Inherit mode: bwrap shares the executor's group (no // `process_group(0)`), so a group-kill would hit the executor. // Killing bwrap alone suffices because `--die-with-parent` makes // the sandbox die with it — bwrap is *not* pid 1 of the namespace // (it forks), so without that flag descendants would survive. - self.inner.child.kill() + child.kill() } } @@ -847,26 +929,31 @@ impl SandboxProcess for BubblewrapSandboxProcess { let stdout_thread = spawn_discard(self.inner.stdout.take()); let stderr_thread = spawn_discard(self.inner.stderr.take()); - let result = match wait_with_timeout(&mut self.inner.child, self.inner.timeout) { - Ok(status) => Ok(status.code().unwrap_or(-1)), - Err(WaitError::Timeout) => { + let result = match self.await_outcome() { + BwrapOutcome::Exited(status) => Ok(status.code().unwrap_or(-1)), + BwrapOutcome::ProviderLost => { + let _ = self.kill(); + let _ = self.inner.lock_child().wait(); + Err(std::io::Error::other(self.lost_provider_message())) + } + BwrapOutcome::Timeout => { // Tree-kill so descendants die too and release any stdout/stderr // pipe write-ends (else the drain threads below could block). // `kill()` group-kills in Pipes mode; in Inherit mode it kills // bwrap, which `--die-with-parent` turns into a full teardown. let _ = self.kill(); - let _ = self.inner.child.wait(); + let _ = self.inner.lock_child().wait(); Err(std::io::Error::new( std::io::ErrorKind::TimedOut, "Bubblewrap: script timed out", )) } - Err(WaitError::Io(error)) => { + BwrapOutcome::Io(error) => { // The child may still be alive; kill+reap it before // `run_teardown()` removes the iptables/proxy enforcement out // from under it. let _ = self.kill(); - let _ = self.inner.child.wait(); + let _ = self.inner.lock_child().wait(); Err(std::io::Error::other(format!( "Bubblewrap: wait failed: {error}" ))) @@ -882,13 +969,16 @@ impl SandboxProcess for BubblewrapSandboxProcess { impl Drop for BubblewrapSandboxProcess { fn drop(&mut self) { + // Disarmed first so the monitor cannot be holding the child lock while + // the reap below waits on it. + self.inner.monitor.take(); // Kill and reap the child *before* removing network enforcement — // otherwise an abandoned-but-running sandbox would keep egressing after // its iptables/proxy rules were torn down, and the child would leak as // a zombie. `kill()` group-kills in `Pipes` mode and relies on // `--die-with-parent` otherwise, then we reap. let _ = self.kill(); - let _ = self.inner.child.wait(); + let _ = self.inner.lock_child().wait(); self.run_teardown(); } } diff --git a/src/backends/bubblewrap/common/src/lib.rs b/src/backends/bubblewrap/common/src/lib.rs index 6e73e1fb6..a23351ffd 100644 --- a/src/backends/bubblewrap/common/src/lib.rs +++ b/src/backends/bubblewrap/common/src/lib.rs @@ -25,6 +25,9 @@ pub mod bwrap_version; pub(crate) mod network_rules; /// Shared by the `bwrap` version probe and the proxy dependency walk. pub(crate) mod probe_exec; +/// Watches the private-namespace network provider for the run's lifetime. +#[cfg(target_os = "linux")] +pub(crate) mod provider_monitor; /// Only [`proxy_network::probe_proxy_enforcement`] is reachable outside the /// crate; every other item stays `pub(crate)`. #[cfg(target_os = "linux")] diff --git a/src/backends/bubblewrap/common/src/provider_monitor.rs b/src/backends/bubblewrap/common/src/provider_monitor.rs new file mode 100644 index 000000000..bad75c245 --- /dev/null +++ b/src/backends/bubblewrap/common/src/provider_monitor.rs @@ -0,0 +1,215 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Watches the sandbox's network provider for the lifetime of the workload. + +use std::fs::File; +use std::io::{ErrorKind, Read}; +use std::os::fd::OwnedFd; +use std::process::Child; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread::{self, JoinHandle}; +use std::time::Duration; + +use wxc_common::sandbox_process::group_kill; + +/// Also the longest a disarmed monitor keeps its thread alive. +const POLL_INTERVAL: Duration = Duration::from_millis(50); + +/// Terminates the sandbox if its network provider dies while the workload runs. +/// +/// The private-namespace modes route the sandbox entirely through +/// `slirp4netns`, so losing it leaves the workload running with no network. +pub(crate) struct ProviderMonitor { + lost: Arc, + disarmed: Arc, + handle: Option>, +} + +impl ProviderMonitor { + /// Start watching `watch`, killing `child` if the provider goes away first. + pub(crate) fn watch(watch: OwnedFd, child: Arc>, group: bool) -> Self { + let lost = Arc::new(AtomicBool::new(false)); + let disarmed = Arc::new(AtomicBool::new(false)); + let thread_lost = Arc::clone(&lost); + let thread_disarmed = Arc::clone(&disarmed); + + let handle = thread::spawn(move || { + let mut watch = File::from(watch); + let mut byte = [0_u8; 1]; + let mut lost_provider = false; + loop { + match watch.read(&mut byte) { + // Every holder of the write end has exited, so neither the + // supervisor nor an orphaned slirp is left to carry traffic. + Ok(0) => { + lost_provider = true; + break; + } + Ok(_) => {} + Err(error) + if matches!( + error.kind(), + ErrorKind::WouldBlock | ErrorKind::Interrupted + ) => {} + Err(_) => break, + } + + // Read before this, never after: teardown disarms while the + // supervisor is still up, so an already-pending EOF is a real + // loss and must not be discarded here. + if thread_disarmed.load(Ordering::SeqCst) { + return; + } + thread::sleep(POLL_INTERVAL); + } + + if !lost_provider { + return; + } + thread_lost.store(true, Ordering::SeqCst); + if thread_disarmed.load(Ordering::SeqCst) { + return; + } + + let Ok(mut child) = child.lock() else { + return; + }; + // Signalling a reaped child could reach an unrelated process that + // inherited its pid; holding the lock keeps the answer true. + if matches!(child.try_wait(), Ok(None)) { + let _ = if group { + group_kill(&mut child) + } else { + child.kill() + }; + } + }); + + Self { + lost, + disarmed, + handle: Some(handle), + } + } + + /// Whether the provider was lost while the workload was running. + pub(crate) fn provider_lost(&self) -> bool { + self.lost.load(Ordering::SeqCst) + } + + /// Stop watching, and report whether a loss had already been observed. + pub(crate) fn disarm(&mut self) -> bool { + self.disarmed.store(true, Ordering::SeqCst); + if let Some(handle) = self.handle.take() { + let _ = handle.join(); + } + self.provider_lost() + } +} + +impl Drop for ProviderMonitor { + fn drop(&mut self) { + self.disarm(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use nix::fcntl::{fcntl, FcntlArg, OFlag}; + use nix::unistd::pipe2; + use std::os::fd::AsRawFd; + use std::process::Command; + use std::time::Instant; + + /// The read end must not block, or a disarmed monitor could never notice. + fn watch_pipe() -> (OwnedFd, OwnedFd) { + let (reader, writer) = pipe2(OFlag::O_CLOEXEC).expect("pipe"); + let flags = fcntl(reader.as_raw_fd(), FcntlArg::F_GETFL).expect("read descriptor flags"); + fcntl( + reader.as_raw_fd(), + FcntlArg::F_SETFL(OFlag::from_bits_truncate(flags) | OFlag::O_NONBLOCK), + ) + .expect("set descriptor nonblocking"); + (reader, writer) + } + + fn sleeping_sandbox() -> Arc> { + Arc::new(Mutex::new( + Command::new("sleep").arg("30").spawn().expect("spawn"), + )) + } + + fn exited_within(child: &Arc>, budget: Duration) -> bool { + let deadline = Instant::now() + budget; + while Instant::now() < deadline { + let exited = child + .lock() + .map(|mut child| matches!(child.try_wait(), Ok(Some(_)))) + .unwrap_or(false); + if exited { + return true; + } + thread::sleep(Duration::from_millis(10)); + } + false + } + + fn reap(child: &Arc>) { + if let Ok(mut child) = child.lock() { + let _ = child.kill(); + let _ = child.wait(); + } + } + + #[test] + fn a_lost_provider_takes_the_sandbox_down_with_it() { + let (watch, writer) = watch_pipe(); + let child = sleeping_sandbox(); + let mut monitor = ProviderMonitor::watch(watch, Arc::clone(&child), false); + + drop(writer); + + assert!( + exited_within(&child, Duration::from_secs(10)), + "the sandbox outlived the provider carrying its only route" + ); + assert!( + monitor.disarm(), + "the loss must reach the run rather than being reported as a workload exit" + ); + reap(&child); + } + + #[test] + fn a_disarmed_monitor_leaves_the_sandbox_alone() { + let (watch, writer) = watch_pipe(); + let child = sleeping_sandbox(); + let mut monitor = ProviderMonitor::watch(watch, Arc::clone(&child), false); + + monitor.disarm(); + drop(writer); + + assert!( + !exited_within(&child, Duration::from_millis(500)), + "teardown closing the watch must not be mistaken for a provider death" + ); + assert!(!monitor.provider_lost()); + reap(&child); + } + + #[test] + fn a_monitor_over_an_already_reaped_sandbox_still_settles() { + let (watch, writer) = watch_pipe(); + let child = Arc::new(Mutex::new(Command::new("true").spawn().expect("spawn"))); + reap(&child); + let mut monitor = ProviderMonitor::watch(watch, Arc::clone(&child), true); + + drop(writer); + + assert!(monitor.disarm(), "the loss is still worth reporting"); + } +} diff --git a/src/backends/bubblewrap/common/src/proxy_network.rs b/src/backends/bubblewrap/common/src/proxy_network.rs index 2180eeca6..d7ba0f9ad 100644 --- a/src/backends/bubblewrap/common/src/proxy_network.rs +++ b/src/backends/bubblewrap/common/src/proxy_network.rs @@ -114,6 +114,12 @@ const INGRESS_CHAIN: &str = "MXC_INGRESS"; /// a redirection, which is the whole reason the parent pins them. const SUPERVISOR_PID_FD: RawFd = 3; const SUPERVISOR_EXIT_FD: RawFd = 4; +/// Held open by the supervisor and inherited by slirp, never written to. +/// +/// The parent keeps only the read end, so it reaches EOF exactly when both +/// have exited -- an orphaned slirp still carries the sandbox's route, and +/// holding the descriptor is what keeps that case from reading as a loss. +const SUPERVISOR_LIVENESS_FD: RawFd = 5; /// Descriptors are staged above every target before being landed, so a source /// already sitting on a target cannot be clobbered mid-remap. const FD_STAGING_BASE: RawFd = 10; @@ -631,6 +637,9 @@ pub(crate) struct ProxyNetworkNamespace { userns: Option, /// Hosts file mounted over `/etc/hosts`, when the endpoint is a hostname. hosts: Option, + /// Read end of the descriptor the supervisor and slirp hold open, taken by + /// the monitor that watches for the network provider dying mid-run. + liveness_reader: Option, /// Restore transactions the supervisor will apply, which sizes the /// readiness budget in [`Self::attach`]. transactions: usize, @@ -689,6 +698,9 @@ impl ProxyNetworkNamespace { pipe2(OFlag::O_CLOEXEC).map_err(|error| format!("Bubblewrap: pipe failed: {error}"))?; let (pid_reader, pid_writer) = pipe2(OFlag::O_CLOEXEC).map_err(|error| format!("Bubblewrap: pipe failed: {error}"))?; + let (liveness_reader, liveness_writer) = + pipe2(OFlag::O_CLOEXEC).map_err(|error| format!("Bubblewrap: pipe failed: {error}"))?; + set_nonblocking(liveness_reader.as_raw_fd())?; let mut command = Command::new("unshare"); command @@ -715,6 +727,7 @@ impl ProxyNetworkNamespace { [ (pid_reader.as_raw_fd(), SUPERVISOR_PID_FD), (exit_reader.as_raw_fd(), SUPERVISOR_EXIT_FD), + (liveness_writer.as_raw_fd(), SUPERVISOR_LIVENESS_FD), ], ); @@ -724,6 +737,10 @@ impl ProxyNetworkNamespace { drop(exit_reader); drop(pid_reader); + // The supervisor tree is now the only holder of the write end, which is + // what makes the read end reach EOF when it dies. + drop(liveness_writer); + if let Err(error) = wait_for_file( state_dir.path().join("userns.ready"), &mut supervisor, @@ -771,6 +788,7 @@ impl ProxyNetworkNamespace { pid_writer: Some(pid_writer), userns: Some(userns), hosts, + liveness_reader: Some(liveness_reader), transactions, script_timeout_ms, }) @@ -865,6 +883,50 @@ impl ProxyNetworkNamespace { Ok(()) } + /// Confirm the network provider is still alive, immediately before the + /// workload's startup gate opens. + /// + /// `slirp.ready` is a latch that stays on disk after slirp dies, so the + /// readiness [`Self::attach`] waited for can already be stale here. + pub(crate) fn check_alive(&mut self) -> Result<(), String> { + match self.supervisor.try_wait() { + Ok(None) => Ok(()), + + // The supervisor's last act is to wait on slirp, so it outlives + // slirp and any exit at all leaves the sandbox with no route. + Ok(Some(status)) => Err(format!( + "Bubblewrap: the proxy network supervisor exited ({status}) after signalling \ + readiness but before the workload started, leaving the sandbox with no \ + network provider ({})", + stderr_detail(&self.state_dir.path().join("supervisor.stderr")) + )), + Err(error) => Err(format!( + "Bubblewrap: failed to inspect the proxy network supervisor before starting \ + the workload: {error}" + )), + } + } + + /// Take the descriptor a [`ProviderMonitor`] watches for provider loss. + pub(crate) fn take_liveness_watch(&mut self) -> Option { + self.liveness_reader.take() + } + + /// Describe the provider loss a monitor observed, for the error the run + /// fails with. + pub(crate) fn lost_provider_detail(&mut self) -> String { + let status = match self.supervisor.try_wait() { + Ok(Some(status)) => format!("exited with {status}"), + Ok(None) => "is still running, so slirp4netns died on its own".to_string(), + Err(error) => format!("could not be inspected: {error}"), + }; + format!( + "Bubblewrap: the sandbox lost its network provider while the workload was \ + running; the proxy network supervisor {status} ({})", + stderr_detail(&self.state_dir.path().join("supervisor.stderr")) + ) + } + /// Stop slirp and reap the namespace supervisor. pub(crate) fn stop(&mut self, logger: &mut Logger) { // Release the PID pipe too: a supervisor still waiting for the sandbox @@ -4640,4 +4702,83 @@ exec sleep 30 ); assert_eq!(supervisor.rule_invocations(), 0); } + + /// A namespace whose supervisor is `script`, every other field inert, for + /// exercising the pre-gate liveness check on its own. + fn namespace_running(script: &str) -> ProxyNetworkNamespace { + let state_dir = tempfile::tempdir().expect("tempdir"); + let stderr = File::create(state_dir.path().join("supervisor.stderr")).expect("stderr"); + let supervisor = Command::new("sh") + .args(["-c", script]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::from(stderr)) + .spawn() + .expect("spawn supervisor"); + + ProxyNetworkNamespace { + state_dir, + supervisor, + exit_writer: None, + pid_writer: None, + userns: None, + hosts: None, + liveness_reader: None, + transactions: 0, + script_timeout_ms: 0, + } + } + + fn await_exit(namespace: &mut ProxyNetworkNamespace) { + let deadline = Instant::now() + Duration::from_secs(20); + while Instant::now() < deadline { + if namespace.supervisor.try_wait().expect("try_wait").is_some() { + return; + } + thread::sleep(Duration::from_millis(10)); + } + panic!("supervisor did not exit"); + } + + #[test] + fn the_gate_check_passes_while_the_supervisor_is_running() { + let mut network = namespace_running("sleep 30"); + + assert!( + network.check_alive().is_ok(), + "a running supervisor must not block the workload" + ); + } + + #[test] + fn the_gate_check_refuses_a_supervisor_that_died_after_signalling_readiness() { + let mut network = namespace_running("echo 'slirp4netns: crashed' >&2; exit 1"); + await_exit(&mut network); + + let error = network + .check_alive() + .expect_err("a dead supervisor must not reach the workload"); + assert!( + error.contains("no network provider"), + "the failure must name the lost provider rather than read as a workload \ + error: {error}" + ); + assert!( + error.contains("slirp4netns: crashed"), + "the failure must carry what the supervisor wrote: {error}" + ); + } + + /// Slirp carries the sandbox's only route, so its exit code says nothing + /// about whether the sandbox still has a network. + #[test] + fn the_gate_check_refuses_even_a_supervisor_that_exited_cleanly() { + let mut network = namespace_running("exit 0"); + await_exit(&mut network); + + assert!( + network.check_alive().is_err(), + "a successful exit still leaves the sandbox with no route" + ); + } } diff --git a/tests/configs/bubblewrap_provider_loss.json b/tests/configs/bubblewrap_provider_loss.json new file mode 100644 index 000000000..edc0e1316 --- /dev/null +++ b/tests/configs/bubblewrap_provider_loss.json @@ -0,0 +1,19 @@ +{ + "version": "0.8.0-alpha", + "containerId": "CLI-Bubblewrap-Provider-Loss", + "containment": "bubblewrap", + "process": { + "commandLine": "echo PROVIDER_LOSS_STARTED; /bin/sleep 985", + "timeout": 90000 + }, + "network": { + "egress": { + "default": "deny", + "allow": [{ "to": [{ "cidr": "203.0.113.0/24" }] }] + }, + "ingress": { + "default": "deny", + "hostLoopback": "deny" + } + } +} diff --git a/tests/scripts/run_bwrap_all_tests.sh b/tests/scripts/run_bwrap_all_tests.sh index 55958976f..f3793adeb 100644 --- a/tests/scripts/run_bwrap_all_tests.sh +++ b/tests/scripts/run_bwrap_all_tests.sh @@ -83,6 +83,7 @@ run_test "Bubblewrap Filesystem" "$SCRIPT_DIR/run_bwrap_filesystem_test.sh" run_test "Bubblewrap Read-Only Denial" "$SCRIPT_DIR/run_bwrap_readonly_denial_test.sh" run_test "Bubblewrap Version Gate" "$SCRIPT_DIR/run_bwrap_version_gate_test.sh" run_test "Bubblewrap Teardown" "$SCRIPT_DIR/run_bwrap_teardown_test.sh" +run_test "Bubblewrap Provider Loss" "$SCRIPT_DIR/run_bwrap_provider_loss_test.sh" run_test "Bubblewrap Object Validation" "$SCRIPT_DIR/run_bwrap_filesystem_object_test.sh" run_test "Bubblewrap Most-Specific Path" "$SCRIPT_DIR/run_bwrap_most_specific_test.sh" run_test "Bubblewrap Denied Masking" "$SCRIPT_DIR/run_bwrap_denied_masking_test.sh" diff --git a/tests/scripts/run_bwrap_provider_loss_test.sh b/tests/scripts/run_bwrap_provider_loss_test.sh new file mode 100644 index 000000000..7aeff91b4 --- /dev/null +++ b/tests/scripts/run_bwrap_provider_loss_test.sh @@ -0,0 +1,101 @@ +#!/bin/bash +# Bubblewrap network-provider loss test. +# +# The 0.8 private-namespace modes give the workload a network whose only route +# is slirp4netns. Readiness was checked once at startup and never again, so +# killing slirp mid-run left the sandbox running against a dead network: every +# connection failed with a generic transport error and the run was attributed +# to whatever the workload reported. This test kills slirp under a live +# workload and pins the failure that replaced that silence. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_DIR="$(dirname "$(dirname "$SCRIPT_DIR")")" +LXC_EXEC="$REPO_DIR/src/target/release/lxc-exec" + +if [ ! -f "$LXC_EXEC" ]; then + LXC_EXEC="$REPO_DIR/src/target/debug/lxc-exec" +fi + +if [ ! -f "$LXC_EXEC" ]; then + echo "Error: lxc-exec not found. Run build.sh first." + exit 1 +fi + +fail() { echo "FAIL: $1"; exit 1; } + +if ! command -v slirp4netns >/dev/null 2>&1; then + echo "SKIP: slirp4netns not installed; this test needs the private-namespace path." + exit 77 +fi + +WORKLOAD_MARKER="sleep 985" +LOG=$(mktemp) +RUN_PID="" + +cleanup() { + [ -n "$RUN_PID" ] && kill -9 "$RUN_PID" 2>/dev/null || true + pkill -f "$WORKLOAD_MARKER" 2>/dev/null || true + rm -f "$LOG" +} +trap cleanup EXIT + +echo "Running Bubblewrap provider-loss test: a slirp4netns killed mid-run fails the run..." + +# Every slirp already on this host, so the one this run adds can be told apart +# from a developer's or a concurrent suite's. +before=$(pgrep -x slirp4netns 2>/dev/null | sort || true) + +"$LXC_EXEC" --experimental "$REPO_DIR/tests/configs/bubblewrap_provider_loss.json" \ + >"$LOG" 2>&1 & +RUN_PID=$! + +# The kill must land while the workload is running, so wait for the workload's +# own marker rather than for the executor to merely have been spawned. +deadline=$((SECONDS + 60)) +while ! grep -qF "PROVIDER_LOSS_STARTED" "$LOG" 2>/dev/null; do + if [ $SECONDS -ge $deadline ]; then + cat "$LOG" + fail "the workload never started, so this proves nothing" + fi + if ! kill -0 "$RUN_PID" 2>/dev/null; then + cat "$LOG" + fail "the run exited before the workload started" + fi + sleep 0.1 +done + +after=$(pgrep -x slirp4netns 2>/dev/null | sort || true) +victim=$(comm -13 <(echo "$before") <(echo "$after") | head -1) + +if [ -z "$victim" ]; then + cat "$LOG" + fail "no slirp4netns was started for this run, so there is nothing to kill" +fi + +kill -9 "$victim" 2>/dev/null || fail "could not kill slirp4netns ($victim)" + +rc=0 +wait "$RUN_PID" || rc=$? +RUN_PID="" +out=$(cat "$LOG") + +if [ "$rc" = 0 ]; then + echo "$out" + fail "the run succeeded despite losing the network its workload depended on" +fi +if ! grep -qF "lost its network provider" <<<"$out"; then + echo "$out" + fail "exited $rc without naming the lost provider, which is the whole point" +fi + +# The sandbox has to actually be gone: reporting the loss while leaving the +# workload running against a dead network would be the same bug with a message. +sleep 2 +if pgrep -f "$WORKLOAD_MARKER" >/dev/null 2>&1; then + pkill -f "$WORKLOAD_MARKER" || true + fail "the workload outlived the provider whose loss was reported" +fi + +echo "PASS: a slirp4netns killed mid-run fails the run" +echo "All Bubblewrap provider-loss tests passed."