From 3136b2eb2fec5bc3d56355a60c32bdb048d28261 Mon Sep 17 00:00:00 2001 From: Luca Bertelli Date: Sun, 4 Oct 2026 09:36:01 +0200 Subject: [PATCH 1/2] feat: require the Firecracker jailer for every microVM (#215) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Require jailer for every cold boot and restore. Existing direct VMMs keep serving until redeployment; their old snapshots cold boot with the same volume. Review order: #219 → #220 → #221 → #222 → #223 → #224 → #225 → #226 → #227 → #228 → #215 → #229 → #230. --- crates/nibrunnerd/src/adapters/vm/jailer.rs | 278 ++++++++++++++++++ crates/nibrunnerd/src/adapters/vm/manager.rs | 217 ++++++++++++-- crates/nibrunnerd/src/adapters/vm/mod.rs | 3 +- .../src/adapters/vm/mount_namespace.rs | 68 ++++- crates/nibrunnerd/src/adapters/vm/process.rs | 89 ++++-- crates/nibrunnerd/src/run.rs | 9 +- crates/nibrunnerd/src/test_support/machine.rs | 3 + 7 files changed, 615 insertions(+), 52 deletions(-) create mode 100644 crates/nibrunnerd/src/adapters/vm/jailer.rs diff --git a/crates/nibrunnerd/src/adapters/vm/jailer.rs b/crates/nibrunnerd/src/adapters/vm/jailer.rs new file mode 100644 index 00000000..d50059ca --- /dev/null +++ b/crates/nibrunnerd/src/adapters/vm/jailer.rs @@ -0,0 +1,278 @@ +#[cfg(target_os = "linux")] +use std::ffi::CString; +use std::path::{Path, PathBuf}; + +use protocol::AppId; + +use super::jailer_inputs::{Inputs, INPUTS_FILENAME}; +use super::mount_namespace::Mount; +#[cfg(target_os = "linux")] +use super::snapshot::{exchange, EXCHANGE_BOOT_ID_FILENAME}; +use crate::install::jailer_identities::Identities; +use crate::json_store::read_json; +#[cfg(target_os = "linux")] +use crate::json_store::{make_directory, write_json}; +use crate::unix_socket::own; + +#[cfg(target_os = "linux")] +const PRIVATE_MODE: u32 = 0o700; +#[cfg(target_os = "linux")] +const ASSETS_MODE: u32 = 0o555; + +pub(crate) struct Jailer { + pub(crate) binary: PathBuf, + pub(crate) identities: Identities, + #[cfg(test)] + fake_preparation: bool, +} + +pub(crate) struct Jail { + pub(crate) uid: u32, + pub(crate) gid: u32, + pub(crate) root: PathBuf, + mounts: Vec, + #[cfg(test)] + fake_mounts: bool, +} + +fn io_error(error: impl std::fmt::Display) -> std::io::Error { + std::io::Error::other(error.to_string()) +} + +pub(crate) fn jail_id(app_id: &AppId) -> String { + use sha2::Digest; + hex::encode(&sha2::Sha256::digest(app_id.as_str().as_bytes())[..16]) +} + +pub(crate) fn root(base: &Path, app_id: &AppId) -> PathBuf { + base.join("firecracker").join(jail_id(app_id)).join("root") +} + +impl Jailer { + pub(crate) fn new(binary: PathBuf, identities: Identities) -> Self { + Self { + binary, + identities, + #[cfg(test)] + fake_preparation: false, + } + } + + #[cfg(test)] + pub(crate) fn for_testing(binary: PathBuf) -> Self { + let mut jailer = Self::new( + binary, + Identities { + uid_base: 100_000, + gid_base: 100_000, + }, + ); + jailer.fake_preparation = true; + jailer + } + + fn identities(&self, slot: u32) -> std::io::Result<(u32, u32)> { + let identity = |base: u32| { + base.checked_add(slot) + .filter(|id| *id > 0 && *id < u32::MAX) + .ok_or_else(|| io_error("the jail identity is outside its reserved range")) + }; + Ok(( + identity(self.identities.uid_base)?, + identity(self.identities.gid_base)?, + )) + } + + pub(crate) fn prepare( + &self, + directory: &Path, + base: &Path, + snapshot_dir: &Path, + app_id: &AppId, + host_boot_id: &str, + ) -> std::io::Result { + #[cfg(test)] + if self.fake_preparation { + let root = root(base, app_id); + crate::json_store::make_directory(&root, 0o700)?; + return Ok(Jail::for_testing(root)); + } + let inputs: Inputs = read_json(&directory.join(INPUTS_FILENAME)) + .map_err(io_error)? + .ok_or_else(|| io_error("the jail has no staged inputs"))?; + let (uid, gid) = self.identities(inputs.slot)?; + #[cfg(not(target_os = "linux"))] + { + let _ = ( + inputs, + uid, + gid, + directory, + base, + snapshot_dir, + app_id, + host_boot_id, + ); + Err(std::io::Error::new( + std::io::ErrorKind::Unsupported, + "the Firecracker jailer requires Linux", + )) + } + #[cfg(target_os = "linux")] + { + let root = root(base, app_id); + if root.exists() { + std::fs::remove_dir_all(&root)?; + } + make_directory(&root, PRIVATE_MODE)?; + let assets = root.join("assets"); + make_directory(&assets, PRIVATE_MODE)?; + let mut mounts = Vec::new(); + for input in &inputs.files { + let target = assets.join(&input.name); + let metadata = std::fs::metadata(&input.source)?; + use std::os::unix::fs::{FileTypeExt, MetadataExt}; + if metadata.file_type().is_block_device() && !input.read_only { + make_device(&target, metadata.rdev(), uid, gid)?; + continue; + } + if !metadata.is_file() { + return Err(io_error(format!("{} is not a VM image", input.source.display()))); + } + if input.read_only && metadata.mode() & 0o004 == 0 { + std::fs::copy(&input.source, &target)?; + set_mode(&target, 0o444)?; + continue; + } + if !input.read_only { + grant_volume_group(&input.source, gid)?; + } + std::fs::File::create(&target)?; + mounts.push(Mount::new(&input.source, &target, input.read_only)?); + } + set_mode(&assets, ASSETS_MODE)?; + let snapshots = exchange(snapshot_dir, app_id); + let exchange_base = snapshots + .parent() + .expect("an exchange belongs to its snapshot directory"); + make_directory(exchange_base, PRIVATE_MODE)?; + crate::json_store::write_text( + &exchange_base.join(EXCHANGE_BOOT_ID_FILENAME), + host_boot_id, + 0o600, + ) + .map_err(io_error)?; + make_directory(&snapshots, PRIVATE_MODE)?; + own(&snapshots, uid, gid)?; + make_directory(&root.join("snapshots"), PRIVATE_MODE)?; + mounts.push(Mount::new(&snapshots, &root.join("snapshots"), false)?); + write_json( + &root.join(super::manager::FIRECRACKER_CONFIG_FILENAME), + &inputs.config, + ) + .map_err(io_error)?; + set_mode(&root.join(super::manager::FIRECRACKER_CONFIG_FILENAME), 0o444)?; + Ok(Jail { + uid, + gid, + root, + mounts, + #[cfg(test)] + fake_mounts: false, + }) + } + } +} + +impl Jail { + #[cfg(test)] + pub(crate) fn for_testing(root: PathBuf) -> Self { + use std::os::unix::fs::MetadataExt; + let metadata = std::fs::metadata(&root).expect("the fake jail directory exists"); + Self { + uid: metadata.uid(), + gid: metadata.gid(), + root, + mounts: Vec::new(), + fake_mounts: true, + } + } + + pub(crate) fn grant_socket(&self, path: &Path) -> std::io::Result<()> { + own(path, self.uid, self.gid) + } + + pub(crate) fn configure_command(&self, command: &mut tokio::process::Command) -> std::io::Result<()> { + #[cfg(test)] + if self.fake_mounts { + return Ok(()); + } + #[cfg(target_os = "linux")] + { + super::mount_namespace::configure(command, self.mounts.clone()); + Ok(()) + } + #[cfg(not(target_os = "linux"))] + { + let _ = (command, &self.mounts); + Err(std::io::Error::new( + std::io::ErrorKind::Unsupported, + "the jailer requires Linux", + )) + } + } +} + +#[cfg(target_os = "linux")] +fn set_mode(path: &Path, mode: u32) -> std::io::Result<()> { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(mode)) +} + +#[cfg(target_os = "linux")] +fn grant_volume_group(path: &Path, gid: u32) -> std::io::Result<()> { + std::os::unix::fs::chown(path, None, Some(gid))?; + set_mode(path, 0o660) +} + +#[cfg(target_os = "linux")] +#[allow( + unsafe_code, + reason = "an NBD node inside the jail must refer to the host device without changing its ownership" +)] +fn make_device(path: &Path, device: u64, uid: u32, gid: u32) -> std::io::Result<()> { + use std::os::unix::ffi::OsStrExt; + let name = CString::new(path.as_os_str().as_bytes()).map_err(io_error)?; + if unsafe { libc::mknod(name.as_ptr(), libc::S_IFBLK | 0o600, device) } < 0 { + return Err(std::io::Error::last_os_error()); + } + own(path, uid, gid) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_slot_outside_the_reserved_identity_range_is_refused() { + let jailer = Jailer::new( + "/jailer".into(), + Identities { + uid_base: u32::MAX, + gid_base: 100_000, + }, + ); + assert!(jailer.identities(1).is_err()); + } + + #[test] + fn app_identifiers_with_underscores_have_distinct_valid_jailer_identifiers() { + let underscored = AppId::parse("app_one").unwrap(); + let hyphenated = AppId::parse("app-one").unwrap(); + let id = jail_id(&underscored); + assert_eq!(id.len(), 32); + assert!(id.chars().all(|character| character.is_ascii_alphanumeric())); + assert_ne!(id, jail_id(&hyphenated)); + assert!(root(Path::new("/jails"), &underscored).ends_with(format!("{id}/root"))); + } +} diff --git a/crates/nibrunnerd/src/adapters/vm/manager.rs b/crates/nibrunnerd/src/adapters/vm/manager.rs index 3ff0549f..d851a6a7 100644 --- a/crates/nibrunnerd/src/adapters/vm/manager.rs +++ b/crates/nibrunnerd/src/adapters/vm/manager.rs @@ -34,6 +34,7 @@ pub struct VmManager { pub snapshot_dir: PathBuf, pub guest_image_dir: PathBuf, pub firecracker: PathBuf, + pub(crate) jailer: super::jailer::Jailer, pub guest_image_version: String, pub processes: VmProcesses, pub network: Arc, @@ -46,12 +47,21 @@ pub struct VmManager { pub in_flight: SnapshotsInFlight, } +struct PreparedVm { + #[cfg(test)] + config_file: PathBuf, + jail: super::jailer::Jail, +} + impl VmManager { pub fn working_dir_for(&self, app_id: &AppId) -> PathBuf { - self.processes - .read_record(app_id) - .and_then(|record| record.jail_root) - .unwrap_or_else(|| self.vm_dir.join(app_id.as_str())) + if let Some(record) = self.processes.read_record(app_id) { + if let Some(root) = record.jail_root { + return root; + } + return self.vm_dir.join(app_id.as_str()); + } + super::jailer::root(&self.vm_dir.join("jailer"), app_id) } fn api(&self, app_id: &AppId) -> FirecrackerApi { @@ -97,7 +107,7 @@ impl VmManager { .await .map_err(host)?; - let working_dir = self.working_dir_for(&request.desired.app_id); + let working_dir = self.vm_dir.join(request.desired.app_id.as_str()); make_directory(&working_dir, VM_DIR_MODE).map_err(|error| VmError::Host(error.to_string()))?; let rendered = render_instance_env(&InstanceEnvContent { @@ -186,15 +196,57 @@ impl VmManager { Ok(()) } - async fn stage(&self, request: &BootRequest) -> Result { + async fn stage(&self, request: &BootRequest) -> Result { let config_file = self.stage_inputs(request).await?; + #[cfg(not(test))] + let _ = config_file; + let jail = self.prepare_jail(&request.desired.app_id).await?; self.attach_channels( &request.desired.app_id, &request.desired.deployment_id, - &self.working_dir_for(&request.desired.app_id), + &jail.root, ) .await?; - Ok(config_file) + let granted = async { + jail.grant_socket(&tenant_log_socket_path(&jail.root)) + .map_err(|error| VmError::Host(error.to_string()))?; + jail.grant_socket(&cron_registration_socket_path(&jail.root)) + .map_err(|error| VmError::Host(error.to_string()))?; + self.network + .set_tap_owner(&request.slot.tap_name, jail.uid) + .await + .map_err(|error| VmError::Host(error.message())) + } + .await; + if let Err(error) = granted { + self.cron_registration.detach(&request.desired.app_id).await; + self.logs.detach(&request.desired.app_id).await; + return Err(error); + } + Ok(PreparedVm { + #[cfg(test)] + config_file, + jail, + }) + } + + async fn prepare_jail(&self, app_id: &AppId) -> Result { + if self.processes.status(app_id).active { + return Err(VmError::Host( + "a running microVM cannot have its jail rebuilt".into(), + )); + } + self.logs.detach(app_id).await; + self.cron_registration.detach(app_id).await; + self.jailer + .prepare( + &self.vm_dir.join(app_id.as_str()), + &self.vm_dir.join("jailer"), + &self.snapshot_dir, + app_id, + self.processes.boot_id(), + ) + .map_err(|error| VmError::Host(error.to_string())) } async fn create_snapshot(&self, app_id: &AppId, state: &Path, memory: &Path) -> Result<(), VmError> { @@ -215,8 +267,22 @@ impl VmManager { let _ = std::fs::remove_file(exchange.join(name)); } let outcome = async { + let jailed = self + .processes + .read_record(app_id) + .is_some_and(|record| record.jail_root.is_some()); + let state_source = if jailed { + PathBuf::from("/snapshots/state") + } else { + exchange.join("state") + }; + let memory_source = if jailed { + PathBuf::from("/snapshots/memory") + } else { + exchange.join("memory") + }; self.api(app_id) - .create_snapshot(&exchange.join("state"), &exchange.join("memory")) + .create_snapshot(&state_source, &memory_source) .await?; for (name, destination) in [("state", state), ("memory", memory)] { publish_snapshot_file(&exchange.join(name), destination) @@ -269,14 +335,13 @@ impl Vmm for VmManager { let app_id = request.desired.app_id.clone(); self.discard_snapshot(&app_id); let staged = std::time::Instant::now(); - let config_file = self.stage(&request).await?; + let prepared = self.stage(&request).await?; let staged_ms = staged.elapsed().as_millis(); let starting = std::time::Instant::now(); - let working_dir = self.working_dir_for(&app_id); let started = self .processes - .spawn(&app_id, &self.firecracker, &working_dir, Some(&config_file)) + .spawn(&app_id, &self.firecracker, &self.jailer, &prepared.jail, true) .await; if let Err(error) = started { self.cron_registration.detach(&app_id).await; @@ -344,20 +409,52 @@ impl Vmm for VmManager { return Err(error); } - let _ = std::fs::remove_file(&paths.stamp_path); + if !self + .processes + .read_record(&request.app_id) + .is_some_and(|record| record.jail_root.is_some()) + { + self.discard_snapshot(&request.app_id); + return Err(VmError::SnapshotUnusable { + reason: "the snapshot was created by a legacy VMM without a jail".into(), + }); + } + let _ = std::fs::remove_file(&paths.stamp_path); let restoring = std::time::Instant::now(); - let working_dir = self.working_dir_for(&request.app_id); + let jail = self.prepare_jail(&request.app_id).await?; + self.readopt(&request.app_id).await?; + jail.grant_socket(&tenant_log_socket_path(&jail.root)) + .map_err(|error| VmError::Host(error.to_string()))?; + jail.grant_socket(&cron_registration_socket_path(&jail.root)) + .map_err(|error| VmError::Host(error.to_string()))?; + self.network + .set_tap_owner(&request.slot.tap_name, jail.uid) + .await + .map_err(|error| VmError::Host(error.message()))?; + let exchange = super::snapshot::exchange(&self.snapshot_dir, &request.app_id); + for (source, name) in [(&paths.state_path, "state"), (&paths.memory_path, "memory")] { + let target = exchange.join(name); + let _ = std::fs::remove_file(&target); + use std::os::unix::fs::PermissionsExt; + std::os::unix::fs::chown(source, Some(0), Some(jail.gid)) + .map_err(|error| VmError::Host(error.to_string()))?; + std::fs::set_permissions(source, std::fs::Permissions::from_mode(0o440)) + .map_err(|error| VmError::Host(error.to_string()))?; + std::fs::hard_link(source, &target).map_err(|error| VmError::Host(error.to_string()))?; + } + let state_path = PathBuf::from("/snapshots/state"); + let memory_path = PathBuf::from("/snapshots/memory"); let started = self .processes - .spawn(&request.app_id, &self.firecracker, &working_dir, None) + .spawn(&request.app_id, &self.firecracker, &self.jailer, &jail, false) .await .map_err(|error| VmError::Host(error.to_string())); let outcome = match started { Err(error) => Err(error), Ok(_) => { let api = self.api(&request.app_id); - match api.load_snapshot(&paths.state_path, &paths.memory_path).await { + match api.load_snapshot(&state_path, &memory_path).await { Err(error) => Err(error), Ok(()) => match api.resume().await { Err(error) => Err(error), @@ -378,6 +475,9 @@ impl Vmm for VmManager { self.processes.stop(&request.app_id).await; } self.discard_snapshot(&request.app_id); + for name in ["state", "memory"] { + let _ = std::fs::remove_file(exchange.join(name)); + } outcome?; self.metrics.sleep_wake.restored(restoring.elapsed()); tracing::info!( @@ -400,7 +500,12 @@ impl Vmm for VmManager { self.processes.forget(app_id); self.logs.detach(app_id).await; self.cron_registration.detach(app_id).await; - let _ = std::fs::remove_dir_all(self.working_dir_for(app_id)); + let _ = std::fs::remove_dir_all(self.vm_dir.join(app_id.as_str())); + let _ = std::fs::remove_dir_all( + super::jailer::root(&self.vm_dir.join("jailer"), app_id) + .parent() + .expect("a jail root has an app directory"), + ); let _ = std::fs::remove_dir_all(super::snapshot::exchange(&self.snapshot_dir, app_id)); Ok(()) } @@ -555,6 +660,7 @@ mod tests { use crate::test_support::mocks; use crate::test_support::*; use protocol::ObjectKey; + use std::os::unix::fs::MetadataExt; struct Fixture { _directory: tempfile::TempDir, @@ -564,7 +670,7 @@ mod tests { } fn fixture() -> Fixture { - let directory = tempfile::tempdir().unwrap(); + let directory = tempfile::tempdir_in("/tmp").unwrap(); let root = directory.path(); let (network, network_spy) = mocks::network(); let state = HostState::shared(); @@ -574,6 +680,7 @@ mod tests { snapshot_dir: root.join("snapshots"), guest_image_dir: root.join("guest"), firecracker: root.join("bin/firecracker"), + jailer: super::super::jailer::Jailer::for_testing(root.join("bin/jailer")), guest_image_version: "6.1.180-test".into(), processes: VmProcesses::new(root.join("run")), network, @@ -643,7 +750,7 @@ mod tests { let request = boot_request(desired_instance(|instance| { instance.hostnames = vec![app_hostname()] })); - let config_file = fixture.manager.stage(&request).await.unwrap(); + let config_file = fixture.manager.stage(&request).await.unwrap().config_file; let config: serde_json::Value = serde_json::from_str(&std::fs::read_to_string(&config_file).unwrap()).unwrap(); @@ -672,6 +779,15 @@ mod tests { assert_eq!(fixture.network.taps()[0].tap_name, "nbr0"); assert_eq!(fixture.network.neighbours()[0].guest_mac, "02:00:0a:c9:00:02"); + assert_eq!( + fixture.network.owners(), + vec![( + "nbr0".into(), + std::fs::metadata(fixture.manager.working_dir_for(&app_id())) + .unwrap() + .uid() + )] + ); assert_eq!(fixture.manager.logs.attached().await, vec![app_id()]); } @@ -684,7 +800,7 @@ mod tests { .layer_image_paths .push(PathBuf::from("/cache/def/layer.img")); fixture.manager.stage(&request).await.unwrap(); - let written = config_drive(&fixture.manager.working_dir_for(&app_id())); + let written = config_drive(&fixture.manager.vm_dir.join(app_id().as_str())); assert!(written.contains("NIBRUN_HTTP_PORT=3000")); assert!(written.contains("NIBRUN_LAYERS=2\n"), "{written}"); } @@ -817,6 +933,31 @@ mod tests { assert!(fixture.manager.cron_registration.attached().await.is_empty()); } + #[tokio::test] + async fn a_tap_that_cannot_be_granted_to_the_jail_leaves_no_partial_channels() { + let mut fixture = fixture(); + let mut network = crate::adapters::net::tap::MockHostNetwork::new(); + network.expect_ensure_tap().returning(|_| Ok(())); + network.expect_refresh_neighbour().returning(|_| Ok(())); + network.expect_set_tap_owner().returning(|name, _| { + Err(crate::adapters::net::tap::NetworkError { + what: "tap ownership", + device: name.into(), + reason: "operation not permitted".into(), + }) + }); + fixture.manager.network = Arc::new(network); + let error = fixture + .manager + .boot(boot_request(desired_instance(|_| {}))) + .await + .unwrap_err(); + assert!(error.message().contains("tap ownership")); + assert!(fixture.manager.logs.attached().await.is_empty()); + assert!(fixture.manager.cron_registration.attached().await.is_empty()); + assert!(fixture.manager.processes.read_record(&app_id()).is_none()); + } + #[tokio::test] async fn a_hypervisor_that_would_not_start_takes_the_log_attachment_down_behind_it() { let fixture = fixture(); @@ -896,7 +1037,7 @@ mod tests { use std::os::unix::fs::MetadataExt; let fixture = fixture(); let short_directory = tempfile::tempdir_in("/tmp").unwrap(); - let root = short_directory.path().join("root"); + let root = super::super::jailer::root(short_directory.path(), &app_id()); make_directory(&root, VM_DIR_MODE).unwrap(); let identity = std::fs::metadata(&root).unwrap(); fixture.state.put_record(instance_record(|_| {})).await; @@ -1121,6 +1262,38 @@ mod tests { assert_eq!(fixture.manager.in_flight.bytes(), 0); } + #[tokio::test] + async fn a_legacy_snapshot_is_refused_before_spawning_so_the_waker_can_cold_boot() { + let fixture = fixture(); + let request = SuspendRequest { + app_id: app_id(), + deployment_id: deployment_id(), + slot: nft_render::describe_slot(0, app_id()), + }; + let paths = snapshot_paths(&fixture.manager.snapshot_dir, &app_id()); + write_json(&paths.stamp_path, &fixture.manager.current_stamp(&request)).unwrap(); + fixture + .manager + .processes + .write_record(&crate::adapters::vm::process::VmRecord { + app_id: app_id(), + pid: 1, + host_boot_id: fixture.manager.processes.boot_id().into(), + started_at_ms: 0, + exit_code: Some(0), + signal: None, + stop_requested: true, + jail_root: None, + jail_uid: None, + }) + .unwrap(); + let refused = fixture.manager.wake(request).await.unwrap_err(); + assert!(matches!(refused, VmError::SnapshotUnusable { .. })); + assert!(refused.message().contains("legacy VMM without a jail")); + assert_eq!(fixture.manager.processes.read_record(&app_id()).unwrap().pid, 1); + assert!(!paths.directory.exists()); + } + #[tokio::test] async fn a_wake_with_no_snapshot_kept_at_all_says_so_rather_than_starting_a_cold_guest() { let fixture = fixture(); @@ -1149,7 +1322,7 @@ mod tests { instance.config.command.environment = tenant_environment(&[("MODE", "production")]) }); fixture.manager.stage(&boot_request(changed)).await.unwrap(); - let written = config_drive(&fixture.manager.working_dir_for(&app_id())); + let written = config_drive(&fixture.manager.vm_dir.join(app_id().as_str())); assert!(written.contains("MODE"), "{written}"); assert_eq!(fixture.manager.logs.attached().await, vec![app_id()]); } diff --git a/crates/nibrunnerd/src/adapters/vm/mod.rs b/crates/nibrunnerd/src/adapters/vm/mod.rs index a99b340d..07f1d0a5 100644 --- a/crates/nibrunnerd/src/adapters/vm/mod.rs +++ b/crates/nibrunnerd/src/adapters/vm/mod.rs @@ -8,5 +8,6 @@ pub mod status; pub use status::{VmExit, VmStatus, UNKNOWN_VM}; -#[cfg(not(test))] pub(crate) mod mount_namespace; + +pub(crate) mod jailer; diff --git a/crates/nibrunnerd/src/adapters/vm/mount_namespace.rs b/crates/nibrunnerd/src/adapters/vm/mount_namespace.rs index a797bc43..4b571df0 100644 --- a/crates/nibrunnerd/src/adapters/vm/mount_namespace.rs +++ b/crates/nibrunnerd/src/adapters/vm/mount_namespace.rs @@ -1,18 +1,45 @@ -pub(crate) fn configure(command: &mut tokio::process::Command) { +#[cfg(target_os = "linux")] +use std::path::Path; + +#[derive(Clone)] +pub(crate) struct Mount { + #[cfg(target_os = "linux")] + source: std::ffi::CString, + #[cfg(target_os = "linux")] + target: std::ffi::CString, + #[cfg(target_os = "linux")] + read_only: bool, +} + +#[cfg(target_os = "linux")] +pub(crate) fn configure(command: &mut tokio::process::Command, mounts: Vec) { #[cfg(target_os = "linux")] { - #[allow(unsafe_code, reason = "a VMM needs private mounts before exec")] + #[allow(unsafe_code, reason = "a VMM needs private bind mounts before exec")] unsafe { - command.pre_exec(enter); + command.pre_exec(move || enter(&mounts)); } } - #[cfg(not(target_os = "linux"))] - let _ = command; } #[cfg(target_os = "linux")] -#[allow(unsafe_code, reason = "mount namespaces have no safe libc wrapper here")] -fn enter() -> std::io::Result<()> { +impl Mount { + pub(crate) fn new(source: &Path, target: &Path, read_only: bool) -> std::io::Result { + use std::os::unix::ffi::OsStrExt; + Ok(Self { + source: std::ffi::CString::new(source.as_os_str().as_bytes()).map_err(std::io::Error::other)?, + target: std::ffi::CString::new(target.as_os_str().as_bytes()).map_err(std::io::Error::other)?, + read_only, + }) + } +} + +#[cfg(target_os = "linux")] +#[allow( + unsafe_code, + reason = "mount namespaces and bind mounts have no safe libc wrapper here" +)] +fn enter(mounts: &[Mount]) -> std::io::Result<()> { if unsafe { libc::unshare(libc::CLONE_NEWNS) } < 0 { return Err(std::io::Error::last_os_error()); } @@ -28,5 +55,32 @@ fn enter() -> std::io::Result<()> { { return Err(std::io::Error::last_os_error()); } + for mount in mounts { + if unsafe { + libc::mount( + mount.source.as_ptr(), + mount.target.as_ptr(), + std::ptr::null(), + libc::MS_BIND, + std::ptr::null(), + ) + } < 0 + { + return Err(std::io::Error::last_os_error()); + } + if mount.read_only + && unsafe { + libc::mount( + std::ptr::null(), + mount.target.as_ptr(), + std::ptr::null(), + libc::MS_BIND | libc::MS_REMOUNT | libc::MS_RDONLY, + std::ptr::null(), + ) + } < 0 + { + return Err(std::io::Error::last_os_error()); + } + } Ok(()) } diff --git a/crates/nibrunnerd/src/adapters/vm/process.rs b/crates/nibrunnerd/src/adapters/vm/process.rs index 9dee6c59..953a207a 100644 --- a/crates/nibrunnerd/src/adapters/vm/process.rs +++ b/crates/nibrunnerd/src/adapters/vm/process.rs @@ -183,34 +183,71 @@ impl VmProcesses { } } - pub async fn spawn( + pub(crate) async fn spawn( &self, app_id: &AppId, binary: &Path, - working_dir: &Path, - config_file: Option<&Path>, + jailer: &super::jailer::Jailer, + jail: &super::jailer::Jail, + boot: bool, ) -> std::io::Result { make_directory(&self.runtime_dir, RUNTIME_DIR_MODE)?; let api_socket = self.api_socket(app_id); let _ = std::fs::remove_file(&api_socket); - let _ = std::fs::remove_file(working_dir.join(guest_contract::vsock::GUEST_VSOCK_FILENAME)); - - let mut command = tokio::process::Command::new(binary); - command.arg("--api-sock").arg(&api_socket); - if let Some(config_file) = config_file { - command.arg("--config-file").arg(config_file); + let _ = std::fs::remove_file(jail.root.join(guest_contract::vsock::GUEST_VSOCK_FILENAME)); + let base = jail + .root + .ancestors() + .nth(3) + .ok_or_else(|| std::io::Error::other("the jail has no base directory"))?; + std::os::unix::fs::symlink(jail.root.join("api.sock"), &api_socket)?; + let mut command = tokio::process::Command::new(&jailer.binary); + command + .arg("--id") + .arg(super::jailer::jail_id(app_id)) + .arg("--exec-file") + .arg(binary) + .arg("--uid") + .arg(jail.uid.to_string()) + .arg("--gid") + .arg(jail.gid.to_string()) + .arg("--chroot-base-dir") + .arg(base) + .arg("--cgroup-version") + .arg("2") + .arg("--parent-cgroup") + .arg("nibrunner-jailer") + .arg("--") + .arg("--api-sock") + .arg("/api.sock"); + if boot { + command.arg("--config-file").arg("/firecracker.json"); } - #[cfg(not(test))] - super::mount_namespace::configure(&mut command); - self.launch(app_id, command, working_dir).await + jail.configure_command(&mut command)?; + self.launch(app_id, command, jail).await + } + + #[cfg(test)] + async fn spawn_test_process( + &self, + app_id: &AppId, + binary: &Path, + working_dir: &Path, + config_file: Option<&Path>, + ) -> std::io::Result { + let jail = super::jailer::Jail::for_testing(working_dir.into()); + let jailer = super::jailer::Jailer::for_testing(binary.into()); + self.spawn(app_id, binary, &jailer, &jail, config_file.is_some()) + .await } async fn launch( &self, app_id: &AppId, mut command: tokio::process::Command, - working_dir: &Path, + jail: &super::jailer::Jail, ) -> std::io::Result { + let working_dir = &jail.root; let console = std::fs::File::create(self.console_path(app_id))?; command .current_dir(working_dir) @@ -243,8 +280,8 @@ impl VmProcesses { exit_code: None, signal: None, stop_requested: false, - jail_root: None, - jail_uid: None, + jail_root: Some(jail.root.clone()), + jail_uid: Some(jail.uid), }; self.write_record(&record)?; @@ -558,13 +595,15 @@ mod tests { make_directory(&working_dir, 0o700).unwrap(); let record = processes - .spawn(&app_id(), Path::new("/bin/echo"), &working_dir, None) + .spawn_test_process(&app_id(), Path::new("/bin/echo"), &working_dir, None) .await .unwrap(); assert_eq!(record.app_id, app_id()); assert_eq!(record.host_boot_id, "boot-1"); assert_eq!(record.exit_code, None); assert!(!record.stop_requested); + assert_eq!(record.jail_root, Some(working_dir.clone())); + assert!(record.jail_uid.is_some()); assert_eq!(processes.adopted_app_ids(), vec![app_id()]); for _ in 0..200 { @@ -578,9 +617,17 @@ mod tests { assert!(settled.loaded); assert!(!settled.active); assert!(!settled.failed, "an exit of 0 is not a failure"); - assert!(std::fs::read_to_string(processes.console_path(&app_id())) - .unwrap() - .contains("--api-sock")); + let console = std::fs::read_to_string(processes.console_path(&app_id())).unwrap(); + for argument in [ + "--id", + "--exec-file", + "--uid", + "--gid", + "--chroot-base-dir", + "--api-sock /api.sock", + ] { + assert!(console.contains(argument), "{console}"); + } } #[tokio::test] @@ -599,7 +646,7 @@ mod tests { .unwrap(); let record = processes - .spawn(&app_id(), &lingering, &working_dir, None) + .spawn_test_process(&app_id(), &lingering, &working_dir, None) .await .unwrap(); signal(record.pid, libc::SIGKILL); @@ -626,7 +673,7 @@ mod tests { let working_dir = directory.path().join("vm"); make_directory(&working_dir, 0o700).unwrap(); assert!(processes - .spawn( + .spawn_test_process( &app_id(), Path::new("/nowhere/nibrunner-no-such-hypervisor"), &working_dir, diff --git a/crates/nibrunnerd/src/run.rs b/crates/nibrunnerd/src/run.rs index 183f4b00..ffaea670 100644 --- a/crates/nibrunnerd/src/run.rs +++ b/crates/nibrunnerd/src/run.rs @@ -15,7 +15,7 @@ use crate::adapters::proxy::activator::AppActivator; use crate::adapters::proxy::{router, Router}; use crate::adapters::vm::layers::LayerImages; use crate::adapters::vm::manager::{verify_guest_image, VmManager}; -use crate::adapters::vm::process::{extract_firecracker, VmProcesses, FIRECRACKER_VERSION}; +use crate::adapters::vm::process::{extract_firecracker, extract_jailer, VmProcesses, FIRECRACKER_VERSION}; use crate::adapters::vm::snapshot::reap_stale_snapshots; use crate::adapters::volumes::initial_contents::ContentsStaging; use crate::adapters::volumes::local_file::LocalFileVolumes; @@ -48,6 +48,12 @@ pub async fn build_host(config: HostConfig) -> Result, StartupError> { .map_err(|error| StartupError::Unusable(error.message()))?; let firecracker = extract_firecracker(&config.firecracker_dir) .map_err(|error| StartupError::Unusable(error.to_string()))?; + let identities = crate::install::jailer_identities::read(config.max_apps) + .map_err(|error| StartupError::Unusable(error.message()))?; + let jailer = crate::adapters::vm::jailer::Jailer::new( + extract_jailer(&config.firecracker_dir).map_err(|error| StartupError::Unusable(error.to_string()))?, + identities, + ); let commands: Arc = Arc::new(HostCommands); let state = HostState::shared(); let repositories = crate::repositories::Repositories::sqlite( @@ -123,6 +129,7 @@ pub async fn build_host(config: HostConfig) -> Result, StartupError> { guest_image_dir: config.guest_image_dir.clone(), guest_image_version: guest_image_version.clone(), firecracker, + jailer, processes, network, volumes: volumes.clone(), diff --git a/crates/nibrunnerd/src/test_support/machine.rs b/crates/nibrunnerd/src/test_support/machine.rs index 6b1e9889..57d0027d 100644 --- a/crates/nibrunnerd/src/test_support/machine.rs +++ b/crates/nibrunnerd/src/test_support/machine.rs @@ -123,6 +123,9 @@ pub async fn started_with(edit: impl FnOnce(&mut crate::config::HostConfig)) -> crate::install::kernel::apply(&config, &mut crate::install::Laid::default()) .expect("this machine takes the settings a host needs"); + crate::install::jailer_identities::ensure(config.max_apps) + .expect("the installer reserves this machine's jailer identities"); + let host = crate::run::build_host(config) .await .expect("a host on this machine"); From 06ce605a8bf2569750a5f48c7460f48799184a43 Mon Sep 17 00:00:00 2001 From: ilbertt Date: Sat, 3 Oct 2026 20:17:43 +0200 Subject: [PATCH 2/2] test: prove jailed VMM isolation across sleep and readoption --- crates/nibrunnerd/tests/guest/isolation.rs | 96 ++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/crates/nibrunnerd/tests/guest/isolation.rs b/crates/nibrunnerd/tests/guest/isolation.rs index 92c88b9b..0af9d69e 100644 --- a/crates/nibrunnerd/tests/guest/isolation.rs +++ b/crates/nibrunnerd/tests/guest/isolation.rs @@ -4,9 +4,11 @@ use std::net::{Ipv4Addr, SocketAddr, TcpListener}; +use nibrunnerd::adapters::vm::process::VmProcesses; use nibrunnerd::test_support::egress::EgressEndpoint; use nibrunnerd::test_support::machine::{RunningHost, Tenant}; use protocol::InstanceState; +use std::os::unix::fs::MetadataExt; /// The tenant gives up after two seconds, so anything under this came back because a rule said /// no rather than because nothing was there. @@ -149,3 +151,97 @@ async fn an_address_the_configuration_denies_cannot_be_reached() { closed.stop().await; } + +const JAILER_IDLE_TIMEOUT_MS: u64 = 60_000; + +#[tokio::test(flavor = "multi_thread")] +async fn jailed_vmm_processes_have_distinct_non_root_identities_and_resume_their_memory_and_volume() { + let Some(host) = crate::host().await else { + return; + }; + let one = host.tenant(1).on_request(JAILER_IDLE_TIMEOUT_MS); + let another = host.tenant(2); + host.deploy(&[one.clone(), another.clone()]).await; + host.until_state(&one.app_id, InstanceState::Running).await; + host.until_state(&another.app_id, InstanceState::Running).await; + let processes = VmProcesses::new(host.host.config.runtime_dir.clone()); + let first = processes.read_record(&one.app_id).expect("the first VMM record"); + let second = processes + .read_record(&another.app_id) + .expect("the second VMM record"); + for record in [&first, &second] { + let root = std::fs::metadata(record.jail_root.as_ref().expect("the recorded jail root")) + .expect("the jail root's owner"); + assert_eq!(record.jail_uid, Some(root.uid())); + let status = std::fs::read_to_string(format!("/proc/{}/status", record.pid)) + .expect("the VMM's process status"); + for (label, identity) in [("Uid:", root.uid()), ("Gid:", root.gid())] { + assert_ne!(identity, 0); + let line = status + .lines() + .find(|line| line.starts_with(label)) + .expect("the process identity line"); + assert!( + line.split_whitespace() + .skip(1) + .all(|value| value.parse::() == Ok(identity)), + "{line}" + ); + } + } + let root = first.jail_root.as_ref().expect("the recorded jail root"); + let process_root = std::path::PathBuf::from(format!("/proc/{}/root", first.pid)); + let actual = std::fs::metadata(&process_root).expect("the VMM root"); + let recorded = std::fs::metadata(root).expect("the recorded root inode"); + assert_eq!((actual.dev(), actual.ino()), (recorded.dev(), recorded.ino())); + assert!( + std::fs::metadata(process_root.join("assets/kernel")) + .unwrap() + .len() + > 0 + ); + assert_ne!(first.jail_root, second.jail_root); + let other_root = second.jail_root.as_ref().expect("the second jail"); + assert_ne!( + std::fs::metadata(root).unwrap().uid(), + std::fs::metadata(other_root).unwrap().uid() + ); + assert_ne!( + std::fs::metadata(root).unwrap().gid(), + std::fs::metadata(other_root).unwrap().gid() + ); + assert!(!process_root.join("etc/nibrunner/config.toml").exists()); + assert_eq!( + host.get(&one, "/remember") + .await + .expect("memory before sleep") + .body, + "1" + ); + assert_eq!( + host.get(&one, "/write?path=kept&body=jailed") + .await + .expect("a volume write") + .status, + 200 + ); + host.let_sleep(&one).await; + assert_eq!( + host.get(&one, "/remember").await.expect("memory after wake").body, + "2" + ); + assert_eq!( + host.get(&one, "/read?path=kept") + .await + .expect("the volume after wake") + .body, + "jailed" + ); + host.host + .vms + .readopt(&one.app_id) + .await + .expect("reattach jailed host channels"); + assert_eq!(host.get(&one, "/").await.expect("the readopted app").status, 200); + host.stop().await; +}