From 42c271cdd03ad45feeb2ba1e240bd46a8600bf45 Mon Sep 17 00:00:00 2001 From: ilbertt Date: Sat, 3 Oct 2026 19:32:17 +0200 Subject: [PATCH 1/4] fix: reach guest services through their VMM directory --- crates/nibrunnerd/src/domain/filesystem/reader.rs | 13 ++++++++----- crates/nibrunnerd/src/domain/reconcile/exports.rs | 6 +----- crates/nibrunnerd/src/test_support/mocks.rs | 6 +++++- crates/nibrunnerd/src/test_support/mod.rs | 2 +- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/crates/nibrunnerd/src/domain/filesystem/reader.rs b/crates/nibrunnerd/src/domain/filesystem/reader.rs index e90e8597..ad9443d9 100644 --- a/crates/nibrunnerd/src/domain/filesystem/reader.rs +++ b/crates/nibrunnerd/src/domain/filesystem/reader.rs @@ -5,9 +5,8 @@ use crate::host::Host; use crate::ports::GuestReading; pub fn guest_vsock_path(host: &Host, app_id: &AppId) -> std::path::PathBuf { - host.config - .vm_dir() - .join(app_id.as_str()) + host.vms + .working_dir(app_id) .join(guest_contract::vsock::GUEST_VSOCK_FILENAME) } @@ -36,12 +35,16 @@ mod tests { use crate::test_support::*; #[tokio::test] - async fn each_guest_is_reached_on_a_socket_inside_its_own_microvm_directory() { + async fn each_guest_is_reached_in_the_directory_selected_by_its_vmm() { let host = test_host().await; let neighbour = AppId::parse("app-2").unwrap(); let path = guest_vsock_path(host.arc(), &app_id()); - assert!(path.starts_with(host.config.vm_dir())); + assert_eq!(path.parent().unwrap(), host.host.vms.working_dir(&app_id())); + assert_ne!( + path.parent().unwrap(), + host.config.vm_dir().join(app_id().as_str()) + ); assert!(path.ends_with(guest_contract::vsock::GUEST_VSOCK_FILENAME)); assert_eq!(path.parent().unwrap().file_name().unwrap(), app_id().as_str()); assert_ne!(path, guest_vsock_path(host.arc(), &neighbour)); diff --git a/crates/nibrunnerd/src/domain/reconcile/exports.rs b/crates/nibrunnerd/src/domain/reconcile/exports.rs index 7477d2f6..2919c5d8 100644 --- a/crates/nibrunnerd/src/domain/reconcile/exports.rs +++ b/crates/nibrunnerd/src/domain/reconcile/exports.rs @@ -108,11 +108,7 @@ async fn write_inner( staging_dir: &std::path::Path, ) -> Result { let crontab = registered_crontab(host, &desired.app_id).await?; - let vsock_path = host - .config - .vm_dir() - .join(desired.app_id.as_str()) - .join(guest_contract::vsock::GUEST_VSOCK_FILENAME); + let vsock_path = crate::domain::filesystem::reader::guest_vsock_path(host, &desired.app_id); let lease = frozen(&desired.app_id, &vsock_path) .await .map_err(|error| error.message())?; diff --git a/crates/nibrunnerd/src/test_support/mocks.rs b/crates/nibrunnerd/src/test_support/mocks.rs index 8adcb720..5d06f70d 100644 --- a/crates/nibrunnerd/src/test_support/mocks.rs +++ b/crates/nibrunnerd/src/test_support/mocks.rs @@ -157,6 +157,10 @@ impl VmmSpy { pub const NOWHERE_VM_DIR: &str = "/nowhere/vm"; pub fn vmm() -> (Arc, VmmSpy) { + vmm_under(PathBuf::from(NOWHERE_VM_DIR)) +} + +pub(crate) fn vmm_under(directory: PathBuf) -> (Arc, VmmSpy) { let spy = VmmSpy::default(); let mut vms = MockVmm::new(); @@ -211,7 +215,7 @@ pub fn vmm() -> (Arc, VmmSpy) { let verdict = spy.verdict.clone(); vms.expect_guest_verdict().returning(move |_| held(&verdict)); vms.expect_working_dir() - .returning(|app_id: &AppId| PathBuf::from(NOWHERE_VM_DIR).join(app_id.as_str())); + .returning(move |app_id: &AppId| directory.join(app_id.as_str())); (Arc::new(vms), spy) } diff --git a/crates/nibrunnerd/src/test_support/mod.rs b/crates/nibrunnerd/src/test_support/mod.rs index 5694c1b0..2b83adca 100644 --- a/crates/nibrunnerd/src/test_support/mod.rs +++ b/crates/nibrunnerd/src/test_support/mod.rs @@ -473,7 +473,7 @@ async fn test_host_over( let state = HostState::shared(); let metrics = Arc::new(crate::domain::metrics::HostMetrics::new()); let (commands, command_log) = mocks::commands_formatting(); - let (vms, vm_spy) = mocks::vmm(); + let (vms, vm_spy) = mocks::vmm_under(config.vm_dir().join("jailer")); let (exports, export_spy) = mocks::exports_accepting(); let artifacts: Arc = artifacts; let cron = Arc::new(crate::domain::cron::registry::CronRegistry::new( From 2e1985716091d9422ab9c9b2016d370542d7288d Mon Sep 17 00:00:00 2001 From: ilbertt Date: Sat, 3 Oct 2026 20:05:05 +0200 Subject: [PATCH 2/4] feat: assign persistent TAPs to non-root VMMs --- crates/nibrunnerd/src/adapters/net/tap.rs | 49 +++++++++++++++++++++ crates/nibrunnerd/src/test_support/mocks.rs | 10 +++++ crates/nibrunnerd/tests/integration.rs | 3 ++ 3 files changed, 62 insertions(+) diff --git a/crates/nibrunnerd/src/adapters/net/tap.rs b/crates/nibrunnerd/src/adapters/net/tap.rs index 74d64a79..99bc511e 100644 --- a/crates/nibrunnerd/src/adapters/net/tap.rs +++ b/crates/nibrunnerd/src/adapters/net/tap.rs @@ -36,6 +36,14 @@ pub trait HostNetwork: Send + Sync { async fn refresh_neighbour(&self, neighbour: &Neighbour) -> Result<(), NetworkError>; + async fn set_tap_owner(&self, tap_name: &str, _uid: u32) -> Result<(), NetworkError> { + Err(NetworkError { + what: "tap ownership", + device: tap_name.into(), + reason: "this network adapter cannot assign an owner".into(), + }) + } + // A tap outlives the process that made it, which is what lets a daemon restart leave every // tenant serving. The same persistence means nothing reclaims one, so an app that leaves has // to say so. @@ -140,6 +148,43 @@ mod linux { Ok(()) } + #[allow( + unsafe_code, + reason = "assigning a persistent TAP to its jailed VMM requires TUNSETOWNER" + )] + fn set_tap_owner(tap_name: &str, uid: u32) -> Result<(), NetworkError> { + use std::os::fd::AsRawFd; + const TUNSETIFF: libc::Ioctl = 0x400454ca; + const TUNSETOWNER: libc::Ioctl = 0x400454cc; + const IFF_TAP: libc::c_short = 0x0002; + const IFF_NO_PI: libc::c_short = 0x1000; + #[repr(C)] + struct InterfaceRequest { + name: [libc::c_char; libc::IFNAMSIZ], + flags: libc::c_short, + padding: [u8; 22], + } + let device = std::fs::OpenOptions::new() + .read(true) + .write(true) + .open("/dev/net/tun") + .map_err(|error| failed("tap ownership", tap_name, error))?; + let mut request = InterfaceRequest { + name: [0; libc::IFNAMSIZ], + flags: IFF_TAP | IFF_NO_PI, + padding: [0; 22], + }; + for (index, byte) in tap_name.as_bytes().iter().take(libc::IFNAMSIZ - 1).enumerate() { + request.name[index] = *byte as libc::c_char; + } + if unsafe { libc::ioctl(device.as_raw_fd(), TUNSETIFF, &mut request) } < 0 + || unsafe { libc::ioctl(device.as_raw_fd(), TUNSETOWNER, uid) } < 0 + { + return Err(failed("tap ownership", tap_name, std::io::Error::last_os_error())); + } + Ok(()) + } + fn allow_route_localnet(tap_name: &str) -> Result<(), NetworkError> { let path = format!("/proc/sys/net/ipv4/conf/{tap_name}/route_localnet"); std::fs::write(&path, b"1").map_err(|error| failed("route_localnet", tap_name, error)) @@ -171,6 +216,10 @@ mod linux { allow_route_localnet(&tap.tap_name) } + async fn set_tap_owner(&self, tap_name: &str, uid: u32) -> Result<(), NetworkError> { + set_tap_owner(tap_name, uid) + } + async fn refresh_neighbour(&self, neighbour: &Neighbour) -> Result<(), NetworkError> { let index = self .index_of(&neighbour.tap_name) diff --git a/crates/nibrunnerd/src/test_support/mocks.rs b/crates/nibrunnerd/src/test_support/mocks.rs index 5d06f70d..bd80f80b 100644 --- a/crates/nibrunnerd/src/test_support/mocks.rs +++ b/crates/nibrunnerd/src/test_support/mocks.rs @@ -505,12 +505,17 @@ pub fn exports_answering( #[derive(Clone, Default)] pub struct NetworkSpy { + owners: Arc>>, taps: Arc>>, neighbours: Arc>>, removed: Arc>>, } impl NetworkSpy { + pub fn owners(&self) -> Vec<(String, u32)> { + held(&self.owners) + } + pub fn taps(&self) -> Vec { held(&self.taps) } @@ -533,6 +538,11 @@ pub fn network() -> (Arc, NetworkSpy) { push(&taps, tap.clone()); Ok(()) }); + let owners = spy.owners.clone(); + network.expect_set_tap_owner().returning(move |name, uid| { + push(&owners, (name.to_string(), uid)); + Ok(()) + }); let neighbours = spy.neighbours.clone(); network .expect_refresh_neighbour() diff --git a/crates/nibrunnerd/tests/integration.rs b/crates/nibrunnerd/tests/integration.rs index 38daab99..138aafa7 100644 --- a/crates/nibrunnerd/tests/integration.rs +++ b/crates/nibrunnerd/tests/integration.rs @@ -270,6 +270,9 @@ async fn a_tap_is_created_addressed_and_given_the_guest_it_will_hold() { .await .expect("a second pass changes nothing"); assert!(network.tap_names().await.contains(&slot.tap_name)); + network.set_tap_owner(&slot.tap_name, 100_000).await.unwrap(); + let owner = std::fs::read_to_string(format!("/sys/class/net/{}/owner", slot.tap_name)).unwrap(); + assert_eq!(owner.trim(), "100000"); network .refresh_neighbour(&Neighbour { From 5d3e0d38c1a73a1157d0cecb57cb42f18eedcf45 Mon Sep 17 00:00:00 2001 From: ilbertt Date: Sat, 3 Oct 2026 20:05:44 +0200 Subject: [PATCH 3/4] refactor: separate VMM command construction from supervision --- crates/nibrunnerd/src/adapters/vm/process.rs | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/crates/nibrunnerd/src/adapters/vm/process.rs b/crates/nibrunnerd/src/adapters/vm/process.rs index 0b2109ee..5a11cc15 100644 --- a/crates/nibrunnerd/src/adapters/vm/process.rs +++ b/crates/nibrunnerd/src/adapters/vm/process.rs @@ -191,19 +191,27 @@ impl VmProcesses { let _ = std::fs::remove_file(&api_socket); let _ = std::fs::remove_file(working_dir.join(guest_contract::vsock::GUEST_VSOCK_FILENAME)); - let console = std::fs::File::create(self.console_path(app_id))?; let mut command = tokio::process::Command::new(binary); + command.arg("--api-sock").arg(&api_socket); + if let Some(config_file) = config_file { + command.arg("--config-file").arg(config_file); + } + self.launch(app_id, command, working_dir).await + } + + async fn launch( + &self, + app_id: &AppId, + mut command: tokio::process::Command, + working_dir: &Path, + ) -> std::io::Result { + let console = std::fs::File::create(self.console_path(app_id))?; command - .arg("--api-sock") - .arg(&api_socket) .current_dir(working_dir) .stdin(Stdio::null()) .stdout(Stdio::from(console.try_clone()?)) .stderr(Stdio::from(console)) .kill_on_drop(false); - if let Some(config_file) = config_file { - command.arg("--config-file").arg(config_file); - } #[cfg(unix)] { #[allow( From f02916cf1446e27916acc4d492717789ae755b5e Mon Sep 17 00:00:00 2001 From: ilbertt Date: Sat, 3 Oct 2026 20:06:37 +0200 Subject: [PATCH 4/4] feat: readopt VMMs from their recorded root and identity --- crates/nibrunnerd/src/adapters/vm/manager.rs | 90 +++++++++++++++++++- crates/nibrunnerd/src/adapters/vm/process.rs | 18 ++++ crates/nibrunnerd/src/unix_socket.rs | 13 +++ 3 files changed, 120 insertions(+), 1 deletion(-) diff --git a/crates/nibrunnerd/src/adapters/vm/manager.rs b/crates/nibrunnerd/src/adapters/vm/manager.rs index 5101e770..5661ddfc 100644 --- a/crates/nibrunnerd/src/adapters/vm/manager.rs +++ b/crates/nibrunnerd/src/adapters/vm/manager.rs @@ -48,7 +48,10 @@ pub struct VmManager { impl VmManager { pub fn working_dir_for(&self, app_id: &AppId) -> PathBuf { - self.vm_dir.join(app_id.as_str()) + self.processes + .read_record(app_id) + .and_then(|record| record.jail_root) + .unwrap_or_else(|| self.vm_dir.join(app_id.as_str())) } fn api(&self, app_id: &AppId) -> FirecrackerApi { @@ -386,6 +389,21 @@ impl Vmm for VmManager { self.logs.detach(app_id).await; return Err(VmError::Host(error.to_string())); } + if self + .processes + .read_record(app_id) + .is_some_and(|record| record.jail_root.is_some()) + { + use std::os::unix::fs::MetadataExt; + let owner = std::fs::metadata(&working_dir).map_err(|error| VmError::Host(error.to_string()))?; + for socket in [ + tenant_log_socket_path(&working_dir), + cron_registration_socket_path(&working_dir), + ] { + crate::unix_socket::own(&socket, owner.uid(), owner.gid()) + .map_err(|error| VmError::Host(error.to_string()))?; + } + } Ok(()) } @@ -776,6 +794,76 @@ mod tests { ); } + #[tokio::test] + async fn a_running_legacy_vmm_keeps_its_existing_channels_until_it_is_redeployed() { + let fixture = fixture(); + fixture.state.put_record(instance_record(|_| {})).await; + fixture + .manager + .processes + .write_record(&crate::adapters::vm::process::VmRecord { + app_id: app_id(), + pid: std::process::id() as i32, + host_boot_id: fixture.manager.processes.boot_id().into(), + started_at_ms: 0, + exit_code: None, + signal: None, + stop_requested: false, + jail_root: None, + jail_uid: None, + }) + .unwrap(); + let legacy_directory = fixture.manager.vm_dir.join(app_id().as_str()); + assert_eq!(fixture.manager.working_dir_for(&app_id()), legacy_directory); + fixture.manager.readopt(&app_id()).await.unwrap(); + assert!( + tokio::net::UnixStream::connect(tenant_log_socket_path(&legacy_directory)) + .await + .is_ok() + ); + assert!( + tokio::net::UnixStream::connect(cron_registration_socket_path(&legacy_directory)) + .await + .is_ok() + ); + } + + #[tokio::test] + async fn a_recorded_jail_is_readopted_using_its_existing_identity() { + use std::os::unix::fs::MetadataExt; + let fixture = fixture(); + let short_directory = tempfile::tempdir_in("/tmp").unwrap(); + let root = short_directory.path().join("root"); + make_directory(&root, VM_DIR_MODE).unwrap(); + let identity = std::fs::metadata(&root).unwrap(); + fixture.state.put_record(instance_record(|_| {})).await; + fixture + .manager + .processes + .write_record(&crate::adapters::vm::process::VmRecord { + app_id: app_id(), + pid: std::process::id() as i32, + host_boot_id: fixture.manager.processes.boot_id().into(), + started_at_ms: 0, + exit_code: None, + signal: None, + stop_requested: false, + jail_root: Some(root.clone()), + jail_uid: None, + }) + .unwrap(); + fixture.manager.readopt(&app_id()).await.unwrap(); + assert_eq!(fixture.manager.working_dir_for(&app_id()), root); + for socket in [ + tenant_log_socket_path(&root), + cron_registration_socket_path(&root), + ] { + let owner = std::fs::metadata(&socket).unwrap(); + assert_eq!((owner.uid(), owner.gid()), (identity.uid(), identity.gid())); + assert!(tokio::net::UnixStream::connect(socket).await.is_ok()); + } + } + #[tokio::test] async fn readopting_a_microvm_this_host_holds_no_record_of_is_a_no_op() { let fixture = fixture(); diff --git a/crates/nibrunnerd/src/adapters/vm/process.rs b/crates/nibrunnerd/src/adapters/vm/process.rs index 5a11cc15..b646f337 100644 --- a/crates/nibrunnerd/src/adapters/vm/process.rs +++ b/crates/nibrunnerd/src/adapters/vm/process.rs @@ -65,6 +65,10 @@ pub struct VmRecord { pub signal: Option, #[serde(default)] pub stop_requested: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub jail_root: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub jail_uid: Option, } impl VmRecord { @@ -237,6 +241,8 @@ impl VmProcesses { exit_code: None, signal: None, stop_requested: false, + jail_root: None, + jail_uid: None, }; self.write_record(&record)?; @@ -335,6 +341,8 @@ mod tests { exit_code: None, signal: None, stop_requested: false, + jail_root: None, + jail_uid: None, }) .unwrap(); let status = processes.status(&app_id()); @@ -357,6 +365,8 @@ mod tests { exit_code: None, signal: None, stop_requested: false, + jail_root: None, + jail_uid: None, }) .unwrap(); let status = processes.status(&app_id()); @@ -377,6 +387,8 @@ mod tests { exit_code: Some(1), signal: None, stop_requested: false, + jail_root: None, + jail_uid: None, }; processes.write_record(&record).unwrap(); let crashed = processes.status(&app_id()); @@ -386,6 +398,8 @@ mod tests { processes .write_record(&VmRecord { stop_requested: true, + jail_root: None, + jail_uid: None, ..record.clone() }) .unwrap(); @@ -461,6 +475,8 @@ mod tests { exit_code: Some(0), signal: None, stop_requested: true, + jail_root: None, + jail_uid: None, }) .unwrap(); std::fs::write(processes.console_path(&app_id()), b"[nibrun] gone\n").unwrap(); @@ -640,6 +656,8 @@ mod tests { exit_code: None, signal: None, stop_requested: false, + jail_root: None, + jail_uid: None, }) .unwrap(); processes.stop(&app_id()).await; diff --git a/crates/nibrunnerd/src/unix_socket.rs b/crates/nibrunnerd/src/unix_socket.rs index ae90874f..edb8769e 100644 --- a/crates/nibrunnerd/src/unix_socket.rs +++ b/crates/nibrunnerd/src/unix_socket.rs @@ -57,6 +57,19 @@ pub(crate) fn bind(path: &Path) -> std::io::Result { Ok(listener) } +#[allow( + unsafe_code, + reason = "a VMM may replace a socket with a symlink; ownership must never follow it" +)] +pub(crate) fn own(path: &Path, uid: u32, gid: u32) -> std::io::Result<()> { + use std::os::unix::ffi::OsStrExt; + let name = std::ffi::CString::new(path.as_os_str().as_bytes()).map_err(std::io::Error::other)?; + if unsafe { libc::lchown(name.as_ptr(), uid, gid) } < 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) +} + pub(crate) async fn connect(path: &Path) -> std::io::Result { use std::os::unix::fs::MetadataExt;