From 42c271cdd03ad45feeb2ba1e240bd46a8600bf45 Mon Sep 17 00:00:00 2001 From: ilbertt Date: Sat, 3 Oct 2026 19:32:17 +0200 Subject: [PATCH 1/2] fix: reach guest services through their VMM directory --- crates/nibrunnerd/src/domain/filesystem/reader.rs | 13 ++++++++----- crates/nibrunnerd/src/domain/reconcile/exports.rs | 6 +----- crates/nibrunnerd/src/test_support/mocks.rs | 6 +++++- crates/nibrunnerd/src/test_support/mod.rs | 2 +- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/crates/nibrunnerd/src/domain/filesystem/reader.rs b/crates/nibrunnerd/src/domain/filesystem/reader.rs index e90e8597..ad9443d9 100644 --- a/crates/nibrunnerd/src/domain/filesystem/reader.rs +++ b/crates/nibrunnerd/src/domain/filesystem/reader.rs @@ -5,9 +5,8 @@ use crate::host::Host; use crate::ports::GuestReading; pub fn guest_vsock_path(host: &Host, app_id: &AppId) -> std::path::PathBuf { - host.config - .vm_dir() - .join(app_id.as_str()) + host.vms + .working_dir(app_id) .join(guest_contract::vsock::GUEST_VSOCK_FILENAME) } @@ -36,12 +35,16 @@ mod tests { use crate::test_support::*; #[tokio::test] - async fn each_guest_is_reached_on_a_socket_inside_its_own_microvm_directory() { + async fn each_guest_is_reached_in_the_directory_selected_by_its_vmm() { let host = test_host().await; let neighbour = AppId::parse("app-2").unwrap(); let path = guest_vsock_path(host.arc(), &app_id()); - assert!(path.starts_with(host.config.vm_dir())); + assert_eq!(path.parent().unwrap(), host.host.vms.working_dir(&app_id())); + assert_ne!( + path.parent().unwrap(), + host.config.vm_dir().join(app_id().as_str()) + ); assert!(path.ends_with(guest_contract::vsock::GUEST_VSOCK_FILENAME)); assert_eq!(path.parent().unwrap().file_name().unwrap(), app_id().as_str()); assert_ne!(path, guest_vsock_path(host.arc(), &neighbour)); diff --git a/crates/nibrunnerd/src/domain/reconcile/exports.rs b/crates/nibrunnerd/src/domain/reconcile/exports.rs index 7477d2f6..2919c5d8 100644 --- a/crates/nibrunnerd/src/domain/reconcile/exports.rs +++ b/crates/nibrunnerd/src/domain/reconcile/exports.rs @@ -108,11 +108,7 @@ async fn write_inner( staging_dir: &std::path::Path, ) -> Result { let crontab = registered_crontab(host, &desired.app_id).await?; - let vsock_path = host - .config - .vm_dir() - .join(desired.app_id.as_str()) - .join(guest_contract::vsock::GUEST_VSOCK_FILENAME); + let vsock_path = crate::domain::filesystem::reader::guest_vsock_path(host, &desired.app_id); let lease = frozen(&desired.app_id, &vsock_path) .await .map_err(|error| error.message())?; diff --git a/crates/nibrunnerd/src/test_support/mocks.rs b/crates/nibrunnerd/src/test_support/mocks.rs index 8adcb720..5d06f70d 100644 --- a/crates/nibrunnerd/src/test_support/mocks.rs +++ b/crates/nibrunnerd/src/test_support/mocks.rs @@ -157,6 +157,10 @@ impl VmmSpy { pub const NOWHERE_VM_DIR: &str = "/nowhere/vm"; pub fn vmm() -> (Arc, VmmSpy) { + vmm_under(PathBuf::from(NOWHERE_VM_DIR)) +} + +pub(crate) fn vmm_under(directory: PathBuf) -> (Arc, VmmSpy) { let spy = VmmSpy::default(); let mut vms = MockVmm::new(); @@ -211,7 +215,7 @@ pub fn vmm() -> (Arc, VmmSpy) { let verdict = spy.verdict.clone(); vms.expect_guest_verdict().returning(move |_| held(&verdict)); vms.expect_working_dir() - .returning(|app_id: &AppId| PathBuf::from(NOWHERE_VM_DIR).join(app_id.as_str())); + .returning(move |app_id: &AppId| directory.join(app_id.as_str())); (Arc::new(vms), spy) } diff --git a/crates/nibrunnerd/src/test_support/mod.rs b/crates/nibrunnerd/src/test_support/mod.rs index 5694c1b0..2b83adca 100644 --- a/crates/nibrunnerd/src/test_support/mod.rs +++ b/crates/nibrunnerd/src/test_support/mod.rs @@ -473,7 +473,7 @@ async fn test_host_over( let state = HostState::shared(); let metrics = Arc::new(crate::domain::metrics::HostMetrics::new()); let (commands, command_log) = mocks::commands_formatting(); - let (vms, vm_spy) = mocks::vmm(); + let (vms, vm_spy) = mocks::vmm_under(config.vm_dir().join("jailer")); let (exports, export_spy) = mocks::exports_accepting(); let artifacts: Arc = artifacts; let cron = Arc::new(crate::domain::cron::registry::CronRegistry::new( From 2e1985716091d9422ab9c9b2016d370542d7288d Mon Sep 17 00:00:00 2001 From: ilbertt Date: Sat, 3 Oct 2026 20:05:05 +0200 Subject: [PATCH 2/2] feat: assign persistent TAPs to non-root VMMs --- crates/nibrunnerd/src/adapters/net/tap.rs | 49 +++++++++++++++++++++ crates/nibrunnerd/src/test_support/mocks.rs | 10 +++++ crates/nibrunnerd/tests/integration.rs | 3 ++ 3 files changed, 62 insertions(+) diff --git a/crates/nibrunnerd/src/adapters/net/tap.rs b/crates/nibrunnerd/src/adapters/net/tap.rs index 74d64a79..99bc511e 100644 --- a/crates/nibrunnerd/src/adapters/net/tap.rs +++ b/crates/nibrunnerd/src/adapters/net/tap.rs @@ -36,6 +36,14 @@ pub trait HostNetwork: Send + Sync { async fn refresh_neighbour(&self, neighbour: &Neighbour) -> Result<(), NetworkError>; + async fn set_tap_owner(&self, tap_name: &str, _uid: u32) -> Result<(), NetworkError> { + Err(NetworkError { + what: "tap ownership", + device: tap_name.into(), + reason: "this network adapter cannot assign an owner".into(), + }) + } + // A tap outlives the process that made it, which is what lets a daemon restart leave every // tenant serving. The same persistence means nothing reclaims one, so an app that leaves has // to say so. @@ -140,6 +148,43 @@ mod linux { Ok(()) } + #[allow( + unsafe_code, + reason = "assigning a persistent TAP to its jailed VMM requires TUNSETOWNER" + )] + fn set_tap_owner(tap_name: &str, uid: u32) -> Result<(), NetworkError> { + use std::os::fd::AsRawFd; + const TUNSETIFF: libc::Ioctl = 0x400454ca; + const TUNSETOWNER: libc::Ioctl = 0x400454cc; + const IFF_TAP: libc::c_short = 0x0002; + const IFF_NO_PI: libc::c_short = 0x1000; + #[repr(C)] + struct InterfaceRequest { + name: [libc::c_char; libc::IFNAMSIZ], + flags: libc::c_short, + padding: [u8; 22], + } + let device = std::fs::OpenOptions::new() + .read(true) + .write(true) + .open("/dev/net/tun") + .map_err(|error| failed("tap ownership", tap_name, error))?; + let mut request = InterfaceRequest { + name: [0; libc::IFNAMSIZ], + flags: IFF_TAP | IFF_NO_PI, + padding: [0; 22], + }; + for (index, byte) in tap_name.as_bytes().iter().take(libc::IFNAMSIZ - 1).enumerate() { + request.name[index] = *byte as libc::c_char; + } + if unsafe { libc::ioctl(device.as_raw_fd(), TUNSETIFF, &mut request) } < 0 + || unsafe { libc::ioctl(device.as_raw_fd(), TUNSETOWNER, uid) } < 0 + { + return Err(failed("tap ownership", tap_name, std::io::Error::last_os_error())); + } + Ok(()) + } + fn allow_route_localnet(tap_name: &str) -> Result<(), NetworkError> { let path = format!("/proc/sys/net/ipv4/conf/{tap_name}/route_localnet"); std::fs::write(&path, b"1").map_err(|error| failed("route_localnet", tap_name, error)) @@ -171,6 +216,10 @@ mod linux { allow_route_localnet(&tap.tap_name) } + async fn set_tap_owner(&self, tap_name: &str, uid: u32) -> Result<(), NetworkError> { + set_tap_owner(tap_name, uid) + } + async fn refresh_neighbour(&self, neighbour: &Neighbour) -> Result<(), NetworkError> { let index = self .index_of(&neighbour.tap_name) diff --git a/crates/nibrunnerd/src/test_support/mocks.rs b/crates/nibrunnerd/src/test_support/mocks.rs index 5d06f70d..bd80f80b 100644 --- a/crates/nibrunnerd/src/test_support/mocks.rs +++ b/crates/nibrunnerd/src/test_support/mocks.rs @@ -505,12 +505,17 @@ pub fn exports_answering( #[derive(Clone, Default)] pub struct NetworkSpy { + owners: Arc>>, taps: Arc>>, neighbours: Arc>>, removed: Arc>>, } impl NetworkSpy { + pub fn owners(&self) -> Vec<(String, u32)> { + held(&self.owners) + } + pub fn taps(&self) -> Vec { held(&self.taps) } @@ -533,6 +538,11 @@ pub fn network() -> (Arc, NetworkSpy) { push(&taps, tap.clone()); Ok(()) }); + let owners = spy.owners.clone(); + network.expect_set_tap_owner().returning(move |name, uid| { + push(&owners, (name.to_string(), uid)); + Ok(()) + }); let neighbours = spy.neighbours.clone(); network .expect_refresh_neighbour() diff --git a/crates/nibrunnerd/tests/integration.rs b/crates/nibrunnerd/tests/integration.rs index 38daab99..138aafa7 100644 --- a/crates/nibrunnerd/tests/integration.rs +++ b/crates/nibrunnerd/tests/integration.rs @@ -270,6 +270,9 @@ async fn a_tap_is_created_addressed_and_given_the_guest_it_will_hold() { .await .expect("a second pass changes nothing"); assert!(network.tap_names().await.contains(&slot.tap_name)); + network.set_tap_owner(&slot.tap_name, 100_000).await.unwrap(); + let owner = std::fs::read_to_string(format!("/sys/class/net/{}/owner", slot.tap_name)).unwrap(); + assert_eq!(owner.trim(), "100000"); network .refresh_neighbour(&Neighbour {