From 42c271cdd03ad45feeb2ba1e240bd46a8600bf45 Mon Sep 17 00:00:00 2001 From: ilbertt Date: Sat, 3 Oct 2026 19:32:17 +0200 Subject: [PATCH 1/6] fix: reach guest services through their VMM directory --- crates/nibrunnerd/src/domain/filesystem/reader.rs | 13 ++++++++----- crates/nibrunnerd/src/domain/reconcile/exports.rs | 6 +----- crates/nibrunnerd/src/test_support/mocks.rs | 6 +++++- crates/nibrunnerd/src/test_support/mod.rs | 2 +- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/crates/nibrunnerd/src/domain/filesystem/reader.rs b/crates/nibrunnerd/src/domain/filesystem/reader.rs index e90e8597..ad9443d9 100644 --- a/crates/nibrunnerd/src/domain/filesystem/reader.rs +++ b/crates/nibrunnerd/src/domain/filesystem/reader.rs @@ -5,9 +5,8 @@ use crate::host::Host; use crate::ports::GuestReading; pub fn guest_vsock_path(host: &Host, app_id: &AppId) -> std::path::PathBuf { - host.config - .vm_dir() - .join(app_id.as_str()) + host.vms + .working_dir(app_id) .join(guest_contract::vsock::GUEST_VSOCK_FILENAME) } @@ -36,12 +35,16 @@ mod tests { use crate::test_support::*; #[tokio::test] - async fn each_guest_is_reached_on_a_socket_inside_its_own_microvm_directory() { + async fn each_guest_is_reached_in_the_directory_selected_by_its_vmm() { let host = test_host().await; let neighbour = AppId::parse("app-2").unwrap(); let path = guest_vsock_path(host.arc(), &app_id()); - assert!(path.starts_with(host.config.vm_dir())); + assert_eq!(path.parent().unwrap(), host.host.vms.working_dir(&app_id())); + assert_ne!( + path.parent().unwrap(), + host.config.vm_dir().join(app_id().as_str()) + ); assert!(path.ends_with(guest_contract::vsock::GUEST_VSOCK_FILENAME)); assert_eq!(path.parent().unwrap().file_name().unwrap(), app_id().as_str()); assert_ne!(path, guest_vsock_path(host.arc(), &neighbour)); diff --git a/crates/nibrunnerd/src/domain/reconcile/exports.rs b/crates/nibrunnerd/src/domain/reconcile/exports.rs index 7477d2f6..2919c5d8 100644 --- a/crates/nibrunnerd/src/domain/reconcile/exports.rs +++ b/crates/nibrunnerd/src/domain/reconcile/exports.rs @@ -108,11 +108,7 @@ async fn write_inner( staging_dir: &std::path::Path, ) -> Result { let crontab = registered_crontab(host, &desired.app_id).await?; - let vsock_path = host - .config - .vm_dir() - .join(desired.app_id.as_str()) - .join(guest_contract::vsock::GUEST_VSOCK_FILENAME); + let vsock_path = crate::domain::filesystem::reader::guest_vsock_path(host, &desired.app_id); let lease = frozen(&desired.app_id, &vsock_path) .await .map_err(|error| error.message())?; diff --git a/crates/nibrunnerd/src/test_support/mocks.rs b/crates/nibrunnerd/src/test_support/mocks.rs index 8adcb720..5d06f70d 100644 --- a/crates/nibrunnerd/src/test_support/mocks.rs +++ b/crates/nibrunnerd/src/test_support/mocks.rs @@ -157,6 +157,10 @@ impl VmmSpy { pub const NOWHERE_VM_DIR: &str = "/nowhere/vm"; pub fn vmm() -> (Arc, VmmSpy) { + vmm_under(PathBuf::from(NOWHERE_VM_DIR)) +} + +pub(crate) fn vmm_under(directory: PathBuf) -> (Arc, VmmSpy) { let spy = VmmSpy::default(); let mut vms = MockVmm::new(); @@ -211,7 +215,7 @@ pub fn vmm() -> (Arc, VmmSpy) { let verdict = spy.verdict.clone(); vms.expect_guest_verdict().returning(move |_| held(&verdict)); vms.expect_working_dir() - .returning(|app_id: &AppId| PathBuf::from(NOWHERE_VM_DIR).join(app_id.as_str())); + .returning(move |app_id: &AppId| directory.join(app_id.as_str())); (Arc::new(vms), spy) } diff --git a/crates/nibrunnerd/src/test_support/mod.rs b/crates/nibrunnerd/src/test_support/mod.rs index 5694c1b0..2b83adca 100644 --- a/crates/nibrunnerd/src/test_support/mod.rs +++ b/crates/nibrunnerd/src/test_support/mod.rs @@ -473,7 +473,7 @@ async fn test_host_over( let state = HostState::shared(); let metrics = Arc::new(crate::domain::metrics::HostMetrics::new()); let (commands, command_log) = mocks::commands_formatting(); - let (vms, vm_spy) = mocks::vmm(); + let (vms, vm_spy) = mocks::vmm_under(config.vm_dir().join("jailer")); let (exports, export_spy) = mocks::exports_accepting(); let artifacts: Arc = artifacts; let cron = Arc::new(crate::domain::cron::registry::CronRegistry::new( From 2e1985716091d9422ab9c9b2016d370542d7288d Mon Sep 17 00:00:00 2001 From: ilbertt Date: Sat, 3 Oct 2026 20:05:05 +0200 Subject: [PATCH 2/6] feat: assign persistent TAPs to non-root VMMs --- crates/nibrunnerd/src/adapters/net/tap.rs | 49 +++++++++++++++++++++ crates/nibrunnerd/src/test_support/mocks.rs | 10 +++++ crates/nibrunnerd/tests/integration.rs | 3 ++ 3 files changed, 62 insertions(+) diff --git a/crates/nibrunnerd/src/adapters/net/tap.rs b/crates/nibrunnerd/src/adapters/net/tap.rs index 74d64a79..99bc511e 100644 --- a/crates/nibrunnerd/src/adapters/net/tap.rs +++ b/crates/nibrunnerd/src/adapters/net/tap.rs @@ -36,6 +36,14 @@ pub trait HostNetwork: Send + Sync { async fn refresh_neighbour(&self, neighbour: &Neighbour) -> Result<(), NetworkError>; + async fn set_tap_owner(&self, tap_name: &str, _uid: u32) -> Result<(), NetworkError> { + Err(NetworkError { + what: "tap ownership", + device: tap_name.into(), + reason: "this network adapter cannot assign an owner".into(), + }) + } + // A tap outlives the process that made it, which is what lets a daemon restart leave every // tenant serving. The same persistence means nothing reclaims one, so an app that leaves has // to say so. @@ -140,6 +148,43 @@ mod linux { Ok(()) } + #[allow( + unsafe_code, + reason = "assigning a persistent TAP to its jailed VMM requires TUNSETOWNER" + )] + fn set_tap_owner(tap_name: &str, uid: u32) -> Result<(), NetworkError> { + use std::os::fd::AsRawFd; + const TUNSETIFF: libc::Ioctl = 0x400454ca; + const TUNSETOWNER: libc::Ioctl = 0x400454cc; + const IFF_TAP: libc::c_short = 0x0002; + const IFF_NO_PI: libc::c_short = 0x1000; + #[repr(C)] + struct InterfaceRequest { + name: [libc::c_char; libc::IFNAMSIZ], + flags: libc::c_short, + padding: [u8; 22], + } + let device = std::fs::OpenOptions::new() + .read(true) + .write(true) + .open("/dev/net/tun") + .map_err(|error| failed("tap ownership", tap_name, error))?; + let mut request = InterfaceRequest { + name: [0; libc::IFNAMSIZ], + flags: IFF_TAP | IFF_NO_PI, + padding: [0; 22], + }; + for (index, byte) in tap_name.as_bytes().iter().take(libc::IFNAMSIZ - 1).enumerate() { + request.name[index] = *byte as libc::c_char; + } + if unsafe { libc::ioctl(device.as_raw_fd(), TUNSETIFF, &mut request) } < 0 + || unsafe { libc::ioctl(device.as_raw_fd(), TUNSETOWNER, uid) } < 0 + { + return Err(failed("tap ownership", tap_name, std::io::Error::last_os_error())); + } + Ok(()) + } + fn allow_route_localnet(tap_name: &str) -> Result<(), NetworkError> { let path = format!("/proc/sys/net/ipv4/conf/{tap_name}/route_localnet"); std::fs::write(&path, b"1").map_err(|error| failed("route_localnet", tap_name, error)) @@ -171,6 +216,10 @@ mod linux { allow_route_localnet(&tap.tap_name) } + async fn set_tap_owner(&self, tap_name: &str, uid: u32) -> Result<(), NetworkError> { + set_tap_owner(tap_name, uid) + } + async fn refresh_neighbour(&self, neighbour: &Neighbour) -> Result<(), NetworkError> { let index = self .index_of(&neighbour.tap_name) diff --git a/crates/nibrunnerd/src/test_support/mocks.rs b/crates/nibrunnerd/src/test_support/mocks.rs index 5d06f70d..bd80f80b 100644 --- a/crates/nibrunnerd/src/test_support/mocks.rs +++ b/crates/nibrunnerd/src/test_support/mocks.rs @@ -505,12 +505,17 @@ pub fn exports_answering( #[derive(Clone, Default)] pub struct NetworkSpy { + owners: Arc>>, taps: Arc>>, neighbours: Arc>>, removed: Arc>>, } impl NetworkSpy { + pub fn owners(&self) -> Vec<(String, u32)> { + held(&self.owners) + } + pub fn taps(&self) -> Vec { held(&self.taps) } @@ -533,6 +538,11 @@ pub fn network() -> (Arc, NetworkSpy) { push(&taps, tap.clone()); Ok(()) }); + let owners = spy.owners.clone(); + network.expect_set_tap_owner().returning(move |name, uid| { + push(&owners, (name.to_string(), uid)); + Ok(()) + }); let neighbours = spy.neighbours.clone(); network .expect_refresh_neighbour() diff --git a/crates/nibrunnerd/tests/integration.rs b/crates/nibrunnerd/tests/integration.rs index 38daab99..138aafa7 100644 --- a/crates/nibrunnerd/tests/integration.rs +++ b/crates/nibrunnerd/tests/integration.rs @@ -270,6 +270,9 @@ async fn a_tap_is_created_addressed_and_given_the_guest_it_will_hold() { .await .expect("a second pass changes nothing"); assert!(network.tap_names().await.contains(&slot.tap_name)); + network.set_tap_owner(&slot.tap_name, 100_000).await.unwrap(); + let owner = std::fs::read_to_string(format!("/sys/class/net/{}/owner", slot.tap_name)).unwrap(); + assert_eq!(owner.trim(), "100000"); network .refresh_neighbour(&Neighbour { From 5d3e0d38c1a73a1157d0cecb57cb42f18eedcf45 Mon Sep 17 00:00:00 2001 From: ilbertt Date: Sat, 3 Oct 2026 20:05:44 +0200 Subject: [PATCH 3/6] refactor: separate VMM command construction from supervision --- crates/nibrunnerd/src/adapters/vm/process.rs | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/crates/nibrunnerd/src/adapters/vm/process.rs b/crates/nibrunnerd/src/adapters/vm/process.rs index 0b2109ee..5a11cc15 100644 --- a/crates/nibrunnerd/src/adapters/vm/process.rs +++ b/crates/nibrunnerd/src/adapters/vm/process.rs @@ -191,19 +191,27 @@ impl VmProcesses { let _ = std::fs::remove_file(&api_socket); let _ = std::fs::remove_file(working_dir.join(guest_contract::vsock::GUEST_VSOCK_FILENAME)); - let console = std::fs::File::create(self.console_path(app_id))?; let mut command = tokio::process::Command::new(binary); + command.arg("--api-sock").arg(&api_socket); + if let Some(config_file) = config_file { + command.arg("--config-file").arg(config_file); + } + self.launch(app_id, command, working_dir).await + } + + async fn launch( + &self, + app_id: &AppId, + mut command: tokio::process::Command, + working_dir: &Path, + ) -> std::io::Result { + let console = std::fs::File::create(self.console_path(app_id))?; command - .arg("--api-sock") - .arg(&api_socket) .current_dir(working_dir) .stdin(Stdio::null()) .stdout(Stdio::from(console.try_clone()?)) .stderr(Stdio::from(console)) .kill_on_drop(false); - if let Some(config_file) = config_file { - command.arg("--config-file").arg(config_file); - } #[cfg(unix)] { #[allow( From f02916cf1446e27916acc4d492717789ae755b5e Mon Sep 17 00:00:00 2001 From: ilbertt Date: Sat, 3 Oct 2026 20:06:37 +0200 Subject: [PATCH 4/6] feat: readopt VMMs from their recorded root and identity --- crates/nibrunnerd/src/adapters/vm/manager.rs | 90 +++++++++++++++++++- crates/nibrunnerd/src/adapters/vm/process.rs | 18 ++++ crates/nibrunnerd/src/unix_socket.rs | 13 +++ 3 files changed, 120 insertions(+), 1 deletion(-) diff --git a/crates/nibrunnerd/src/adapters/vm/manager.rs b/crates/nibrunnerd/src/adapters/vm/manager.rs index 5101e770..5661ddfc 100644 --- a/crates/nibrunnerd/src/adapters/vm/manager.rs +++ b/crates/nibrunnerd/src/adapters/vm/manager.rs @@ -48,7 +48,10 @@ pub struct VmManager { impl VmManager { pub fn working_dir_for(&self, app_id: &AppId) -> PathBuf { - self.vm_dir.join(app_id.as_str()) + self.processes + .read_record(app_id) + .and_then(|record| record.jail_root) + .unwrap_or_else(|| self.vm_dir.join(app_id.as_str())) } fn api(&self, app_id: &AppId) -> FirecrackerApi { @@ -386,6 +389,21 @@ impl Vmm for VmManager { self.logs.detach(app_id).await; return Err(VmError::Host(error.to_string())); } + if self + .processes + .read_record(app_id) + .is_some_and(|record| record.jail_root.is_some()) + { + use std::os::unix::fs::MetadataExt; + let owner = std::fs::metadata(&working_dir).map_err(|error| VmError::Host(error.to_string()))?; + for socket in [ + tenant_log_socket_path(&working_dir), + cron_registration_socket_path(&working_dir), + ] { + crate::unix_socket::own(&socket, owner.uid(), owner.gid()) + .map_err(|error| VmError::Host(error.to_string()))?; + } + } Ok(()) } @@ -776,6 +794,76 @@ mod tests { ); } + #[tokio::test] + async fn a_running_legacy_vmm_keeps_its_existing_channels_until_it_is_redeployed() { + let fixture = fixture(); + fixture.state.put_record(instance_record(|_| {})).await; + fixture + .manager + .processes + .write_record(&crate::adapters::vm::process::VmRecord { + app_id: app_id(), + pid: std::process::id() as i32, + host_boot_id: fixture.manager.processes.boot_id().into(), + started_at_ms: 0, + exit_code: None, + signal: None, + stop_requested: false, + jail_root: None, + jail_uid: None, + }) + .unwrap(); + let legacy_directory = fixture.manager.vm_dir.join(app_id().as_str()); + assert_eq!(fixture.manager.working_dir_for(&app_id()), legacy_directory); + fixture.manager.readopt(&app_id()).await.unwrap(); + assert!( + tokio::net::UnixStream::connect(tenant_log_socket_path(&legacy_directory)) + .await + .is_ok() + ); + assert!( + tokio::net::UnixStream::connect(cron_registration_socket_path(&legacy_directory)) + .await + .is_ok() + ); + } + + #[tokio::test] + async fn a_recorded_jail_is_readopted_using_its_existing_identity() { + use std::os::unix::fs::MetadataExt; + let fixture = fixture(); + let short_directory = tempfile::tempdir_in("/tmp").unwrap(); + let root = short_directory.path().join("root"); + make_directory(&root, VM_DIR_MODE).unwrap(); + let identity = std::fs::metadata(&root).unwrap(); + fixture.state.put_record(instance_record(|_| {})).await; + fixture + .manager + .processes + .write_record(&crate::adapters::vm::process::VmRecord { + app_id: app_id(), + pid: std::process::id() as i32, + host_boot_id: fixture.manager.processes.boot_id().into(), + started_at_ms: 0, + exit_code: None, + signal: None, + stop_requested: false, + jail_root: Some(root.clone()), + jail_uid: None, + }) + .unwrap(); + fixture.manager.readopt(&app_id()).await.unwrap(); + assert_eq!(fixture.manager.working_dir_for(&app_id()), root); + for socket in [ + tenant_log_socket_path(&root), + cron_registration_socket_path(&root), + ] { + let owner = std::fs::metadata(&socket).unwrap(); + assert_eq!((owner.uid(), owner.gid()), (identity.uid(), identity.gid())); + assert!(tokio::net::UnixStream::connect(socket).await.is_ok()); + } + } + #[tokio::test] async fn readopting_a_microvm_this_host_holds_no_record_of_is_a_no_op() { let fixture = fixture(); diff --git a/crates/nibrunnerd/src/adapters/vm/process.rs b/crates/nibrunnerd/src/adapters/vm/process.rs index 5a11cc15..b646f337 100644 --- a/crates/nibrunnerd/src/adapters/vm/process.rs +++ b/crates/nibrunnerd/src/adapters/vm/process.rs @@ -65,6 +65,10 @@ pub struct VmRecord { pub signal: Option, #[serde(default)] pub stop_requested: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub jail_root: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub jail_uid: Option, } impl VmRecord { @@ -237,6 +241,8 @@ impl VmProcesses { exit_code: None, signal: None, stop_requested: false, + jail_root: None, + jail_uid: None, }; self.write_record(&record)?; @@ -335,6 +341,8 @@ mod tests { exit_code: None, signal: None, stop_requested: false, + jail_root: None, + jail_uid: None, }) .unwrap(); let status = processes.status(&app_id()); @@ -357,6 +365,8 @@ mod tests { exit_code: None, signal: None, stop_requested: false, + jail_root: None, + jail_uid: None, }) .unwrap(); let status = processes.status(&app_id()); @@ -377,6 +387,8 @@ mod tests { exit_code: Some(1), signal: None, stop_requested: false, + jail_root: None, + jail_uid: None, }; processes.write_record(&record).unwrap(); let crashed = processes.status(&app_id()); @@ -386,6 +398,8 @@ mod tests { processes .write_record(&VmRecord { stop_requested: true, + jail_root: None, + jail_uid: None, ..record.clone() }) .unwrap(); @@ -461,6 +475,8 @@ mod tests { exit_code: Some(0), signal: None, stop_requested: true, + jail_root: None, + jail_uid: None, }) .unwrap(); std::fs::write(processes.console_path(&app_id()), b"[nibrun] gone\n").unwrap(); @@ -640,6 +656,8 @@ mod tests { exit_code: None, signal: None, stop_requested: false, + jail_root: None, + jail_uid: None, }) .unwrap(); processes.stop(&app_id()).await; diff --git a/crates/nibrunnerd/src/unix_socket.rs b/crates/nibrunnerd/src/unix_socket.rs index ae90874f..edb8769e 100644 --- a/crates/nibrunnerd/src/unix_socket.rs +++ b/crates/nibrunnerd/src/unix_socket.rs @@ -57,6 +57,19 @@ pub(crate) fn bind(path: &Path) -> std::io::Result { Ok(listener) } +#[allow( + unsafe_code, + reason = "a VMM may replace a socket with a symlink; ownership must never follow it" +)] +pub(crate) fn own(path: &Path, uid: u32, gid: u32) -> std::io::Result<()> { + use std::os::unix::ffi::OsStrExt; + let name = std::ffi::CString::new(path.as_os_str().as_bytes()).map_err(std::io::Error::other)?; + if unsafe { libc::lchown(name.as_ptr(), uid, gid) } < 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) +} + pub(crate) async fn connect(path: &Path) -> std::io::Result { use std::os::unix::fs::MetadataExt; From 68740a13cfb295f1ca16da8c2cbff31981c83bf8 Mon Sep 17 00:00:00 2001 From: ilbertt Date: Sat, 3 Oct 2026 20:07:04 +0200 Subject: [PATCH 5/6] fix: verify recorded VMM peers before API commands --- .../src/adapters/vm/firecracker_api.rs | 48 +++++++++++++++++++ crates/nibrunnerd/src/adapters/vm/manager.rs | 6 ++- 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/crates/nibrunnerd/src/adapters/vm/firecracker_api.rs b/crates/nibrunnerd/src/adapters/vm/firecracker_api.rs index 8f720a0e..d1932935 100644 --- a/crates/nibrunnerd/src/adapters/vm/firecracker_api.rs +++ b/crates/nibrunnerd/src/adapters/vm/firecracker_api.rs @@ -46,15 +46,22 @@ struct LoadSnapshot<'a> { pub struct FirecrackerApi { socket_path: PathBuf, + expected_process: Option<(i32, Option)>, } impl FirecrackerApi { pub fn at(socket_path: impl Into) -> Self { Self { socket_path: socket_path.into(), + expected_process: None, } } + pub(crate) fn require_process(mut self, pid: i32, uid: Option) -> Self { + self.expected_process = Some((pid, uid)); + self + } + async fn call(&self, method: Method, path: &str, body: &B) -> Result<(), VmError> { let unreachable = |reason: String| VmError::Unreachable { socket_path: self.socket_path.display().to_string(), @@ -64,6 +71,26 @@ impl FirecrackerApi { let stream = tokio::net::UnixStream::connect(&self.socket_path) .await .map_err(|error| unreachable(error.to_string()))?; + if let Some((pid, uid)) = self.expected_process { + #[cfg(target_os = "linux")] + { + let peer = stream + .peer_cred() + .map_err(|error| unreachable(error.to_string()))?; + if peer.pid() != Some(pid) || uid.is_some_and(|uid| peer.uid() != uid) { + return Err(VmError::Host( + "the Firecracker API socket does not belong to the recorded VMM".into(), + )); + } + } + #[cfg(not(target_os = "linux"))] + { + let _ = (pid, uid); + return Err(VmError::Host( + "checking a jailed VMM's API peer requires Linux".into(), + )); + } + } let (mut sender, connection) = hyper::client::conn::http1::handshake(TokioIo::new(stream)) .await .map_err(|error| unreachable(error.to_string()))?; @@ -372,4 +399,25 @@ mod tests { let error = FirecrackerApi::at(&absent).pause().await.unwrap_err(); assert!(error.message().contains(&absent.display().to_string()), "{error}"); } + #[cfg(target_os = "linux")] + #[tokio::test] + async fn a_jailed_api_refuses_another_process_before_sending_any_command() { + let directory = tempfile::tempdir().unwrap(); + let (socket, seen) = FakeVmm::listening(directory.path(), hyper::StatusCode::NO_CONTENT, "").await; + let api = FirecrackerApi::at(socket).require_process(std::process::id() as i32 + 1, None); + assert!(matches!(api.pause().await, Err(VmError::Host(_)))); + assert!(seen.lock().unwrap().is_empty()); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn a_jailed_api_refuses_an_unexpected_uid_before_sending_any_command() { + use std::os::unix::fs::MetadataExt; + let directory = tempfile::tempdir().unwrap(); + let uid = std::fs::metadata(directory.path()).unwrap().uid(); + let (socket, seen) = FakeVmm::listening(directory.path(), hyper::StatusCode::NO_CONTENT, "").await; + let api = FirecrackerApi::at(socket).require_process(std::process::id() as i32, Some(uid + 1)); + assert!(matches!(api.pause().await, Err(VmError::Host(_)))); + assert!(seen.lock().unwrap().is_empty()); + } } diff --git a/crates/nibrunnerd/src/adapters/vm/manager.rs b/crates/nibrunnerd/src/adapters/vm/manager.rs index 5661ddfc..dcfd6d94 100644 --- a/crates/nibrunnerd/src/adapters/vm/manager.rs +++ b/crates/nibrunnerd/src/adapters/vm/manager.rs @@ -55,7 +55,11 @@ impl VmManager { } fn api(&self, app_id: &AppId) -> FirecrackerApi { - FirecrackerApi::at(self.processes.api_socket(app_id)) + let api = FirecrackerApi::at(self.processes.api_socket(app_id)); + match self.processes.read_record(app_id) { + Some(record) if record.jail_root.is_some() => api.require_process(record.pid, record.jail_uid), + _ => api, + } } fn current_stamp(&self, request: &SuspendRequest) -> SnapshotStamp { From b5bf96004a8059c7563ba58ce606c2bd34d6bd9c Mon Sep 17 00:00:00 2001 From: ilbertt Date: Sat, 3 Oct 2026 20:07:38 +0200 Subject: [PATCH 6/6] feat: reserve stable non-root identities during installation --- .../src/install/jailer_identities.rs | 232 ++++++++++++++++++ crates/nibrunnerd/src/install/mod.rs | 9 + .../nibrunnerd/src/install/prerequisites.rs | 7 +- 3 files changed, 243 insertions(+), 5 deletions(-) create mode 100644 crates/nibrunnerd/src/install/jailer_identities.rs diff --git a/crates/nibrunnerd/src/install/jailer_identities.rs b/crates/nibrunnerd/src/install/jailer_identities.rs new file mode 100644 index 00000000..c4955a79 --- /dev/null +++ b/crates/nibrunnerd/src/install/jailer_identities.rs @@ -0,0 +1,232 @@ +use super::InstallError; + +const ACCOUNT: &str = "nibrunner-jailer"; +const RESERVATION_SIZE: u32 = 65_536; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct Identities { + pub(crate) uid_base: u32, + pub(crate) gid_base: u32, +} + +struct Reservation { + owner: String, + start: u32, + end: u32, +} + +fn ask(arguments: &[&str]) -> Result { + let output = std::process::Command::new("getent") + .args(arguments) + .output() + .map_err(|error| format!("getent could not read the jailer identities: {error}"))?; + if !output.status.success() { + return Err(format!( + "the jailer {} could not be read; run nibrunnerd install", + arguments[0] + )); + } + String::from_utf8(output.stdout).map_err(|_| "the account database is not UTF-8".into()) +} + +fn reservations(text: &str) -> Result, String> { + text.lines() + .filter(|line| !line.trim().is_empty() && !line.starts_with('#')) + .map(|line| { + let fields: Vec<_> = line.split(':').collect(); + let invalid = || "a subordinate identity reservation is malformed".to_string(); + if fields.len() != 3 || fields[0].is_empty() { + return Err(invalid()); + } + let start: u32 = fields[1].parse().map_err(|_| invalid())?; + let count: u32 = fields[2].parse().map_err(|_| invalid())?; + let end = start.checked_add(count).ok_or_else(invalid)?; + if start == 0 || count == 0 { + return Err(invalid()); + } + Ok(Reservation { + owner: fields[0].to_owned(), + start, + end, + }) + }) + .collect() +} + +fn reserved_base(text: &str, accounts: &str, uid: u32, max_apps: u32) -> Result { + let ranges = reservations(text)?; + let numeric_owner = uid.to_string(); + let mut own = ranges + .iter() + .filter(|range| range.owner == ACCOUNT || range.owner == numeric_owner); + let selected = own.next().ok_or_else(|| { + format!("{ACCOUNT} has no subordinate identity reservation; run nibrunnerd install") + })?; + if own.next().is_some() { + return Err(format!( + "{ACCOUNT} must have exactly one subordinate identity reservation" + )); + } + if max_apps == 0 || selected.end - selected.start < max_apps { + return Err(format!( + "the jailer identity reservation does not fit {max_apps} app slots" + )); + } + for range in &ranges { + if !std::ptr::eq(range, selected) && range.start < selected.end && selected.start < range.end { + return Err(format!( + "the jailer identity reservation overlaps {}", + range.owner + )); + } + } + for line in accounts.lines().filter(|line| !line.is_empty()) { + let fields: Vec<_> = line.split(':').collect(); + let id: u32 = fields + .get(2) + .and_then(|field| field.parse().ok()) + .ok_or_else(|| "an account database entry is malformed".to_string())?; + if (selected.start..selected.end).contains(&id) { + return Err(format!( + "the jailer identity reservation contains the {} account", + fields[0] + )); + } + } + Ok(selected.start) +} + +fn account_uid(passwd: &str, shadow: &str) -> Result { + let fields: Vec<_> = passwd.trim_end().split(':').collect(); + let password: Vec<_> = shadow.trim_end().split(':').collect(); + if fields.len() != 7 + || fields[0] != ACCOUNT + || !matches!(fields[6], "/usr/sbin/nologin" | "/sbin/nologin") + || password.len() != 9 + || password[0] != ACCOUNT + || !password[1].starts_with(['!', '*']) + { + return Err(format!("{ACCOUNT} must be a locked account with a nologin shell")); + } + let uid: u32 = fields[2] + .parse() + .map_err(|_| format!("{ACCOUNT} has an invalid UID"))?; + let gid: u32 = fields[3] + .parse() + .map_err(|_| format!("{ACCOUNT} has an invalid GID"))?; + if uid == 0 || gid == 0 || uid == u32::MAX || gid == u32::MAX { + return Err(format!("{ACCOUNT} must have a non-root UID and GID")); + } + Ok(uid) +} + +pub(crate) fn read(max_apps: u32) -> Result { + read_identities(max_apps).map_err(InstallError::Refused) +} + +fn read_identities(max_apps: u32) -> Result { + let uid = account_uid(&ask(&["passwd", ACCOUNT])?, &ask(&["shadow", ACCOUNT])?)?; + let base = |file: &str, database: &str| { + let text = std::fs::read_to_string(file) + .map_err(|error| format!("{file} could not be read: {error}; run nibrunnerd install"))?; + reserved_base(&text, &ask(&[database])?, uid, max_apps).map_err(|error| format!("{file}: {error}")) + }; + Ok(Identities { + uid_base: base("/etc/subuid", "passwd")?, + gid_base: base("/etc/subgid", "group")?, + }) +} + +pub(crate) fn ensure(max_apps: u32) -> Result { + reserve_identities(max_apps).map_err(InstallError::Refused) +} + +fn reserve_identities(max_apps: u32) -> Result { + if super::service_user::ids(ACCOUNT).is_some() { + read_identities(max_apps)?; + return Ok(false); + } + if max_apps == 0 || max_apps > RESERVATION_SIZE { + return Err("the requested app slots do not fit the jailer identity reservation".into()); + } + // useradd holds the subordinate-ID database locks and avoids other users' reservations. + let output = std::process::Command::new("useradd") + .args([ + "--system", + "--user-group", + "--no-create-home", + "--shell", + "/usr/sbin/nologin", + "--home-dir", + "/nonexistent", + "--add-subids-for-system", + "--key", + &format!("SUB_UID_COUNT={RESERVATION_SIZE}"), + "--key", + &format!("SUB_GID_COUNT={RESERVATION_SIZE}"), + ACCOUNT, + ]) + .output() + .map_err(|error| format!("the jailer identity reservation could not be created: {error}"))?; + if !output.status.success() { + return Err(format!( + "the jailer identity reservation could not be created: {}", + String::from_utf8_lossy(&output.stderr).trim() + )); + } + read_identities(max_apps)?; + Ok(true) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_reserved_range_is_stable_and_fits_every_slot() { + let text = "container:100000:65536\nnibrunner-jailer:165536:65536\n"; + let accounts = "root:x:0:0:root:/root:/bin/sh\n"; + assert_eq!(reserved_base(text, accounts, 999, 1000).unwrap(), 165536); + assert_eq!(reserved_base(text, accounts, 999, 65536).unwrap(), 165536); + assert!(reserved_base(text, accounts, 999, 65537).is_err()); + } + + #[test] + fn a_reservation_may_name_its_owner_by_numeric_uid() { + assert_eq!(reserved_base("999:200000:8", "", 999, 8).unwrap(), 200000); + } + + #[test] + fn overlapping_container_ranges_and_host_accounts_are_refused() { + assert!(reserved_base("nibrunner-jailer:200000:8\ncontainer:200004:8", "", 999, 8).is_err()); + for accounts in ["user:x:200004:1000:User:/home/user:/bin/sh", "group:x:200004:"] { + assert!(reserved_base("nibrunner-jailer:200000:8", accounts, 999, 8).is_err()); + } + assert!(reserved_base("nibrunner-jailer:200000:8\ncontainer:199999:9", "", 999, 8).is_err()); + assert!(reserved_base("nibrunner-jailer:200000:8\ncontainer:200008:8", "", 999, 8).is_ok()); + } + + #[test] + fn missing_ambiguous_and_invalid_reservations_are_refused() { + for text in [ + "", + "nibrunner-jailer:1:8\n999:20:8", + "nibrunner-jailer:0:8", + "nibrunner-jailer:4294967290:8", + "nibrunner-jailer:200000:0", + "nibrunner-jailer:bad:8", + ] { + assert!(reserved_base(text, "", 999, 8).is_err(), "{text}"); + } + } + + #[test] + fn the_reservation_account_is_non_root_locked_and_cannot_log_in() { + let passwd = "nibrunner-jailer:x:999:999::/nonexistent:/usr/sbin/nologin\n"; + let shadow = "nibrunner-jailer:!:20000::::::\n"; + assert_eq!(account_uid(passwd, shadow).unwrap(), 999); + assert!(account_uid(&passwd.replace(":999:", ":0:"), shadow).is_err()); + assert!(account_uid(&passwd.replace("/usr/sbin/nologin", "/bin/sh"), shadow).is_err()); + assert!(account_uid(passwd, &shadow.replace(":!:", ":password:")).is_err()); + } +} diff --git a/crates/nibrunnerd/src/install/mod.rs b/crates/nibrunnerd/src/install/mod.rs index 05f21d3b..dc6c780d 100644 --- a/crates/nibrunnerd/src/install/mod.rs +++ b/crates/nibrunnerd/src/install/mod.rs @@ -9,6 +9,7 @@ //! asks systemd for it. pub mod guest_image; +pub(crate) mod jailer_identities; pub mod kernel; pub mod max_apps; pub mod prerequisites; @@ -132,6 +133,14 @@ pub async fn run( })?; } + if jailer_identities::ensure(config.max_apps)? { + let identities = jailer_identities::read(config.max_apps)?; + laid.did(format!( + "jailer identity ranges reserved from UID {} and GID {}", + identities.uid_base, identities.gid_base + )); + } + if let Some(settings) = config.volumes.zerofs() { if let service_user::Made::Created = service_user::ensure(service_user::ZEROFS_USER)? { laid.did(format!("{} account created", service_user::ZEROFS_USER)); diff --git a/crates/nibrunnerd/src/install/prerequisites.rs b/crates/nibrunnerd/src/install/prerequisites.rs index 691bb206..d0986d18 100644 --- a/crates/nibrunnerd/src/install/prerequisites.rs +++ b/crates/nibrunnerd/src/install/prerequisites.rs @@ -34,6 +34,8 @@ pub fn check(config: &HostConfig) -> Vec { for tool in ["nft", "mke2fs"] { checks.push(binary(tool, "install the nftables and e2fsprogs packages")); } + checks.push(binary("useradd", "install the passwd package")); + checks.push(binary("getent", "install the libc-bin package")); // What sets this host's kernel settings, which `install` does rather than asks for. checks.push(binary("modprobe", "install the kmod package")); @@ -50,11 +52,6 @@ pub fn check(config: &HostConfig) -> Vec { "fusermount3", "install the nbd-client, e2fsprogs and fuse3 packages", ), - // ZeroFS does not run as root, and this is what creates the account it does run as. - ( - "useradd", - "install the passwd package, or create the zerofs account by hand", - ), ] { checks.push(binary(tool, remedy)); }