From 0b4767a6410913dde91508c3594042f4a5003dc2 Mon Sep 17 00:00:00 2001 From: Kasper Nielsen Date: Sat, 17 Oct 2026 23:06:46 +0200 Subject: [PATCH 01/12] feat(portmapper): add the NAT-PMP and PCP wire layer Add the ts_portmapper crate's wire layer, modelled on Go's net/portmapper: - strict NAT-PMP (RFC 6886) and PCP (RFC 6887) codecs; - default-gateway lookup from the macOS routing socket and Linux /proc/net/route, validated as private and on-link; - GatewayCap, the only code that sends to the LAN: sockets bound to the host's own address and, where the OS allows it, its interface, gateway:5351 as the only destination, and a policy checked before every send; - a one-shot probe, map and delete, and a probe tool that maps a test port and reports what the gateway supports; - behind the test-harness feature, a loopback fake NAT-PMP/PCP router with its own RFC decoder, and TestCap, a loopback transport. Signed-off-by: Kasper Nielsen Co-Authored-By: GPT-6.1 (codex) Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 15 + Cargo.toml | 2 + supply-chain/config.toml | 3 + ts_portmapper/Cargo.toml | 52 + ts_portmapper/README.md | 8 + ts_portmapper/src/blocking.rs | 109 ++ ts_portmapper/src/cap.rs | 1568 +++++++++++++++++++++++++++ ts_portmapper/src/gateway/darwin.rs | 652 +++++++++++ ts_portmapper/src/gateway/linux.rs | 436 ++++++++ ts_portmapper/src/gateway/mod.rs | 750 +++++++++++++ ts_portmapper/src/gateway/other.rs | 9 + ts_portmapper/src/lib.rs | 116 ++ ts_portmapper/src/oneshot.rs | 1124 +++++++++++++++++++ ts_portmapper/src/pcp.rs | 1005 +++++++++++++++++ ts_portmapper/src/pmp.rs | 610 +++++++++++ ts_portmapper/src/probe_tool.rs | 1198 ++++++++++++++++++++ ts_portmapper/src/testing/cap.rs | 203 ++++ ts_portmapper/src/testing/clock.rs | 28 + ts_portmapper/src/testing/mod.rs | 48 + ts_portmapper/src/testing/router.rs | 1131 +++++++++++++++++++ ts_portmapper/src/testing/tests.rs | 637 +++++++++++ 21 files changed, 9704 insertions(+) create mode 100644 ts_portmapper/Cargo.toml create mode 100644 ts_portmapper/README.md create mode 100644 ts_portmapper/src/blocking.rs create mode 100644 ts_portmapper/src/cap.rs create mode 100644 ts_portmapper/src/gateway/darwin.rs create mode 100644 ts_portmapper/src/gateway/linux.rs create mode 100644 ts_portmapper/src/gateway/mod.rs create mode 100644 ts_portmapper/src/gateway/other.rs create mode 100644 ts_portmapper/src/lib.rs create mode 100644 ts_portmapper/src/oneshot.rs create mode 100644 ts_portmapper/src/pcp.rs create mode 100644 ts_portmapper/src/pmp.rs create mode 100644 ts_portmapper/src/probe_tool.rs create mode 100644 ts_portmapper/src/testing/cap.rs create mode 100644 ts_portmapper/src/testing/clock.rs create mode 100644 ts_portmapper/src/testing/mod.rs create mode 100644 ts_portmapper/src/testing/router.rs create mode 100644 ts_portmapper/src/testing/tests.rs diff --git a/Cargo.lock b/Cargo.lock index 3eb3a71c..638d3657 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2343,6 +2343,21 @@ dependencies = [ "url", ] +[[package]] +name = "geiserx_ts_portmapper" +version = "0.57.8" +dependencies = [ + "getrandom 0.4.3", + "if-addrs", + "libc", + "proptest", + "socket2", + "thiserror 2.0.21", + "tokio", + "tracing", + "tracing-test", +] + [[package]] name = "geiserx_ts_python" version = "0.57.8" diff --git a/Cargo.toml b/Cargo.toml index 1c5af4aa..6ae2030c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,6 +38,7 @@ members = [ "ts_packetfilter_serde", "ts_packetfilter_state", "ts_peercapability", + "ts_portmapper", "ts_python", "ts_runtime", "ts_test_util", @@ -185,6 +186,7 @@ ts_packetfilter_serde = { path = "ts_packetfilter_serde", package = "geiserx_ts_ ts_packetfilter_state = { path = "ts_packetfilter_state", package = "geiserx_ts_packetfilter_state", version = "0.57.8" } # x-release-please-version ts_peercapability = { path = "ts_peercapability", package = "geiserx_ts_peercapability", version = "0.57.8" } # x-release-please-version ts_http_util = { path = "ts_http_util", package = "geiserx_ts_http_util", version = "0.57.8" } # x-release-please-version +ts_portmapper = { path = "ts_portmapper", package = "geiserx_ts_portmapper", version = "0.57.8" } # x-release-please-version ts_tka = { path = "ts_tka", package = "geiserx_ts_tka", version = "0.57.8" } # x-release-please-version ts_tls_util = { path = "ts_tls_util", package = "geiserx_ts_tls_util", version = "0.57.8" } # x-release-please-version ts_runtime = { path = "ts_runtime", package = "geiserx_ts_runtime", version = "0.57.8" } # x-release-please-version diff --git a/supply-chain/config.toml b/supply-chain/config.toml index 337995c3..77617f5b 100644 --- a/supply-chain/config.toml +++ b/supply-chain/config.toml @@ -115,6 +115,9 @@ audit-as-crates-io = false [policy.geiserx_ts_peercapability] audit-as-crates-io = false +[policy.geiserx_ts_portmapper] +audit-as-crates-io = false + [policy.geiserx_ts_python] audit-as-crates-io = false diff --git a/ts_portmapper/Cargo.toml b/ts_portmapper/Cargo.toml new file mode 100644 index 00000000..31beff65 --- /dev/null +++ b/ts_portmapper/Cargo.toml @@ -0,0 +1,52 @@ +[package] +name = "geiserx_ts_portmapper" +version.workspace = true +description = "tailscale NAT port mapper (NAT-PMP and PCP)" +categories = ["network-programming"] +keywords = ["tailscale", "nat-pmp", "pcp", "portmapper"] + +repository.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lib] +name = "ts_portmapper" + +[dependencies] +# Every dependency below is already in the workspace lock, so this crate adds no new crate to the +# graph. Deliberately no HTTP, TLS or route crate: the `portmapper_lan_only` check enforces that. +thiserror.workspace = true +# `net` for the UDP exchange sockets, `time` for the absolute exchange deadline, `macros` for the +# probe's `select!`, `rt` for the runtime check before registering a socket; `sync` is used by the +# fake-router harness. All are already in the graph. +tokio = { workspace = true, features = ["net", "time", "sync", "rt", "macros"] } +# Interface binding (`IP_BOUND_IF` on macOS, `SO_BINDTODEVICE` on Linux) for the LAN sockets. The +# `all` feature is what exposes the binding calls; `ts_magicsock` already enables it. +socket2 = { version = "0.6", features = ["all"] } +# Interface address enumeration for the gateway's on-link check (already used by `ts_host_net`, +# `ts_forwarder` and `ts_magicsock`). +if-addrs = "0.15" +# PCP mapping nonces come from the OS CSPRNG (RFC 6887 §11.1). Already pulled in by `ts_keys`. +getrandom = "0.4" +# Lifecycle logging (obtained, renewed, lost, released). Already in the workspace graph. +tracing.workspace = true + +[target.'cfg(unix)'.dependencies] +# macOS: `sysctl(NET_RT_DUMP2)` for the routing table; Linux: `if_nametoindex`. Both: uid/euid for +# the probe's diagnostics line. +libc = "0.2" + +[dev-dependencies] +proptest.workspace = true +tracing-test = { workspace = true, features = ["no-env-filter"] } +tokio = { workspace = true, features = ["test-util", "macros", "rt", "rt-multi-thread"] } + +[features] +# Exposes the fake NAT-PMP/PCP router and `TestCap`, the loopback test transport, to other crates' +# tests. Never enable it from a normal (non-dev) dependency: the `portmapper_lan_only` check fails +# if any crate does. +test-harness = [] + +[lints] +workspace = true diff --git a/ts_portmapper/README.md b/ts_portmapper/README.md new file mode 100644 index 00000000..7ba9f19f --- /dev/null +++ b/ts_portmapper/README.md @@ -0,0 +1,8 @@ +# ts_portmapper + +Ask the LAN gateway for an inbound UDP mapping with NAT-PMP (RFC 6886) or PCP (RFC 6887), so peers +behind hard NATs can reach this node directly. Modelled on Go Tailscale's `net/portmapper`. + +Every packet goes through `GatewayCap`: the only destination is a validated private, on-link +gateway's port 5351, sockets are bound to the host's own address and, where the OS allows it, its +interface, and a policy is checked before each send. UPnP-IGD is not implemented. diff --git a/ts_portmapper/src/blocking.rs b/ts_portmapper/src/blocking.rs new file mode 100644 index 00000000..95575176 --- /dev/null +++ b/ts_portmapper/src/blocking.rs @@ -0,0 +1,109 @@ +//! Bounded OS work, independent of Tokio's worker and runtime-shutdown threads. +//! +//! A cancelled waiter leaves its permit with the OS operation until it ends. Thus repeated +//! cancellation cannot grow the number of blocked threads. Threads are detached so a stuck +//! syscall cannot hold up dropping a Tokio runtime; results still require a generation check. +//! Waiters give up after one second, including time waiting for a worker slot. + +use tokio::sync::{Semaphore, oneshot}; + +static SLOTS: Semaphore = Semaphore::const_new(4); + +pub(crate) async fn run(f: impl FnOnce() -> R + Send + 'static) -> Option { + run_on(&SLOTS, f).await +} + +async fn run_on( + slots: &'static Semaphore, + f: impl FnOnce() -> R + Send + 'static, +) -> Option { + tokio::time::timeout(std::time::Duration::from_secs(1), async move { + let permit = slots.acquire().await.ok()?; + let (tx, rx) = oneshot::channel(); + std::thread::Builder::new() + .name("portmapper-os".to_owned()) + .spawn(move || { + let _permit = permit; + drop(tx.send(f())); + }) + .ok()?; + rx.await.ok() + }) + .await + .ok() + .flatten() +} + +#[cfg(test)] +mod tests { + #[tokio::test] + async fn os_wait_is_bounded_when_worker_keeps_running() { + let result = super::run(|| { + std::thread::sleep(std::time::Duration::from_millis(1500)); + 42 + }) + .await; + assert_eq!( + result, None, + "a slow OS call must not hold its async waiter indefinitely" + ); + } +} + +#[cfg(test)] +mod worker_tests { + static TEST_SLOTS: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(4); + + #[tokio::test] + async fn cancelled_workers_keep_slots_and_recover() { + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::{Arc, Condvar, Mutex}; + let gate = Arc::new((Mutex::new(false), Condvar::new())); + let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel(); + let finished = Arc::new(AtomicUsize::new(0)); + let mut jobs = Vec::new(); + for _ in 0..4 { + let gate = gate.clone(); + let tx = tx.clone(); + let finished = finished.clone(); + jobs.push(tokio::spawn(super::run_on(&TEST_SLOTS, move || { + tx.send(()).unwrap(); + let ready = gate.0.lock().unwrap(); + drop(gate.1.wait_while(ready, |open| !*open).unwrap()); + finished.fetch_add(1, Ordering::SeqCst); + }))); + } + for _ in 0..4 { + tokio::time::timeout(std::time::Duration::from_secs(2), rx.recv()) + .await + .unwrap() + .unwrap(); + } + for job in &jobs { + job.abort(); + } + for job in jobs { + drop(job.await); + } + let fifth_started = Arc::new(AtomicUsize::new(0)); + let count = fifth_started.clone(); + let blocked = + super::run_on(&TEST_SLOTS, move || count.fetch_add(1, Ordering::SeqCst)).await; + *gate.0.lock().unwrap() = true; + gate.1.notify_all(); + assert!(blocked.is_none()); + assert_eq!( + fifth_started.load(Ordering::SeqCst), + 0, + "aborted waiters must not release running OS slots" + ); + while finished.load(Ordering::SeqCst) != 4 { + tokio::task::yield_now().await; + } + assert_eq!( + super::run_on(&TEST_SLOTS, || 42).await, + Some(42), + "completion must return slots without a lost wakeup" + ); + } +} diff --git a/ts_portmapper/src/cap.rs b/ts_portmapper/src/cap.rs new file mode 100644 index 00000000..4d4f262c --- /dev/null +++ b/ts_portmapper/src/cap.rs @@ -0,0 +1,1568 @@ +//! [`GatewayCap`]: the validated gateway capability, and the only module in this crate that +//! creates sockets or sends. +//! +//! A `GatewayCap` is built once per network generation from a [`Gateway`] and re-checks it: +//! private gateway and self address, an on-link prefix of length 8 to 31, and a usable interface +//! (up, not loopback, not point-to-point) that holds `self_ip/prefix`. Every exchange uses a fresh +//! UDP socket with `SO_DONTROUTE` (the kernel refuses a destination that is not directly +//! connected), bound to `self_ip:0` and to the interface (`IP_BOUND_IF` on macOS, +//! `SO_BINDTODEVICE` on Linux), sends only to `gateway:5351` ([`LanDest::Pxp`]; no API takes an +//! address), checks the [`PolicyHandle`] and the cap's own validity before **every** send, and +//! accepts only datagrams whose source is exactly `gateway:5351`. +//! +//! Without interface binding ([`BindMode::SourceIp`]: Linux without permission for +//! `SO_BINDTODEVICE`, other platforms) the egress interface is whatever the kernel's full policy +//! routing picks, which can be a tunnel (a VPN, or a Tailscale subnet route covering the LAN). So +//! that mode needs a gateway lookup, and the cap requires, at build time and before every send, +//! that the lookup still reports the same gateway **and** that the kernel's own source selection +//! for the gateway is `self_ip`. +//! +//! Residual risk, by design: NAT-PMP is unauthenticated, so an on-LAN spoofer can fake service +//! availability, force failures and churn, or report a false external address. PCP nonces stop +//! off-path spoofing only. Mitigations here: bounded parse work (at most [`MAX_EXAMINED`] +//! datagrams per exchange), an absolute deadline that junk cannot extend, and a request target +//! that is always our own `self_ip` and port, never chosen by a reply. + +use std::{ + fmt, + future::Future, + io, + net::{SocketAddr, SocketAddrV4}, + sync::{ + Arc, Mutex, MutexGuard, + atomic::{AtomicBool, Ordering}, + }, + time::Duration, +}; + +use socket2::{Domain, Protocol, Socket, Type}; +use tokio::time::Instant; + +use crate::{ + PXP_PORT, + gateway::{ + Gateway, GatewayLookup, IfInfo, LAN_PREFIX_LENS, interfaces, is_private, kernel_source, + prefix_contains, + }, +}; + +/// The kernel's preferred source address for a destination (see [`kernel_source`]). +type KernelSource = Arc Option + Send + Sync>; + +/// Go's per-exchange budget (`portMapServiceTimeout`): an absolute deadline, never extended. +pub const EXCHANGE_TIMEOUT: Duration = Duration::from_millis(250); +/// Most datagrams examined in one exchange, whatever their source or content. +pub const MAX_EXAMINED: usize = 64; +/// Receive buffer: the PCP maximum of 1100 bytes plus one, so a datagram that fills it is known +/// to be oversize (truncated) and is dropped. +pub const RECV_BUF: usize = crate::pcp::MAX_LEN + 1; + +/// Where a LAN packet may go. Phases 0–1 allow exactly one destination. +#[non_exhaustive] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LanDest { + /// The gateway's NAT-PMP/PCP port, `gateway:5351`. + Pxp, +} + +#[derive(Debug)] +struct PolicyInner { + /// Serializes final gate checks + one nonblocking send against disable/invalidation. + /// Never held across readiness, route lookup, socket construction or any await. + send: Mutex<()>, + lan_sends: AtomicBool, +} + +/// Whether LAN sends are allowed right now. Shared between the owner (which flips it, for example +/// on `only-tcp-443`) and every cap, which checks it before each send. +#[derive(Debug, Clone)] +pub struct PolicyHandle(Arc); + +impl PolicyHandle { + /// A policy that starts with LAN sends allowed or not. + pub fn new(lan_sends_allowed: bool) -> Self { + Self(Arc::new(PolicyInner { + send: Mutex::new(()), + lan_sends: AtomicBool::new(lan_sends_allowed), + })) + } + + /// Whether LAN sends are allowed. + pub fn lan_sends_allowed(&self) -> bool { + self.0.lan_sends.load(Ordering::SeqCst) + } + + /// Allow or forbid LAN sends; takes effect at the next send of every cap sharing this handle. + pub fn set_lan_sends_allowed(&self, allowed: bool) { + let _permit = self.send_permit(); + self.0.lan_sends.store(allowed, Ordering::SeqCst); + } + + fn send_permit(&self) -> MutexGuard<'_, ()> { + self.0 + .send + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } +} + +/// A network generation: bumped whenever the gateway identity changes. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)] +pub struct NetGen(pub u64); + +/// How a cap's sockets are tied to the gateway's interface. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BindMode { + /// macOS `IP_BOUND_IF`, plus the source address. + IfScope, + /// Linux `SO_BINDTODEVICE`, plus the source address. + Device, + /// Source address only (Linux without permission for `SO_BINDTODEVICE`, other platforms), + /// with the gateway revalidated before each send when a lookup is available. + SourceIp, + /// The loopback test transport. + LoopbackTest, +} + +impl fmt::Display for BindMode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::IfScope => "ifscope", + Self::Device => "device", + Self::SourceIp => "source-ip", + Self::LoopbackTest => "loopback-test", + }) + } +} + +/// Why a send was refused before touching a socket. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DenyReason { + /// The policy forbids LAN sends. + Policy, + /// The cap was invalidated (network change, close). + Invalidated, + /// The owner is closing. + Closing, + /// The gateway no longer revalidates (source-IP bind mode). + NetworkChanged, +} + +impl fmt::Display for DenyReason { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::Policy => "policy forbids LAN sends", + Self::Invalidated => "capability invalidated", + Self::Closing => "closing", + Self::NetworkChanged => "gateway no longer validates", + }) + } +} + +/// A capability error. I/O errors are kept verbatim (errno name, number and message), so that, +/// for example, macOS Local Network privacy (`EHOSTUNREACH`/`EPERM`) is diagnosable. +#[derive(Debug, thiserror::Error)] +pub enum CapError { + /// The send was refused before any socket was touched. + #[error("LAN send denied: {0}")] + Denied(DenyReason), + /// The gateway failed validation. + #[error("gateway rejected: {0}")] + InvalidGateway(&'static str), + /// A refused probe-listener port. + #[error("port {0} is reserved for the node; pick another test port")] + ReservedPort(u16), + /// No tokio runtime to register the socket with. + #[error("no tokio runtime")] + NoRuntime, + /// An OS error, verbatim. + #[error("{op}: {}{detail}", errno_label(*.raw_os_error))] + Io { + /// What was being done: `socket`, `bind`, `bind-interface`, `send`, `recv`. + op: &'static str, + /// The error kind. + kind: io::ErrorKind, + /// The raw OS error number, if any. + raw_os_error: Option, + /// The OS error message. + detail: String, + }, +} + +impl CapError { + /// Wrap an I/O error from `op`, keeping it verbatim. + pub fn io(op: &'static str, e: &io::Error) -> Self { + Self::Io { + op, + kind: e.kind(), + raw_os_error: e.raw_os_error(), + detail: e.to_string(), + } + } +} + +/// `"EHOSTUNREACH: "` for well-known errno values, else empty. +fn errno_label(raw: Option) -> String { + raw.and_then(errno_name) + .map(|n| format!("{n}: ")) + .unwrap_or_default() +} + +/// Symbolic names for the errno values a LAN send can plausibly hit. +pub fn errno_name(code: i32) -> Option<&'static str> { + #[cfg(unix)] + { + let names = [ + (libc::EPERM, "EPERM"), + (libc::EACCES, "EACCES"), + (libc::EHOSTUNREACH, "EHOSTUNREACH"), + (libc::ENETUNREACH, "ENETUNREACH"), + (libc::EHOSTDOWN, "EHOSTDOWN"), + (libc::ENETDOWN, "ENETDOWN"), + (libc::EADDRNOTAVAIL, "EADDRNOTAVAIL"), + (libc::EADDRINUSE, "EADDRINUSE"), + (libc::ECONNREFUSED, "ECONNREFUSED"), + (libc::ENOBUFS, "ENOBUFS"), + (libc::ENXIO, "ENXIO"), + (libc::ENODEV, "ENODEV"), + ]; + names.iter().find(|(c, _)| *c == code).map(|(_, n)| *n) + } + #[cfg(not(unix))] + { + let _ = code; + None + } +} + +/// Tells an exchange whether to keep receiving. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Flow { + /// Keep receiving until the deadline. + Continue, + /// Stop: the reply (or replies) this exchange waited for arrived. + Done, +} + +/// What happened during one exchange. +#[derive(Debug)] +pub struct ExchangeStats { + /// Datagrams received and looked at (all sources). + pub examined: usize, + /// Datagrams dropped because their source was not exactly `gateway:5351`. + pub wrong_source: usize, + /// Datagrams dropped because they filled the receive buffer (truncated). + pub oversize: usize, + /// The reply handler returned [`Flow::Done`]. + pub done: bool, + /// The exchange stopped at [`MAX_EXAMINED`]. + pub hit_cap: bool, + /// The absolute deadline the exchange ran against. + pub deadline: Instant, + /// A receive error that ended the exchange, verbatim. + pub recv_error: Option, +} + +/// The checks every send passes first. +pub(crate) struct SendGate { + policy: PolicyHandle, + valid: AtomicBool, + closing: AtomicBool, + revalidate: Option bool + Send + Sync>>, +} + +impl SendGate { + pub(crate) fn new( + policy: PolicyHandle, + revalidate: Option bool + Send + Sync>>, + ) -> Self { + Self { + policy, + valid: AtomicBool::new(true), + closing: AtomicBool::new(false), + revalidate, + } + } + + pub(crate) fn check(&self) -> Result<(), CapError> { + if !self.policy.lan_sends_allowed() { + return Err(CapError::Denied(DenyReason::Policy)); + } + if !self.valid.load(Ordering::SeqCst) { + return Err(CapError::Denied(DenyReason::Invalidated)); + } + if self.closing.load(Ordering::SeqCst) { + return Err(CapError::Denied(DenyReason::Closing)); + } + Ok(()) + } + + /// OS route checks run off the async worker. Cancellation leaves bounded OS work behind, + /// and the atomic policy/generation gate is rechecked after the result arrives. + pub(crate) async fn check_async(&self, deadline: Instant) -> Result<(), CapError> { + self.check()?; + if let Some(revalidate) = self.revalidate.clone() { + let valid = + tokio::time::timeout_at(deadline, crate::blocking::run(move || revalidate())) + .await + .map_err(|_| CapError::InvalidGateway("route validation timed out"))?; + if valid != Some(true) { + self.invalidate(); + return Err(CapError::Denied(DenyReason::NetworkChanged)); + } + } + self.check() + } + + pub(crate) fn invalidate(&self) { + let _permit = self.policy.send_permit(); + self.valid.store(false, Ordering::SeqCst); + } + + pub(crate) fn set_closing(&self) { + let _permit = self.policy.send_permit(); + self.closing.store(true, Ordering::SeqCst); + } + + /// The only admitted send. Callers supply a nonblocking syscall; no await or socket/route + /// setup is allowed under this permit. Invalidation/disable cannot return during the + /// final check-to-syscall window. + pub(crate) fn send(&self, send: impl FnOnce() -> io::Result) -> Result { + #[cfg(test)] + send_tests::before_send(); + let _permit = self.policy.send_permit(); + self.check()?; + send().map_err(|e| CapError::io("send", &e)) + } + + pub(crate) fn is_valid(&self) -> bool { + self.valid.load(Ordering::SeqCst) + } +} + +/// Send `reqs` to `to` (each send gated), then receive until `on_reply` says [`Flow::Done`], the +/// absolute `deadline` passes, or [`MAX_EXAMINED`] datagrams were seen. Only datagrams whose +/// source, as presented by `present`, is exactly `expected` reach `on_reply`. +/// +/// Shared by [`GatewayCap`] and the test transport, so the acceptance logic under test is the +/// production logic. +pub(crate) async fn exchange_on( + sock: &tokio::net::UdpSocket, + gate: &SendGate, + to: SocketAddr, + expected: SocketAddrV4, + present: &(dyn Fn(SocketAddr) -> Option + Sync), + reqs: &[&[u8]], + deadline: Instant, + on_reply: &mut (dyn FnMut(&[u8]) -> Flow + Send), +) -> Result { + for req in reqs { + loop { + gate.check()?; + tokio::time::timeout_at(deadline, sock.writable()) + .await + .map_err(|_| CapError::io("send-ready", &io::Error::from(io::ErrorKind::TimedOut)))? + .map_err(|e| CapError::io("send-ready", &e))?; + // Source-IP revalidation follows readiness. Neither wait owns the send permit. + gate.check_async(deadline).await?; + let sent = gate.send(|| { + if Instant::now() >= deadline { + return Err(io::ErrorKind::TimedOut.into()); + } + sock.try_send_to(req, to) + }); + match sent { + Ok(_) => break, + Err(CapError::Io { + kind: io::ErrorKind::WouldBlock, + .. + }) => continue, + Err(e) => return Err(e), + } + } + } + let mut stats = ExchangeStats { + examined: 0, + wrong_source: 0, + oversize: 0, + done: false, + hit_cap: false, + deadline, + recv_error: None, + }; + let mut buf = [0u8; RECV_BUF]; + loop { + if stats.examined >= MAX_EXAMINED { + stats.hit_cap = true; + break; + } + let (n, src) = match tokio::time::timeout_at(deadline, sock.recv_from(&mut buf)).await { + Err(_elapsed) => break, + Ok(Err(e)) => { + stats.recv_error = Some(CapError::io("recv", &e)); + break; + } + Ok(Ok(got)) => got, + }; + stats.examined += 1; + if n >= RECV_BUF { + stats.oversize += 1; + continue; + } + if present(src) != Some(expected) { + stats.wrong_source += 1; + continue; + } + if on_reply(&buf[..n]) == Flow::Done { + stats.done = true; + break; + } + } + Ok(stats) +} + +/// A receive-only UDP socket for the probe's hold window. It reports each datagram's length and +/// source; the payload is never exposed and nothing is ever sent from it. +#[derive(Debug)] +pub struct ProbeListener { + sock: tokio::net::UdpSocket, + port: u16, + buf: Vec, +} + +impl ProbeListener { + pub(crate) fn new(sock: tokio::net::UdpSocket) -> Result { + let port = sock + .local_addr() + .map_err(|e| CapError::io("bind", &e))? + .port(); + Ok(Self { + sock, + port, + buf: vec![0u8; 65536], + }) + } + + /// The bound (nonzero) port. + pub fn port(&self) -> u16 { + self.port + } + + /// Wait for one datagram; returns its length and source. + pub async fn recv(&mut self) -> Result<(usize, SocketAddr), CapError> { + self.sock + .recv_from(&mut self.buf) + .await + .map_err(|e| CapError::io("recv", &e)) + } +} + +pub(crate) mod sealed { + /// Only this crate's transports implement [`super::PxpTransport`]. + pub trait Sealed {} +} + +/// The transport the one-shot client and probe run on: [`GatewayCap`] in production, `TestCap` +/// (feature `test-harness`) in tests. Sealed: no other implementation can exist. +pub trait PxpTransport: sealed::Sealed + Send + Sync { + /// The gateway this transport talks to. + fn gateway(&self) -> &Gateway; + + /// How sockets are bound. + fn bind_mode(&self) -> BindMode; + + /// Send `reqs` to `dest` from one fresh socket, then receive replies (see [`Flow`]) until the + /// absolute `deadline`. + fn exchange( + &self, + dest: LanDest, + reqs: &[&[u8]], + deadline: Instant, + on_reply: &mut (dyn FnMut(&[u8]) -> Flow + Send), + ) -> impl Future> + Send; + + /// One unacknowledged, non-blocking send (best-effort release from `Drop`; no runtime + /// needed). + fn send_nowait(&self, dest: LanDest, req: &[u8]) -> Result<(), CapError>; + + /// Bind the probe's dedicated receive-only test socket on `self_ip:port` (0 = ephemeral). + /// Must be called within a tokio runtime. + fn bind_probe_listener(&self, port: u16) -> Result; + + /// Refuse every later send. + fn invalidate(&self); + + /// Whether sends are still possible (not invalidated). + fn is_valid(&self) -> bool; +} + +/// The validated gateway capability. See the module docs. +pub struct GatewayCap { + gateway: Gateway, + generation: NetGen, + bind_mode: BindMode, + gate: SendGate, + /// Always [`PXP_PORT`] in production; a test may point the real cap at a loopback router. + pxp_port: u16, +} + +impl fmt::Debug for GatewayCap { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("GatewayCap") + .field("gateway", &self.gateway) + .field("generation", &self.generation) + .field("bind_mode", &self.bind_mode) + .field("valid", &self.gate.is_valid()) + .finish_non_exhaustive() + } +} + +/// Re-check the properties a [`Gateway`] promises, whatever produced it. +fn check_gateway(g: &Gateway) -> Result<(), CapError> { + if !is_private(g.gw) { + return Err(CapError::InvalidGateway( + "gateway is not private (RFC 1918)", + )); + } + if !is_private(g.self_ip) { + return Err(CapError::InvalidGateway( + "self address is not private (RFC 1918)", + )); + } + if !LAN_PREFIX_LENS.contains(&g.prefix_len) { + return Err(CapError::InvalidGateway( + "self prefix length is not 8 to 31", + )); + } + if g.gw == g.self_ip || !prefix_contains(g.self_ip, g.prefix_len, g.gw) { + return Err(CapError::InvalidGateway("gateway is not on-link")); + } + if g.if_index == 0 || g.if_name.is_empty() { + return Err(CapError::InvalidGateway("no interface")); + } + Ok(()) +} + +/// Re-check the gateway's interface against the host's interface list: it exists under the same +/// index and name, is usable, and holds `self_ip/prefix_len`. +fn check_interface(g: &Gateway, ifs: &[IfInfo]) -> Result<(), CapError> { + let iface = ifs + .iter() + .find(|i| i.index == g.if_index && i.name == g.if_name) + .ok_or(CapError::InvalidGateway("interface not found"))?; + if !iface.usable() { + return Err(CapError::InvalidGateway( + "interface is down, loopback or point-to-point", + )); + } + if !iface.v4.contains(&(g.self_ip, g.prefix_len)) { + return Err(CapError::InvalidGateway( + "self address is not on the interface", + )); + } + Ok(()) +} + +fn new_socket() -> Result { + let s = Socket::new(Domain::IPV4, Type::DGRAM, Some(Protocol::UDP)) + .map_err(|e| CapError::io("socket", &e))?; + s.set_nonblocking(true) + .map_err(|e| CapError::io("socket", &e))?; + set_dontroute(&s)?; + Ok(s) +} + +/// `SO_DONTROUTE`: send only to directly connected destinations, so an off-link "gateway" fails +/// with `ENETUNREACH` instead of following a route. It makes "on-link" a kernel-enforced property. +#[cfg(unix)] +fn set_dontroute(s: &Socket) -> Result<(), CapError> { + use std::os::fd::AsRawFd; + let on: libc::c_int = 1; + // SAFETY: `s` is an open socket for the duration of the call, and `on` is a live `c_int` + // whose size is passed as the option length. + let r = unsafe { + libc::setsockopt( + s.as_raw_fd(), + libc::SOL_SOCKET, + libc::SO_DONTROUTE, + std::ptr::from_ref(&on).cast(), + libc::socklen_t::try_from(std::mem::size_of::()).unwrap_or(4), + ) + }; + if r == 0 { + Ok(()) + } else { + Err(CapError::io("socket", &io::Error::last_os_error())) + } +} + +#[cfg(not(unix))] +fn set_dontroute(_: &Socket) -> Result<(), CapError> { + Ok(()) +} + +/// Whether `SO_DONTROUTE` is set on `s`. +#[cfg(all(test, unix))] +fn dontroute_is_set(s: &Socket) -> bool { + use std::os::fd::AsRawFd; + let mut v: libc::c_int = 0; + let mut len = libc::socklen_t::try_from(std::mem::size_of::()).unwrap_or(4); + // SAFETY: `s` is open; `v` and `len` are live and `len` holds `v`'s size. + let r = unsafe { + libc::getsockopt( + s.as_raw_fd(), + libc::SOL_SOCKET, + libc::SO_DONTROUTE, + std::ptr::from_mut(&mut v).cast(), + &raw mut len, + ) + }; + r == 0 && v != 0 +} + +/// Decide the bind mode with a trial socket (bound to nothing, sends nothing). +fn probe_bind_mode(g: &Gateway) -> Result { + let s = new_socket()?; + #[cfg(target_os = "macos")] + { + s.bind_device_by_index_v4(std::num::NonZeroU32::new(g.if_index)) + .map_err(|e| CapError::io("bind-interface", &e))?; + Ok(BindMode::IfScope) + } + #[cfg(target_os = "linux")] + { + match s.bind_device(Some(g.if_name.as_bytes())) { + Ok(()) => Ok(BindMode::Device), + Err(e) if e.raw_os_error() == Some(libc::EPERM) => Ok(BindMode::SourceIp), + Err(e) => Err(CapError::io("bind-interface", &e)), + } + } + #[cfg(not(any(target_os = "macos", target_os = "linux")))] + { + drop((s, g)); + Ok(BindMode::SourceIp) + } +} + +/// Tie `s` to the gateway's interface per `mode`. +fn apply_bind_mode(s: &Socket, g: &Gateway, mode: BindMode) -> Result<(), CapError> { + match mode { + #[cfg(target_os = "macos")] + BindMode::IfScope => s + .bind_device_by_index_v4(std::num::NonZeroU32::new(g.if_index)) + .map_err(|e| CapError::io("bind-interface", &e)), + #[cfg(target_os = "linux")] + BindMode::Device => s + .bind_device(Some(g.if_name.as_bytes())) + .map_err(|e| CapError::io("bind-interface", &e)), + BindMode::SourceIp => Ok(()), + _ => { + let _ = (s, g); + Err(CapError::InvalidGateway( + "bind mode unavailable on this platform", + )) + } + } +} + +impl GatewayCap { + /// Validate `gateway` and build the capability for network generation `generation`. + /// + /// In [`BindMode::SourceIp`] (the socket is not interface-bound) `lookup` is required: the + /// cap is refused without one, and before each send the lookup must still report the same + /// gateway and the kernel must still pick `self_ip` as the source for the gateway. + pub fn validate( + gateway: Gateway, + generation: NetGen, + policy: PolicyHandle, + lookup: Option>, + ) -> Result, CapError> { + check_gateway(&gateway)?; + let ifs = + interfaces().map_err(|_| CapError::InvalidGateway("interface list unavailable"))?; + let bind_mode = probe_bind_mode(&gateway)?; + Self::validate_with( + gateway, + generation, + policy, + lookup, + bind_mode, + &ifs, + Arc::new(kernel_source), + ) + } + + /// [`GatewayCap::validate`] with the bind mode, interface list and kernel source selection + /// supplied (tests inject them; no socket is created here). + fn validate_with( + gateway: Gateway, + generation: NetGen, + policy: PolicyHandle, + lookup: Option>, + bind_mode: BindMode, + ifs: &[IfInfo], + kernel_source: KernelSource, + ) -> Result, CapError> { + check_gateway(&gateway)?; + check_interface(&gateway, ifs)?; + let revalidate: Option bool + Send + Sync>> = match bind_mode { + BindMode::SourceIp => { + let Some(lookup) = lookup else { + return Err(CapError::InvalidGateway( + "source-ip bind mode needs a gateway lookup", + )); + }; + let (gw, me) = (gateway.gw, gateway.self_ip); + if kernel_source(gw) != Some(me) { + return Err(CapError::InvalidGateway( + "the kernel routes to the gateway from another address", + )); + } + let expect = gateway.clone(); + Some(Arc::new(move || { + matches!(lookup.lookup(), Ok(Some(g)) if g.same_network(&expect)) + && kernel_source(gw) == Some(me) + })) + } + BindMode::IfScope | BindMode::Device | BindMode::LoopbackTest => None, + }; + Ok(Arc::new(Self { + gateway, + generation, + bind_mode, + gate: SendGate::new(policy, revalidate), + pxp_port: PXP_PORT, + })) + } + + /// The network generation this cap belongs to. + pub fn generation(&self) -> NetGen { + self.generation + } + + /// Whether `lookup` still reports the same gateway, self address and interface. + pub fn revalidate(&self, lookup: &dyn GatewayLookup) -> bool { + matches!(lookup.lookup(), Ok(Some(g)) if g.same_network(&self.gateway)) + } + + /// Refuse new work: later sends fail with [`DenyReason::Closing`]. + pub fn set_closing(&self) { + self.gate.set_closing(); + } + + fn dest(&self, dest: LanDest) -> SocketAddr { + match dest { + LanDest::Pxp => SocketAddr::from((self.gateway.gw, self.pxp_port)), + } + } + + /// A fresh socket on `self_ip:port`, interface-bound per the bind mode. Fails closed if the + /// address or interface is gone. + fn socket(&self, port: u16) -> Result { + let s = new_socket()?; + apply_bind_mode(&s, &self.gateway, self.bind_mode)?; + s.bind(&SocketAddrV4::new(self.gateway.self_ip, port).into()) + .map_err(|e| CapError::io("bind", &e))?; + Ok(s) + } + + fn tokio_socket(&self, port: u16) -> Result { + if tokio::runtime::Handle::try_current().is_err() { + return Err(CapError::NoRuntime); + } + let std_sock: std::net::UdpSocket = self.socket(port)?.into(); + tokio::net::UdpSocket::from_std(std_sock).map_err(|e| CapError::io("socket", &e)) + } +} + +impl sealed::Sealed for GatewayCap {} + +impl PxpTransport for GatewayCap { + fn gateway(&self) -> &Gateway { + &self.gateway + } + + fn bind_mode(&self) -> BindMode { + self.bind_mode + } + + async fn exchange( + &self, + dest: LanDest, + reqs: &[&[u8]], + deadline: Instant, + on_reply: &mut (dyn FnMut(&[u8]) -> Flow + Send), + ) -> Result { + // Refuse before creating a socket at all. + self.gate.check()?; + let sock = self.tokio_socket(0)?; + let to = self.dest(dest); + let expected = SocketAddrV4::new(self.gateway.gw, self.pxp_port); + let present = |src: SocketAddr| match src { + SocketAddr::V4(a) => Some(a), + SocketAddr::V6(a) => a + .ip() + .to_ipv4_mapped() + .map(|ip| SocketAddrV4::new(ip, a.port())), + }; + exchange_on( + &sock, &self.gate, to, expected, &present, reqs, deadline, on_reply, + ) + .await + } + + fn send_nowait(&self, dest: LanDest, req: &[u8]) -> Result<(), CapError> { + self.gate.check()?; + #[cfg(test)] + send_tests::pause_nowait_after_check(); + // Without interface binding a safe send needs an asynchronous route recheck. The + // no-runtime release path fails closed rather than running OS discovery here. + if self.bind_mode == BindMode::SourceIp { + return Err(CapError::Denied(DenyReason::NetworkChanged)); + } + let s = self.socket(0)?; + self.gate + .send(|| s.send_to(req, &self.dest(dest).into())) + .map(drop) + } + + fn bind_probe_listener(&self, port: u16) -> Result { + ProbeListener::new(self.tokio_socket(port)?) + } + + fn invalidate(&self) { + self.gate.invalidate(); + } + + fn is_valid(&self) -> bool { + self.gate.is_valid() + } +} + +/// The loopback interface's index, for tests that run the real cap against loopback. +#[cfg(test)] +pub(crate) fn loopback_gateway() -> Option { + use std::net::Ipv4Addr; + let ifs = crate::gateway::interfaces().ok()?; + let lo = ifs + .into_iter() + .find(|i| i.loopback && i.v4.iter().any(|(a, _)| *a == Ipv4Addr::LOCALHOST))?; + Some(Gateway { + gw: Ipv4Addr::LOCALHOST, + self_ip: Ipv4Addr::LOCALHOST, + prefix_len: 8, + if_index: lo.index, + if_name: lo.name, + }) +} + +#[cfg(test)] +impl GatewayCap { + /// The real cap, skipping the private-address checks, pointed at a loopback "router" on + /// `127.0.0.1:pxp_port`, so the production socket, bind and gate code runs without any + /// packet leaving the host. + pub(crate) fn loopback_for_test(pxp_port: u16, policy: PolicyHandle) -> Option> { + let gateway = loopback_gateway()?; + let bind_mode = probe_bind_mode(&gateway).ok()?; + Some(Arc::new(Self { + gateway, + generation: NetGen(1), + bind_mode, + gate: SendGate::new(policy, None), + pxp_port, + })) + } +} + +#[cfg(test)] +mod tests { + use std::net::Ipv4Addr; + + use super::*; + + fn gw(gw: &str, me: &str, prefix: u8) -> Gateway { + Gateway { + gw: gw.parse().unwrap(), + self_ip: me.parse().unwrap(), + prefix_len: prefix, + if_index: 4, + if_name: "en0".to_owned(), + } + } + + /// The cap re-validates whatever `Gateway` it is handed. + #[test] + fn validation_rejects_bad_gateways() { + let p = PolicyHandle::new(true); + for (g, why) in [ + (gw("203.0.113.1", "192.168.1.20", 24), "not private"), + (gw("192.168.1.1", "100.64.0.2", 8), "self not private"), + (gw("192.168.2.1", "192.168.1.20", 24), "not on-link"), + (gw("192.168.1.20", "192.168.1.20", 24), "gateway is self"), + (gw("127.0.0.1", "127.0.0.1", 8), "loopback"), + ] { + assert!( + matches!( + GatewayCap::validate(g, NetGen(1), p.clone(), None), + Err(CapError::InvalidGateway(_)) + ), + "{why}" + ); + } + let mut no_if = gw("192.168.1.1", "192.168.1.20", 24); + no_if.if_index = 0; + assert!(GatewayCap::validate(no_if, NetGen(1), p.clone(), None).is_err()); + // A /0 (or /1../7) prefix would make any private gateway "on-link"; /32 has no + // neighbours. + for (g, why) in [ + (gw("10.0.0.1", "192.168.1.20", 0), "/0"), + (gw("10.0.0.1", "10.1.2.3", 7), "/7"), + (gw("192.168.1.1", "192.168.1.20", 32), "/32"), + ] { + assert!( + matches!( + GatewayCap::validate(g, NetGen(1), p.clone(), None), + Err(CapError::InvalidGateway(_)) + ), + "{why}" + ); + } + } + + fn en0() -> IfInfo { + IfInfo { + name: "en0".to_owned(), + index: 4, + up: true, + loopback: false, + point_to_point: false, + v4: vec![("192.168.1.20".parse().unwrap(), 24)], + } + } + + struct FixedLookup(Gateway); + + impl GatewayLookup for FixedLookup { + fn lookup(&self) -> Result, crate::LookupError> { + Ok(Some(self.0.clone())) + } + } + + fn build( + mode: BindMode, + ifs: &[IfInfo], + lookup: bool, + ks: KernelSource, + ) -> Result, CapError> { + let g = gw("192.168.1.1", "192.168.1.20", 24); + let lookup: Option> = + lookup.then(|| Arc::new(FixedLookup(g.clone())) as _); + GatewayCap::validate_with(g, NetGen(1), PolicyHandle::new(true), lookup, mode, ifs, ks) + } + + fn ks_is(ip: &'static str) -> KernelSource { + Arc::new(move |_| Some(ip.parse().unwrap())) + } + + /// The interface is re-checked: present under the same index and name, usable, and holding + /// `self_ip/prefix`. + #[test] + fn validation_rechecks_the_interface() { + let ks = || ks_is("192.168.1.20"); + assert!(build(BindMode::IfScope, &[en0()], false, ks()).is_ok()); + let mut down = en0(); + down.up = false; + let mut lo = en0(); + lo.loopback = true; + let mut p2p = en0(); + p2p.point_to_point = true; + let mut other_addr = en0(); + other_addr.v4 = vec![("192.168.1.21".parse().unwrap(), 24)]; + let mut other_prefix = en0(); + other_prefix.v4 = vec![("192.168.1.20".parse().unwrap(), 16)]; + let mut renamed = en0(); + renamed.name = "en1".to_owned(); + for (i, ifs) in [ + vec![], + vec![down], + vec![lo], + vec![p2p], + vec![other_addr], + vec![other_prefix], + vec![renamed], + ] + .iter() + .enumerate() + { + assert!( + matches!( + build(BindMode::IfScope, ifs, false, ks()), + Err(CapError::InvalidGateway(_)) + ), + "case {i}" + ); + } + } + + /// Source-IP mode: no lookup, or a kernel that would send to the gateway from another + /// address (policy routing into a tunnel), is refused. + #[test] + fn source_ip_mode_needs_lookup_and_kernel_agreement() { + assert!(build(BindMode::SourceIp, &[en0()], true, ks_is("192.168.1.20")).is_ok()); + assert!(matches!( + build(BindMode::SourceIp, &[en0()], false, ks_is("192.168.1.20")), + Err(CapError::InvalidGateway(_)) + )); + // A Tailscale subnet route covering the LAN: the kernel picks the tailnet address. + assert!(matches!( + build(BindMode::SourceIp, &[en0()], true, ks_is("100.64.0.5")), + Err(CapError::InvalidGateway(_)) + )); + assert!(matches!( + build(BindMode::SourceIp, &[en0()], true, Arc::new(|_| None)), + Err(CapError::InvalidGateway(_)) + )); + // Interface-bound modes do not consult the kernel's source selection. + assert!(build(BindMode::IfScope, &[en0()], false, Arc::new(|_| None)).is_ok()); + } + + /// Source-IP mode re-checks the kernel's source selection before every send. + #[tokio::test] + async fn source_ip_mode_revalidates_before_each_send() { + let via_lan = Arc::new(AtomicBool::new(true)); + let flag = via_lan.clone(); + let ks: KernelSource = Arc::new(move |_| { + Some(if flag.load(Ordering::SeqCst) { + "192.168.1.20".parse().unwrap() + } else { + "100.64.0.5".parse().unwrap() + }) + }); + let cap = build(BindMode::SourceIp, &[en0()], true, ks).unwrap(); + assert!( + cap.gate + .check_async(Instant::now() + EXCHANGE_TIMEOUT) + .await + .is_ok() + ); + via_lan.store(false, Ordering::SeqCst); + assert!(matches!( + cap.gate + .check_async(Instant::now() + EXCHANGE_TIMEOUT) + .await, + Err(CapError::Denied(DenyReason::NetworkChanged)) + )); + // Once invalidated, it stays invalid. + via_lan.store(true, Ordering::SeqCst); + assert!( + cap.gate + .check_async(Instant::now() + EXCHANGE_TIMEOUT) + .await + .is_err() + ); + } + + /// Every cap socket carries `SO_DONTROUTE`; the loopback cap still works with it (the other + /// loopback tests send through such sockets). + #[cfg(unix)] + #[test] + fn cap_sockets_are_dontroute() { + assert!(dontroute_is_set(&new_socket().unwrap())); + let cap = GatewayCap::loopback_for_test(9, PolicyHandle::new(true)) + .expect("loopback capability construction must succeed"); + assert!(dontroute_is_set(&cap.socket(0).unwrap())); + } + + /// A loopback "router" that counts what it receives (a raw socket in this module, the only + /// one allowed to create sockets). + async fn counting_listener() -> (Arc, u16) { + let s = tokio::net::UdpSocket::bind((Ipv4Addr::LOCALHOST, 0)) + .await + .unwrap(); + let port = s.local_addr().unwrap().port(); + (Arc::new(s), port) + } + + async fn received(s: &tokio::net::UdpSocket) -> usize { + let mut buf = [0u8; 64]; + let mut n = 0; + while let Ok(Ok(_)) = + tokio::time::timeout(Duration::from_millis(50), s.recv_from(&mut buf)).await + { + n += 1; + } + n + } + + /// A send with the policy off, after `invalidate()`, or while closing is refused + /// without a socket write. + #[tokio::test] + async fn denied_sends_write_nothing() { + let (router, port) = counting_listener().await; + let policy = PolicyHandle::new(true); + let cap = GatewayCap::loopback_for_test(port, policy.clone()) + .expect("loopback capability construction must succeed"); + let deadline = Instant::now() + EXCHANGE_TIMEOUT; + let mut ignore = |_: &[u8]| Flow::Continue; + + policy.set_lan_sends_allowed(false); + let r = cap + .exchange(LanDest::Pxp, &[b"x"], deadline, &mut ignore) + .await; + assert!(matches!(r, Err(CapError::Denied(DenyReason::Policy)))); + assert!(matches!( + cap.send_nowait(LanDest::Pxp, b"x"), + Err(CapError::Denied(DenyReason::Policy)) + )); + policy.set_lan_sends_allowed(true); + + cap.set_closing(); + let r = cap + .exchange(LanDest::Pxp, &[b"x"], deadline, &mut ignore) + .await; + assert!(matches!(r, Err(CapError::Denied(DenyReason::Closing)))); + assert_eq!(received(&router).await, 0); + + let cap = GatewayCap::loopback_for_test(port, policy.clone()).unwrap(); + cap.invalidate(); + assert!(!cap.is_valid()); + let r = cap + .exchange(LanDest::Pxp, &[b"x"], deadline, &mut ignore) + .await; + assert!(matches!(r, Err(CapError::Denied(DenyReason::Invalidated)))); + assert!(cap.send_nowait(LanDest::Pxp, b"x").is_err()); + assert_eq!(received(&router).await, 0); + + // The same cap, allowed, does reach the loopback router: the denials above were real. + let cap = GatewayCap::loopback_for_test(port, policy).unwrap(); + cap.send_nowait(LanDest::Pxp, b"x").unwrap(); + let mut ignore = |_: &[u8]| Flow::Continue; + let deadline = Instant::now() + Duration::from_millis(20); + cap.exchange(LanDest::Pxp, &[b"y"], deadline, &mut ignore) + .await + .unwrap(); + assert_eq!(received(&router).await, 2); + } + + /// Binding to an address this host does not have fails closed, before any send. + #[tokio::test] + async fn bind_to_missing_self_ip_fails_closed() { + let mut g = loopback_gateway().expect("loopback interface must exist"); + // 192.0.2.0/24 is TEST-NET-1: never configured on a host. + g.self_ip = Ipv4Addr::new(192, 0, 2, 77); + let cap = GatewayCap { + bind_mode: probe_bind_mode(&g).unwrap(), + gateway: g, + generation: NetGen(1), + gate: SendGate::new(PolicyHandle::new(true), None), + pxp_port: 9, + }; + let err = cap.socket(0).unwrap_err(); + assert!(matches!(err, CapError::Io { op: "bind", .. }), "{err}"); + } + + /// The real cap's sockets carry `self_ip` as their source and are interface-bound. + #[tokio::test] + async fn real_cap_source_is_self_ip() { + let (router, port) = counting_listener().await; + let cap = GatewayCap::loopback_for_test(port, PolicyHandle::new(true)) + .expect("loopback capability construction must succeed"); + if cfg!(target_os = "macos") { + assert_eq!(cap.bind_mode(), BindMode::IfScope); + } + cap.send_nowait(LanDest::Pxp, b"hello").unwrap(); + let mut buf = [0u8; 16]; + let (n, src) = tokio::time::timeout(Duration::from_secs(2), router.recv_from(&mut buf)) + .await + .unwrap() + .unwrap(); + assert_eq!((n, src.ip()), (5, Ipv4Addr::LOCALHOST.into())); + } + + #[test] + fn errors_are_verbatim() { + #[cfg(unix)] + { + let e = CapError::io("send", &io::Error::from_raw_os_error(libc::EHOSTUNREACH)); + let s = e.to_string(); + assert!(s.starts_with("send: EHOSTUNREACH: "), "{s}"); + assert!( + s.contains(&format!("os error {}", libc::EHOSTUNREACH)), + "{s}" + ); + let e = CapError::io("send", &io::Error::from_raw_os_error(libc::EPERM)); + assert!(e.to_string().starts_with("send: EPERM: ")); + } + let e = CapError::io("recv", &io::Error::other("boom")); + assert_eq!(e.to_string(), "recv: boom"); + } + + #[test] + fn policy_handle_is_shared() { + let a = PolicyHandle::new(true); + let b = a.clone(); + b.set_lan_sends_allowed(false); + assert!(!a.lan_sends_allowed()); + } +} + +#[cfg(test)] +pub(crate) mod send_tests { + use super::*; + use std::{ + future::{Future, poll_fn}, + net::Ipv4Addr, + task::Poll, + time::Duration, + }; + + #[tokio::test] + async fn fenced_during_source_route_validation() { + for disable in [false, true] { + let recv = std::net::UdpSocket::bind((Ipv4Addr::LOCALHOST, 0)).unwrap(); + recv.set_nonblocking(true).unwrap(); + let to = recv.local_addr().unwrap(); + let sock = tokio::net::UdpSocket::bind((Ipv4Addr::LOCALHOST, 0)) + .await + .unwrap(); + sock.writable().await.unwrap(); + let entered = Arc::new(tokio::sync::Notify::new()); + let open = Arc::new((Mutex::new(false), std::sync::Condvar::new())); + let (hit, barrier) = (entered.clone(), open.clone()); + let policy = PolicyHandle::new(true); + let gate = SendGate::new( + policy.clone(), + Some(Arc::new(move || { + hit.notify_one(); + let (lock, cv) = &*barrier; + let mut ready = lock.lock().unwrap(); + while !*ready { + ready = cv.wait(ready).unwrap(); + } + true + })), + ); + let expected = match to { + SocketAddr::V4(a) => a, + _ => unreachable!(), + }; + let present = |a| match a { + SocketAddr::V4(a) => Some(a), + _ => None, + }; + let mut on_reply = |_: &[u8]| Flow::Done; + let reqs = [&b"route-cut"[..]]; + let mut exchange = Box::pin(exchange_on( + &sock, + &gate, + to, + expected, + &present, + &reqs, + Instant::now() + Duration::from_secs(2), + &mut on_reply, + )); + tokio::select! { + _ = entered.notified() => {}, + _ = &mut exchange => panic!("route check did not pause"), + } + if disable { + policy.set_lan_sends_allowed(false); + } else { + gate.invalidate(); + } + *open.0.lock().unwrap() = true; + open.1.notify_one(); + assert!(matches!(exchange.await, Err(CapError::Denied(_)))); + assert_eq!( + recv.recv(&mut [0u8; 32]).unwrap_err().kind(), + io::ErrorKind::WouldBlock + ); + } + } + + type Hook = (std::sync::mpsc::Sender<()>, std::sync::mpsc::Receiver<()>); + std::thread_local! { + static NOWAIT_HOOK: std::cell::RefCell> = const { std::cell::RefCell::new(None) }; + } + pub(super) fn pause_nowait_after_check() { + let hook = NOWAIT_HOOK.with(|hook| hook.borrow_mut().take()); + if let Some((reached, resume)) = hook { + reached.send(()).unwrap(); + resume.recv().unwrap(); + } + } + + fn nowait_fence(disable: bool) { + let recv = std::net::UdpSocket::bind((Ipv4Addr::LOCALHOST, 0)).unwrap(); + recv.set_nonblocking(true).unwrap(); + let policy = PolicyHandle::new(true); + let cap = std::sync::Arc::new( + GatewayCap::loopback_for_test(recv.local_addr().unwrap().port(), policy.clone()) + .unwrap(), + ); + let (reached_tx, reached_rx) = std::sync::mpsc::channel(); + let (resume_tx, resume_rx) = std::sync::mpsc::channel(); + let sender = cap.clone(); + let job = std::thread::spawn(move || { + NOWAIT_HOOK.with(|hook| *hook.borrow_mut() = Some((reached_tx, resume_rx))); + assert!(tokio::runtime::Handle::try_current().is_err()); + let request = crate::pmp::delete_request(std::num::NonZeroU16::new(41641).unwrap()); + sender.send_nowait(LanDest::Pxp, &request) + }); + reached_rx.recv_timeout(Duration::from_secs(1)).unwrap(); + if disable { + policy.set_lan_sends_allowed(false); + } else { + cap.invalidate(); + } + resume_tx.send(()).unwrap(); + let result = job.join().unwrap(); + let mut buf = [0u8; 32]; + assert!( + result.is_err(), + "final delete admitted after fence: {result:?}" + ); + assert_eq!( + recv.recv(&mut buf).unwrap_err().kind(), + io::ErrorKind::WouldBlock, + "must receive zero datagrams after the fence" + ); + } + + async fn waiting_send_fence(disable: bool) { + let recv = std::net::UdpSocket::bind((Ipv4Addr::LOCALHOST, 0)).unwrap(); + recv.set_nonblocking(true).unwrap(); + let dest = recv.local_addr().unwrap(); + let send = tokio::net::UdpSocket::bind((Ipv4Addr::LOCALHOST, 0)) + .await + .unwrap(); + let policy = PolicyHandle::new(true); + let gate = SendGate::new(policy.clone(), None); + let expected = match dest { + SocketAddr::V4(a) => a, + _ => unreachable!(), + }; + let present = |a| match a { + SocketAddr::V4(a) => Some(a), + _ => None, + }; + let mut on_reply = |_: &[u8]| Flow::Done; + let request = crate::pmp::delete_request(std::num::NonZeroU16::new(41641).unwrap()); + let requests = [&request[..]]; + let mut exchange = Box::pin(exchange_on( + &send, + &gate, + dest, + expected, + &present, + &requests, + Instant::now() + Duration::from_millis(20), + &mut on_reply, + )); + poll_fn(|cx| { + assert!(exchange.as_mut().poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + let mut buf = [0u8; 32]; + assert_eq!( + recv.recv(&mut buf).unwrap_err().kind(), + std::io::ErrorKind::WouldBlock, + "test cut must precede the actual send" + ); + if disable { + policy.set_lan_sends_allowed(false); + } else { + gate.invalidate(); + } + drop(exchange.await); + assert_eq!( + recv.recv(&mut buf).unwrap_err().kind(), + io::ErrorKind::WouldBlock, + "must receive zero datagrams after the fence" + ); + } + + #[test] + fn nowait_invalidation_before_socket_send() { + nowait_fence(false); + } + #[test] + fn nowait_policy_disable_before_socket_send() { + nowait_fence(true); + } + #[tokio::test] + async fn invalidated_while_waiting_for_first_udp_send() { + waiting_send_fence(false).await; + } + #[tokio::test] + async fn policy_disabled_while_waiting_for_first_udp_send() { + waiting_send_fence(true).await; + } + + std::thread_local! { + static BEFORE_SEND: std::cell::RefCell>> = const { std::cell::RefCell::new(None) }; + } + pub(crate) fn before_send() { + BEFORE_SEND.with(|hook| { + if let Some(hook) = hook.borrow_mut().as_mut() { + hook(); + } + }); + } + pub(crate) struct ClearHook; + impl Drop for ClearHook { + fn drop(&mut self) { + BEFORE_SEND.with(|hook| drop(hook.borrow_mut().take())); + } + } + + async fn ready_send_fence(disable: bool, between: bool) { + let recv = Arc::new(std::net::UdpSocket::bind((Ipv4Addr::LOCALHOST, 0)).unwrap()); + recv.set_nonblocking(!between).unwrap(); + recv.set_read_timeout(Some(Duration::from_secs(1))).unwrap(); + let dest = recv.local_addr().unwrap(); + let send = tokio::net::UdpSocket::bind((Ipv4Addr::LOCALHOST, 0)) + .await + .unwrap(); + send.writable().await.unwrap(); + let policy = PolicyHandle::new(true); + let gate = Arc::new(SendGate::new(policy.clone(), None)); + let entered = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let (g, r, done) = (gate.clone(), recv.clone(), entered.clone()); + let mut calls = 0; + BEFORE_SEND.with(|hook| { + *hook.borrow_mut() = Some(Box::new(move || { + calls += 1; + if calls != if between { 2 } else { 1 } { + return; + } + if between { + let mut before = [0u8; 32]; + let n = r.recv(&mut before).unwrap(); + assert_eq!(&before[..n], &crate::pmp::public_address_request()); + r.set_nonblocking(true).unwrap(); + } + if disable { + policy.set_lan_sends_allowed(false); + } else { + g.invalidate(); + } + done.store(true, Ordering::SeqCst); + })) + }); + let _clear = ClearHook; + let addr = crate::pmp::public_address_request(); + let map = crate::pmp::map_request( + std::num::NonZeroU16::new(41641).unwrap(), + 0, + std::num::NonZeroU32::new(7200).unwrap(), + ); + let both = [&addr[..], &map[..]]; + let reqs = if between { &both[..] } else { &both[1..] }; + let expected = match dest { + SocketAddr::V4(a) => a, + _ => unreachable!(), + }; + let present = |a| match a { + SocketAddr::V4(a) => Some(a), + _ => None, + }; + let mut on_reply = |_: &[u8]| Flow::Done; + let result = exchange_on( + &send, + &gate, + dest, + expected, + &present, + reqs, + Instant::now() + Duration::from_millis(20), + &mut on_reply, + ) + .await; + assert!( + entered.load(Ordering::SeqCst), + "scheduling cut was not reached" + ); + let mut buf = [0u8; 32]; + assert_eq!( + recv.recv(&mut buf).unwrap_err().kind(), + std::io::ErrorKind::WouldBlock, + "must receive zero datagrams after the fence returns" + ); + assert!(matches!(result, Err(CapError::Denied(_)))); + } + + #[tokio::test] + async fn fence_after_ready_gate_check() { + for disable in [false, true] { + ready_send_fence(disable, false).await; + } + } + #[tokio::test] + async fn fence_between_pmp_sends() { + for disable in [false, true] { + ready_send_fence(disable, true).await; + } + } + #[test] + fn send_permit_serializes_fences() { + for disable in [false, true] { + let recv = std::net::UdpSocket::bind((Ipv4Addr::LOCALHOST, 0)).unwrap(); + recv.set_nonblocking(true).unwrap(); + let dest = recv.local_addr().unwrap(); + let send = std::net::UdpSocket::bind((Ipv4Addr::LOCALHOST, 0)).unwrap(); + send.set_nonblocking(true).unwrap(); + let policy = PolicyHandle::new(true); + let gate = Arc::new(SendGate::new(policy.clone(), None)); + let (entered, at_send) = std::sync::mpsc::channel(); + let (resume, blocked) = std::sync::mpsc::channel(); + let g = gate.clone(); + let sender = std::thread::spawn(move || { + g.send(|| { + // Controlled preemption of a sender inside its final check/syscall window. + // This wait is test instrumentation; real send closures are nonblocking. + entered.send(()).unwrap(); + blocked.recv().unwrap(); + send.send_to(b"one", dest) + }) + }); + at_send.recv_timeout(Duration::from_secs(1)).unwrap(); + let (started, at_fence) = std::sync::mpsc::channel(); + let (done, returned) = std::sync::mpsc::channel(); + let g = gate.clone(); + let fence = std::thread::spawn(move || { + started.send(()).unwrap(); + if disable { + policy.set_lan_sends_allowed(false); + } else { + g.invalidate(); + } + done.send(()).unwrap(); + }); + at_fence.recv_timeout(Duration::from_secs(1)).unwrap(); + let early = returned.recv_timeout(Duration::from_millis(50)).is_ok(); + resume.send(()).unwrap(); + sender.join().unwrap().unwrap(); + fence.join().unwrap(); + assert!( + !early, + "fence returned while a sender still held the check/syscall window" + ); + let mut buf = [0u8; 32]; + // The first send linearized before the fence; no send may follow it. + recv.set_nonblocking(false).unwrap(); + recv.set_read_timeout(Some(Duration::from_secs(1))).unwrap(); + assert_eq!(recv.recv(&mut buf).unwrap(), 3); + recv.set_nonblocking(true).unwrap(); + let other = std::net::UdpSocket::bind((Ipv4Addr::LOCALHOST, 0)).unwrap(); + other.set_nonblocking(true).unwrap(); + assert!(gate.send(|| other.send_to(b"two", dest)).is_err()); + assert_eq!( + recv.recv(&mut buf).unwrap_err().kind(), + io::ErrorKind::WouldBlock + ); + } + } +} diff --git a/ts_portmapper/src/gateway/darwin.rs b/ts_portmapper/src/gateway/darwin.rs new file mode 100644 index 00000000..45e754d1 --- /dev/null +++ b/ts_portmapper/src/gateway/darwin.rs @@ -0,0 +1,652 @@ +//! macOS default routes from the routing-socket dump `sysctl(CTL_NET, PF_ROUTE, 0, AF_UNSPEC, +//! NET_RT_DUMP2, 0)`. +//! +//! The parser is pure, bounds-checked and compiled on every target so it can be tested and fuzzed +//! anywhere. It never casts the byte buffer to a struct: header fields are read with +//! `from_ne_bytes` at fixed offsets (asserted against `libc::rt_msghdr2` in a macOS test), and +//! sockaddrs are walked by their own `sa_len`, never past it or the message end. The only +//! `unsafe` in this module is the two `sysctl` calls. +//! +//! The dump is requested for `AF_UNSPEC` (as Go's `route.FetchRIB` does) rather than `AF_INET`, so +//! an IPv6-only default can be reported as such instead of as "offline". + +use std::net::Ipv4Addr; + +use super::{DefaultRoute, DefaultRoutes, LookupError}; + +/// Largest dump accepted. +pub const MAX_DUMP: usize = 4 << 20; + +/// `RTM_VERSION` on macOS. +const RTM_VERSION: u8 = 5; +/// `RTM_GET2`: the message type `NET_RT_DUMP2` returns. +const RTM_GET2: u8 = 0x14; +/// `sizeof(struct rt_msghdr2)`. +const HDR2_LEN: usize = 92; +/// Field offsets in `struct rt_msghdr2`. +const OFF_VERSION: usize = 2; +const OFF_TYPE: usize = 3; +const OFF_INDEX: usize = 4; +const OFF_FLAGS: usize = 8; +const OFF_ADDRS: usize = 12; + +const RTF_UP: u32 = 0x1; +const RTF_GATEWAY: u32 = 0x2; +const RTF_IFSCOPE: u32 = 0x100_0000; + +const RTAX_DST: usize = 0; +const RTAX_GATEWAY: usize = 1; +const RTAX_NETMASK: usize = 2; +const RTAX_IFA: usize = 5; +const RTAX_MAX: usize = 8; + +const AF_INET: u8 = 2; +const AF_LINK: u8 = 18; +const AF_INET6: u8 = 30; + +/// One decoded sockaddr. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Sa { + V4(Ipv4Addr), + V6([u8; 16]), + /// A link-layer address (`sockaddr_dl`), as used by interface routes such as utun defaults. + Link, + /// A netmask in any form, reduced to whether it is all zeros (`/0`). + Mask { + zero: bool, + }, + Other, +} + +fn parse_err(offset: usize, what: &'static str) -> LookupError { + LookupError::Parse { offset, what } +} + +fn ne_u16(b: &[u8], at: usize) -> Option { + b.get(at..at + 2).map(|s| u16::from_ne_bytes([s[0], s[1]])) +} + +fn ne_u32(b: &[u8], at: usize) -> Option { + b.get(at..at + 4) + .map(|s| u32::from_ne_bytes([s[0], s[1], s[2], s[3]])) +} + +/// Bytes a sockaddr of length `sa_len` occupies in a routing message (Darwin's `ROUNDUP32`). +const fn roundup(sa_len: usize) -> usize { + if sa_len == 0 { + 4 + } else { + 1 + ((sa_len - 1) | 3) + } +} + +/// Decode the sockaddr `sa` (exactly `sa_len` bytes) found at position `rtax`. +fn decode_sa(rtax: usize, sa: &[u8]) -> Sa { + let family = sa.get(1).copied().unwrap_or(0); + if rtax == RTAX_NETMASK { + // Netmasks are often compact: a short `sa_len`, family 0, trailing zero bytes omitted. Only + // "is it /0" matters here, and that holds iff every byte after the family is zero. + return Sa::Mask { + zero: sa.iter().skip(2).all(|&b| b == 0), + }; + } + match family { + AF_INET => match sa.get(4..8) { + Some(a) => Sa::V4(Ipv4Addr::new(a[0], a[1], a[2], a[3])), + None => Sa::Other, + }, + AF_INET6 => match sa.get(8..24) { + Some(a) => { + let mut v = [0u8; 16]; + v.copy_from_slice(a); + Sa::V6(v) + } + None => Sa::Other, + }, + AF_LINK => Sa::Link, + _ => Sa::Other, + } +} + +/// Parse a `NET_RT_DUMP2` dump into its usable default routes: `UP | GATEWAY`, not +/// `RTF_IFSCOPE`, destination `0.0.0.0/0` (or `::/0`) and an IP gateway. +pub fn parse_rt_dump(b: &[u8]) -> Result { + let mut out = DefaultRoutes::default(); + let mut off = 0; + while off < b.len() { + let msg = &b[off..]; + let msglen = usize::from(ne_u16(msg, 0).ok_or(parse_err(off, "truncated header"))?); + if msglen < 4 || msglen > msg.len() { + return Err(parse_err(off, "bad message length")); + } + let msg = &msg[..msglen]; + if msg[OFF_VERSION] != RTM_VERSION { + return Err(parse_err(off, "unexpected rtm_version")); + } + if msg[OFF_TYPE] == RTM_GET2 { + if msglen < HDR2_LEN { + return Err(parse_err(off, "message shorter than rt_msghdr2")); + } + parse_route(msg, off, &mut out)?; + } + off += msglen; + } + Ok(out) +} + +/// Parse one `RTM_GET2` message (`msg` is exactly `rtm_msglen` bytes, at least a header long). +fn parse_route(msg: &[u8], base: usize, out: &mut DefaultRoutes) -> Result<(), LookupError> { + let index = ne_u16(msg, OFF_INDEX).unwrap_or(0); + let flags = ne_u32(msg, OFF_FLAGS).unwrap_or(0); + let present = ne_u32(msg, OFF_ADDRS).unwrap_or(0); + + let mut addrs: [Option; RTAX_MAX] = [None; RTAX_MAX]; + let mut at = HDR2_LEN; + for (rtax, slot) in addrs.iter_mut().enumerate() { + if present & (1 << rtax) == 0 { + continue; + } + let Some(&sa_len) = msg.get(at) else { + return Err(parse_err(base + at, "sockaddr past message end")); + }; + let sa_len = usize::from(sa_len); + if sa_len > msg.len() - at { + return Err(parse_err(base + at, "sockaddr overruns message")); + } + *slot = Some(decode_sa(rtax, &msg[at..at + sa_len])); + at += roundup(sa_len).min(msg.len() - at); + } + + let usable = flags & RTF_UP != 0 && flags & RTF_GATEWAY != 0 && flags & RTF_IFSCOPE == 0; + if !usable || addrs[RTAX_NETMASK] != Some(Sa::Mask { zero: true }) { + return Ok(()); + } + match (addrs[RTAX_DST], addrs[RTAX_GATEWAY]) { + (Some(Sa::V4(dst)), Some(Sa::V4(gw))) if dst.is_unspecified() && !gw.is_unspecified() => { + let ifa = match addrs[RTAX_IFA] { + Some(Sa::V4(a)) if !a.is_unspecified() => Some(a), + _ => None, + }; + out.v4.push(DefaultRoute { + gw, + if_index: Some(u32::from(index)), + if_name: None, + metric: 0, + ifa, + }); + } + (Some(Sa::V6(dst)), Some(Sa::V6(_))) if dst == [0; 16] => out.has_v6_default = true, + _ => {} + } + Ok(()) +} + +/// Fetch the raw `NET_RT_DUMP2` dump: size query, a buffer 1.25× that, retried on `ENOMEM` (the +/// table grew in between) up to 3 times, capped at [`MAX_DUMP`]. +#[cfg(target_os = "macos")] +pub fn fetch_rt_dump() -> Result, LookupError> { + use std::{io, ptr}; + + let os_err = |e: io::Error| match e.raw_os_error() { + Some(libc::EPERM | libc::EACCES) => LookupError::Permission(e), + _ => LookupError::Os(e), + }; + let mut mib = [ + libc::CTL_NET, + libc::PF_ROUTE, + 0, + libc::AF_UNSPEC, + libc::NET_RT_DUMP2, + 0, + ]; + let miblen = mib.len() as libc::c_uint; + for _ in 0..3 { + let mut need: libc::size_t = 0; + // SAFETY: a size query: `mib` is valid for `miblen` ints, the old buffer is null, and + // `need` is a valid `size_t` the kernel writes the required size to. No new value is set. + let r = unsafe { + libc::sysctl( + mib.as_mut_ptr(), + miblen, + ptr::null_mut(), + &mut need, + ptr::null_mut(), + 0, + ) + }; + if r != 0 { + return Err(os_err(io::Error::last_os_error())); + } + if need > MAX_DUMP { + return Err(LookupError::TooLarge); + } + let cap = (need + need / 4).clamp(512, MAX_DUMP); + let mut buf = vec![0u8; cap]; + let mut got: libc::size_t = cap; + // SAFETY: `buf` is valid for writes of `got` bytes; the kernel writes at most `got` bytes + // and stores the length it wrote in `got`. No new value is set. + let r = unsafe { + libc::sysctl( + mib.as_mut_ptr(), + miblen, + buf.as_mut_ptr().cast(), + &mut got, + ptr::null_mut(), + 0, + ) + }; + if r != 0 { + let e = io::Error::last_os_error(); + if e.raw_os_error() == Some(libc::ENOMEM) { + continue; + } + return Err(os_err(e)); + } + buf.truncate(got.min(cap)); + return Ok(buf); + } + Err(LookupError::TooLarge) +} + +/// The host's default routes. +#[cfg(target_os = "macos")] +pub(super) fn default_routes() -> Result { + parse_rt_dump(&fetch_rt_dump()?) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::gateway::{IfInfo, select}; + + fn sin(ip: [u8; 4]) -> Vec { + let mut s = vec![16, AF_INET, 0, 0]; + s.extend_from_slice(&ip); + s.extend_from_slice(&[0; 8]); + s + } + + fn sin6(ip: [u8; 16]) -> Vec { + let mut s = vec![28, AF_INET6, 0, 0, 0, 0, 0, 0]; + s.extend_from_slice(&ip); + s.extend_from_slice(&[0; 4]); + s + } + + /// A `sockaddr_dl` with no name or address (20 bytes, as the kernel pads it). + fn sdl(index: u16) -> Vec { + let mut s = vec![20, AF_LINK]; + s.extend_from_slice(&index.to_ne_bytes()); + s.extend_from_slice(&[0; 16]); + s + } + + /// The compact `/0` netmask: `sa_len` 0, occupying 4 bytes. + fn mask0() -> Vec { + vec![0, 0, 0, 0] + } + + /// A compact /24 netmask: `sa_len` 7, family 0xff, three mask bytes after the 4-byte head. + fn mask24() -> Vec { + vec![7, 0xff, 0, 0, 255, 255, 255, 0] + } + + /// One `RTM_GET2` message with the given flags, interface and `(RTAX index, sockaddr)` list. + fn msg(flags: u32, index: u16, sas: &[(usize, Vec)]) -> Vec { + let mut m = vec![0u8; HDR2_LEN]; + m[OFF_VERSION] = RTM_VERSION; + m[OFF_TYPE] = RTM_GET2; + m[OFF_INDEX..OFF_INDEX + 2].copy_from_slice(&index.to_ne_bytes()); + m[OFF_FLAGS..OFF_FLAGS + 4].copy_from_slice(&flags.to_ne_bytes()); + let mut present = 0u32; + for (rtax, sa) in sas { + present |= 1 << rtax; + let mut sa = sa.clone(); + sa.resize(roundup(usize::from(sa[0])), 0); + m.extend_from_slice(&sa); + } + m[OFF_ADDRS..OFF_ADDRS + 4].copy_from_slice(&present.to_ne_bytes()); + let len = u16::try_from(m.len()).unwrap(); + m[0..2].copy_from_slice(&len.to_ne_bytes()); + m + } + + const UG: u32 = RTF_UP | RTF_GATEWAY; + + fn default_v4(gw: [u8; 4], index: u16, flags: u32) -> Vec { + msg( + flags, + index, + &[ + (RTAX_DST, sin([0; 4])), + (RTAX_GATEWAY, sin(gw)), + (RTAX_NETMASK, mask0()), + ], + ) + } + + fn en(name: &str, index: u32, addr: [u8; 4]) -> IfInfo { + IfInfo { + name: name.to_owned(), + index, + up: true, + loopback: false, + point_to_point: false, + v4: vec![(addr.into(), 24)], + } + } + + /// The macOS fixture list. + #[test] + fn fixtures() { + let utun_default = msg( + RTF_UP | RTF_IFSCOPE, + 14, + &[ + (RTAX_DST, sin([0; 4])), + (RTAX_GATEWAY, sdl(14)), + (RTAX_NETMASK, mask0()), + ], + ); + let lan = msg( + RTF_UP, + 4, + &[ + (RTAX_DST, sin([192, 168, 1, 0])), + (RTAX_GATEWAY, sdl(4)), + (RTAX_NETMASK, mask24()), + ], + ); + let dump = [ + // Scoped defaults (RTF_IFSCOPE) are skipped; the unscoped one wins. + default_v4([10, 0, 0, 1], 5, UG | RTF_IFSCOPE), + utun_default, + lan, + default_v4([192, 168, 1, 1], 4, UG), + // An IPv6 default. + msg( + UG, + 4, + &[ + (RTAX_DST, sin6([0; 16])), + ( + RTAX_GATEWAY, + sin6([0xfe, 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]), + ), + (RTAX_NETMASK, mask0()), + ], + ), + ] + .concat(); + let r = parse_rt_dump(&dump).unwrap(); + assert_eq!( + r.v4, + vec![DefaultRoute { + gw: Ipv4Addr::new(192, 168, 1, 1), + if_index: Some(4), + if_name: None, + metric: 0, + ifa: None, + }] + ); + assert!(r.has_v6_default); + let ifs = [ + en("en0", 4, [192, 168, 1, 20]), + en("en5", 5, [10, 0, 0, 20]), + ]; + let gw = select(&r, &ifs, &|_| None).unwrap().unwrap(); + assert_eq!( + (gw.if_name.as_str(), gw.self_ip), + ("en0", Ipv4Addr::new(192, 168, 1, 20)) + ); + } + + #[test] + fn utun_default_without_gateway_flag_is_ignored() { + let d = msg( + RTF_UP, + 14, + &[ + (RTAX_DST, sin([0; 4])), + (RTAX_GATEWAY, sdl(14)), + (RTAX_NETMASK, mask0()), + ], + ); + assert_eq!(parse_rt_dump(&d).unwrap(), DefaultRoutes::default()); + } + + /// A VPN's unscoped default via a point-to-point interface, carrying an RFC 1918 gateway. + #[test] + fn vpn_default_is_interface_unusable() { + let r = parse_rt_dump(&default_v4([10, 8, 0, 1], 14, UG)).unwrap(); + let mut utun = en("utun3", 14, [10, 8, 0, 2]); + utun.point_to_point = true; + assert!(matches!( + select(&r, &[utun], &|_| None), + Err(LookupError::InterfaceUnusable) + )); + // The interface vanished between the dump and the interface scan. + assert!(matches!( + select(&r, &[], &|_| None), + Err(LookupError::InterfaceUnusable) + )); + } + + #[test] + fn ipv6_only_default() { + let d = msg( + UG, + 4, + &[ + (RTAX_DST, sin6([0; 16])), + ( + RTAX_GATEWAY, + sin6([0xfe, 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]), + ), + (RTAX_NETMASK, sin6([0; 16])), + ], + ); + let r = parse_rt_dump(&d).unwrap(); + assert!(r.v4.is_empty() && r.has_v6_default); + assert!(matches!( + select(&r, &[], &|_| None), + Err(LookupError::GatewayIpv6) + )); + } + + #[test] + fn rtax_ifa_is_the_self_hint() { + let d = msg( + UG, + 4, + &[ + (RTAX_DST, sin([0; 4])), + (RTAX_GATEWAY, sin([192, 168, 1, 1])), + (RTAX_NETMASK, mask0()), + (4, sdl(4)), + (RTAX_IFA, sin([192, 168, 1, 21])), + ], + ); + let r = parse_rt_dump(&d).unwrap(); + assert_eq!(r.v4[0].ifa, Some(Ipv4Addr::new(192, 168, 1, 21))); + let mut i = en("en0", 4, [192, 168, 1, 20]); + i.v4.push((Ipv4Addr::new(192, 168, 1, 21), 24)); + let gw = select(&r, &[i], &|_| None).unwrap().unwrap(); + assert_eq!(gw.self_ip, Ipv4Addr::new(192, 168, 1, 21)); + } + + #[test] + fn non_default_masks_are_not_defaults() { + let d = msg( + UG, + 4, + &[ + (RTAX_DST, sin([0; 4])), + (RTAX_GATEWAY, sin([192, 168, 1, 1])), + (RTAX_NETMASK, mask24()), + ], + ); + assert!(parse_rt_dump(&d).unwrap().v4.is_empty()); + // No netmask at all: a host route, not a default. + let host = msg( + UG, + 4, + &[ + (RTAX_DST, sin([0; 4])), + (RTAX_GATEWAY, sin([192, 168, 1, 1])), + ], + ); + assert!(parse_rt_dump(&host).unwrap().v4.is_empty()); + } + + /// Truncated and garbled buffers are `Parse` errors, never a panic. + #[test] + fn truncated_and_garbled() { + let good = [ + default_v4([192, 168, 1, 1], 4, UG), + default_v4([10, 0, 0, 1], 5, UG), + ] + .concat(); + for cut in [1, 3, 50, HDR2_LEN + 2, good.len() - 1] { + assert!( + matches!(parse_rt_dump(&good[..cut]), Err(LookupError::Parse { .. })), + "cut at {cut}" + ); + } + let mut bad_version = good.clone(); + bad_version[OFF_VERSION] = 4; + assert!(matches!( + parse_rt_dump(&bad_version), + Err(LookupError::Parse { offset: 0, .. }) + )); + // A sockaddr whose length runs past the message. + let mut overrun = default_v4([192, 168, 1, 1], 4, UG); + overrun[HDR2_LEN] = 200; + assert!(matches!( + parse_rt_dump(&overrun), + Err(LookupError::Parse { .. }) + )); + // A message length shorter than the header. + let mut short = default_v4([192, 168, 1, 1], 4, UG); + short[0..2].copy_from_slice(&40u16.to_ne_bytes()); + assert!(matches!( + parse_rt_dump(&short), + Err(LookupError::Parse { .. }) + )); + // Messages of another type are skipped by length. + let mut other = default_v4([192, 168, 1, 1], 4, UG); + other[OFF_TYPE] = 0x12; + assert_eq!(parse_rt_dump(&other).unwrap(), DefaultRoutes::default()); + assert_eq!(parse_rt_dump(&[]).unwrap(), DefaultRoutes::default()); + } + + #[test] + fn roundup_matches_darwin() { + assert_eq!( + [0, 1, 4, 5, 16, 20, 28].map(roundup), + [4, 4, 4, 8, 16, 20, 28] + ); + } + + proptest::proptest! { + #[test] + fn never_panics(b in proptest::collection::vec(proptest::prelude::any::(), 0..600)) { + drop(parse_rt_dump(&b)); + } + + /// Garbage after a valid prefix never panics either. + #[test] + fn valid_prefix_then_garbage(tail in proptest::collection::vec(proptest::prelude::any::(), 0..200)) { + let mut b = default_v4([192, 168, 1, 1], 4, UG); + b.extend_from_slice(&tail); + drop(parse_rt_dump(&b)); + } + } + + /// The header offsets and constants match the platform headers. + #[cfg(target_os = "macos")] + #[test] + fn header_layout_matches_libc() { + use core::mem::{offset_of, size_of}; + assert_eq!(size_of::(), HDR2_LEN); + assert_eq!(offset_of!(libc::rt_msghdr2, rtm_msglen), 0); + assert_eq!(offset_of!(libc::rt_msghdr2, rtm_version), OFF_VERSION); + assert_eq!(offset_of!(libc::rt_msghdr2, rtm_type), OFF_TYPE); + assert_eq!(offset_of!(libc::rt_msghdr2, rtm_index), OFF_INDEX); + assert_eq!(offset_of!(libc::rt_msghdr2, rtm_flags), OFF_FLAGS); + assert_eq!(offset_of!(libc::rt_msghdr2, rtm_addrs), OFF_ADDRS); + assert_eq!(libc::RTM_VERSION, i32::from(RTM_VERSION)); + assert_eq!(libc::RTM_GET2, i32::from(RTM_GET2)); + assert_eq!(libc::RTF_UP as u32, RTF_UP); + assert_eq!(libc::RTF_GATEWAY as u32, RTF_GATEWAY); + assert_eq!(libc::RTF_IFSCOPE as u32, RTF_IFSCOPE); + assert_eq!(libc::RTAX_DST as usize, RTAX_DST); + assert_eq!(libc::RTAX_GATEWAY as usize, RTAX_GATEWAY); + assert_eq!(libc::RTAX_NETMASK as usize, RTAX_NETMASK); + assert_eq!(libc::RTAX_IFA as usize, RTAX_IFA); + assert_eq!(libc::RTAX_MAX as usize, RTAX_MAX); + assert_eq!(libc::AF_INET as u8, AF_INET); + assert_eq!(libc::AF_INET6 as u8, AF_INET6); + assert_eq!(libc::AF_LINK as u8, AF_LINK); + } + + /// Port of Go's `TestFetchRoutingTable` (tailscale/tailscale#1345): repeated fetches succeed + /// and agree on the default routes, and the live dump parses. A local read; no packet is sent. + #[cfg(target_os = "macos")] + #[test] + fn fetch_routing_table_repeatedly() { + let first = parse_rt_dump(&fetch_rt_dump().unwrap()).unwrap(); + for _ in 0..9 { + let again = parse_rt_dump(&fetch_rt_dump().unwrap()).unwrap(); + assert_eq!(again, first); + } + } + + /// Adapted from Go's `TestLikelyHomeRouterIPSyscallExec`: our best unscoped IPv4 default + /// agrees with `netstat -rn -f inet` (exec of a local tool; no network traffic). + #[cfg(target_os = "macos")] + #[test] + fn agrees_with_netstat() { + let Ok(out) = std::process::Command::new("/usr/sbin/netstat") + .args(["-r", "-n", "-f", "inet"]) + .output() + else { + return; + }; + let text = String::from_utf8_lossy(&out.stdout); + // Same filter as Go's `likelyHomeRouterIPDarwinExec`: "default", flags with G and without + // I (interface-scoped), an IPv4 gateway. + let netstat: Vec<(Ipv4Addr, String)> = text + .lines() + .filter_map(|l| { + let f: Vec<&str> = l.split_whitespace().collect(); + match f.as_slice() { + ["default", gw, flags, netif, ..] + if flags.contains('G') && !flags.contains('I') => + { + Some((gw.parse().ok()?, (*netif).to_owned())) + } + _ => None, + } + }) + .collect(); + let ours = parse_rt_dump(&fetch_rt_dump().unwrap()).unwrap(); + let ifs = crate::gateway::interfaces().unwrap(); + let mut ours: Vec<(Ipv4Addr, String)> = ours + .v4 + .iter() + .map(|r| { + let name = ifs + .iter() + .find(|i| Some(i.index) == r.if_index) + .map(|i| i.name.clone()) + .unwrap_or_default(); + (r.gw, name) + }) + .collect(); + let mut netstat = netstat; + ours.sort(); + netstat.sort(); + assert_eq!(ours, netstat); + } +} diff --git a/ts_portmapper/src/gateway/linux.rs b/ts_portmapper/src/gateway/linux.rs new file mode 100644 index 00000000..b1036905 --- /dev/null +++ b/ts_portmapper/src/gateway/linux.rs @@ -0,0 +1,436 @@ +//! Linux default routes from `/proc/net/route` (IPv4) and `/proc/net/ipv6_route` (only to tell +//! "offline" from "IPv6-only"). +//! +//! The parsers are pure and compiled on every target so they can be tested and fuzzed anywhere; +//! only the file reads are Linux-specific. +//! +//! Limitation: `/proc/net/route` shows the main routing table only. With policy routing (`ip rule`, +//! fwmark) the gateway may be reached through another table. Sends are safe anyway: the cap binds +//! its sockets to the interface (`SO_BINDTODEVICE`) and sets `SO_DONTROUTE`; when binding is not +//! permitted ([`crate::BindMode::SourceIp`]) the cap requires the kernel's source selection for the +//! gateway to be `self_ip`, at build time and before every send. The selection itself can still be +//! wrong under policy routing (`--portmap-gateway` covers that). A netlink backend is a later +//! improvement. + +use super::{DefaultRoute, LookupError}; + +/// Largest routing file read; anything bigger is [`LookupError::TooLarge`]. +pub const MAX_BYTES: usize = 64 * 1024; +/// Most lines accepted (header included); more is [`LookupError::TooLarge`]. +pub const MAX_LINES: usize = 1000; + +const RTF_UP: u32 = 0x1; +const RTF_GATEWAY: u32 = 0x2; + +fn parse_err(offset: usize, what: &'static str) -> LookupError { + LookupError::Parse { offset, what } +} + +/// Split capped input into lines, enforcing the byte and line caps. +fn capped_lines(text: &[u8]) -> Result, LookupError> { + if text.len() > MAX_BYTES { + return Err(LookupError::TooLarge); + } + let text = std::str::from_utf8(text).map_err(|e| parse_err(e.valid_up_to(), "not UTF-8"))?; + let lines: Vec<&str> = text.lines().filter(|l| !l.trim().is_empty()).collect(); + if lines.len() > MAX_LINES { + return Err(LookupError::TooLarge); + } + Ok(lines) +} + +fn hex_u32(field: &str, line: usize) -> Result { + if field.is_empty() || field.len() > 8 { + return Err(parse_err(line, "bad hex field")); + } + u32::from_str_radix(field, 16).map_err(|_| parse_err(line, "bad hex field")) +} + +/// Parse `/proc/net/route` into its usable IPv4 default routes: destination and mask 0, flags +/// `UP` and `GATEWAY`, and a nonzero gateway. +/// +/// Addresses are the kernel's `__be32` printed as a host-order integer, so the bytes come back +/// with `to_ne_bytes` (`0101A8C0` is 192.168.1.1 on a little-endian host). +pub fn parse_proc_net_route(text: &[u8]) -> Result, LookupError> { + let lines = capped_lines(text)?; + let Some((header, rows)) = lines.split_first() else { + return Err(parse_err(0, "empty file")); + }; + let cols: Vec<&str> = header.split_whitespace().collect(); + let col = |name: &str| { + cols.iter() + .position(|c| c.eq_ignore_ascii_case(name)) + .ok_or(parse_err(0, "missing column")) + }; + let (c_if, c_dst, c_gw, c_flags, c_metric, c_mask) = ( + col("Iface")?, + col("Destination")?, + col("Gateway")?, + col("Flags")?, + col("Metric")?, + col("Mask")?, + ); + let needed = [c_if, c_dst, c_gw, c_flags, c_metric, c_mask] + .into_iter() + .max() + .unwrap_or(0); + + let mut out = Vec::new(); + for (i, row) in rows.iter().enumerate() { + let line = i + 1; + let f: Vec<&str> = row.split_whitespace().collect(); + if f.len() <= needed { + return Err(parse_err(line, "short line")); + } + let dst = hex_u32(f[c_dst], line)?; + let mask = hex_u32(f[c_mask], line)?; + let flags = hex_u32(f[c_flags], line)?; + let gw = hex_u32(f[c_gw], line)?; + let metric = f[c_metric] + .parse::() + .map_err(|_| parse_err(line, "bad metric"))?; + let usable = flags & RTF_UP != 0 && flags & RTF_GATEWAY != 0; + if dst != 0 || mask != 0 || !usable || gw == 0 { + continue; + } + out.push(DefaultRoute { + gw: gw.to_ne_bytes().into(), + if_index: None, + if_name: Some(f[c_if].to_owned()), + metric, + ifa: None, + }); + } + Ok(out) +} + +/// Whether `/proc/net/ipv6_route` has a usable default route (`::/0` with `UP` and `GATEWAY`). +pub fn parse_proc_net_ipv6_route_has_default(text: &[u8]) -> Result { + for (line, row) in capped_lines(text)?.into_iter().enumerate() { + let f: Vec<&str> = row.split_whitespace().collect(); + let [dst, plen, _src, _splen, _nh, _metric, _ref, _use, flags, ..] = f.as_slice() else { + return Err(parse_err(line, "short line")); + }; + let flags = hex_u32(flags, line)?; + let is_default = dst.len() == 32 && dst.bytes().all(|b| b == b'0') && *plen == "00"; + if is_default && flags & RTF_UP != 0 && flags & RTF_GATEWAY != 0 { + return Ok(true); + } + } + Ok(false) +} + +/// Combine `/proc/net/route` (`v4`, `None` if missing) with the advisory IPv6 check. +/// +/// `/proc/net/ipv6_route` only tells "offline" from "IPv6-only", so `read_v6` is called only when +/// there is no IPv4 default route, and any error reading or parsing it (for example +/// [`LookupError::TooLarge`] on a host with hundreds of veth routes) counts as "no IPv6 default" +/// rather than failing a lookup whose IPv4 answer is fine. +pub fn default_routes_from( + v4: Option<&[u8]>, + read_v6: impl FnOnce() -> Result>, LookupError>, +) -> Result { + let v4 = match v4 { + Some(b) => parse_proc_net_route(b)?, + None => Vec::new(), + }; + let has_v6_default = v4.is_empty() + && matches!( + read_v6().map(|b| b.map(|b| parse_proc_net_ipv6_route_has_default(&b))), + Ok(Some(Ok(true))) + ); + Ok(super::DefaultRoutes { v4, has_v6_default }) +} + +/// Read a routing file, at most [`MAX_BYTES`] of it. A missing file reads as empty. +#[cfg(target_os = "linux")] +fn read_capped(path: &str) -> Result>, LookupError> { + use std::io::Read; + let f = match std::fs::File::open(path) { + Ok(f) => f, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(e) if e.kind() == std::io::ErrorKind::PermissionDenied => { + return Err(LookupError::Permission(e)); + } + Err(e) => return Err(LookupError::Os(e)), + }; + let mut buf = Vec::new(); + f.take(MAX_BYTES as u64 + 1) + .read_to_end(&mut buf) + .map_err(LookupError::Os)?; + if buf.len() > MAX_BYTES { + return Err(LookupError::TooLarge); + } + Ok(Some(buf)) +} + +/// The host's default routes. +#[cfg(target_os = "linux")] +pub(super) fn default_routes() -> Result { + let v4 = read_capped("/proc/net/route")?; + default_routes_from(v4.as_deref(), || read_capped("/proc/net/ipv6_route")) +} + +#[cfg(test)] +mod tests { + use std::{fmt::Write, net::Ipv4Addr}; + + use super::*; + use crate::gateway::{DefaultRoutes, IfInfo, select}; + + const HEADER: &str = + "Iface\tDestination\tGateway\tFlags\tRefCnt\tUse\tMetric\tMask\tMTU\tWindow\tIRTT\n"; + + /// An address as `/proc/net/route` prints it on this host. + fn hex(ip: &str) -> String { + let ip: Ipv4Addr = ip.parse().unwrap(); + format!("{:08X}", u32::from_ne_bytes(ip.octets())) + } + + fn row(iface: &str, dst: &str, gw: &str, flags: u32, metric: u32, mask: &str) -> String { + format!( + "{iface}\t{}\t{}\t{flags:04X}\t0\t0\t{metric}\t{}\t0\t0\t0\n", + hex(dst), + hex(gw), + hex(mask) + ) + } + + fn eth(name: &str, addr: &str) -> IfInfo { + IfInfo { + name: name.to_owned(), + index: 7, + up: true, + loopback: false, + point_to_point: false, + v4: vec![(addr.parse().unwrap(), 24)], + } + } + + fn select_from(text: &str, ifs: &[IfInfo]) -> Result, LookupError> { + let v4 = parse_proc_net_route(text.as_bytes())?; + select( + &DefaultRoutes { + v4, + has_v6_default: false, + }, + ifs, + &|_| None, + ) + } + + /// The kernel's byte order on a little-endian host. + #[cfg(target_endian = "little")] + #[test] + fn hex_byte_order_golden() { + let text = + format!("{HEADER}eth0\t00000000\t0101A8C0\t0003\t0\t0\t100\t00000000\t0\t0\t0\n"); + let r = parse_proc_net_route(text.as_bytes()).unwrap(); + assert_eq!(r[0].gw, Ipv4Addr::new(192, 168, 1, 1)); + assert_eq!(r[0].if_name.as_deref(), Some("eth0")); + assert_eq!(r[0].metric, 100); + } + + /// Metric selection, ties, down routes and non-default routes. + #[test] + fn selection_fixtures() { + let ifs = [eth("eth0", "192.168.1.20"), eth("wlan0", "10.0.0.20")]; + let text = format!( + "{HEADER}{}{}{}{}", + row("wlan0", "0.0.0.0", "10.0.0.1", 0x3, 600, "0.0.0.0"), + row("eth0", "0.0.0.0", "192.168.1.1", 0x3, 100, "0.0.0.0"), + row("eth0", "192.168.1.0", "0.0.0.0", 0x1, 100, "255.255.255.0"), + // A down default (no RTF_UP) with a better metric is ignored. + row("wlan0", "0.0.0.0", "10.0.0.1", 0x2, 1, "0.0.0.0"), + ); + let got = select_from(&text, &ifs).unwrap().unwrap(); + assert_eq!(got.gw, Ipv4Addr::new(192, 168, 1, 1)); + assert_eq!(got.if_name, "eth0"); + + let tie = format!( + "{HEADER}{}{}", + row("wlan0", "0.0.0.0", "10.0.0.1", 0x3, 100, "0.0.0.0"), + row("eth0", "0.0.0.0", "192.168.1.1", 0x3, 100, "0.0.0.0"), + ); + assert!(matches!( + select_from(&tie, &ifs), + Err(LookupError::Ambiguous) + )); + + // A public best default never falls through to the private one. + let public = format!( + "{HEADER}{}{}", + row("eth0", "0.0.0.0", "203.0.113.1", 0x3, 0, "0.0.0.0"), + row("wlan0", "0.0.0.0", "10.0.0.1", 0x3, 100, "0.0.0.0"), + ); + assert!(matches!( + select_from(&public, &ifs), + Err(LookupError::NotPrivate) + )); + } + + /// Go's `TestAwsAppRunnerDefaultRouteInterface` fixture: the default's gateway is link-local + /// (169.254.175.249), so there is no usable gateway. + #[cfg(target_endian = "little")] + #[test] + fn aws_app_runner_fixture() { + let text = format!( + "{HEADER}eth0\t00000000\tF9AFFEA9\t0003\t0\t0\t0\t00000000\t0\t0\t0\n\ + *\tFEA9FEA9\t00000000\t0005\t0\t0\t0\tFFFFFFFF\t0\t0\t0\n\ + ecs-eth0\t02AAFEA9\t01ACFEA9\t0007\t0\t0\t0\tFFFFFFFF\t0\t0\t0\n\ + ecs-eth0\t00ACFEA9\t00000000\t0001\t0\t0\t0\t00FFFFFF\t0\t0\t0\n\ + eth0\t00AFFEA9\t00000000\t0001\t0\t0\t0\t00FFFFFF\t0\t0\t0\n" + ); + let r = parse_proc_net_route(text.as_bytes()).unwrap(); + assert_eq!(r.len(), 1); + assert_eq!(r[0].gw, Ipv4Addr::new(169, 254, 175, 249)); + assert!(matches!( + select_from(&text, &[]), + Err(LookupError::NotPrivate) + )); + } + + /// Go's `TestGoogleCloudRunDefaultRouteInterface` fixture: the default has no gateway, so + /// there is nothing to map through. + #[test] + fn cloud_run_fixture() { + let text = format!( + "{HEADER}eth0\t8008FEA9\t00000000\t0001\t0\t0\t0\t01FFFFFF\t0\t0\t0\n\ + eth1\t00000000\t00000000\t0001\t0\t0\t0\t00000000\t0\t0\t0\n" + ); + assert!(parse_proc_net_route(text.as_bytes()).unwrap().is_empty()); + } + + /// Port of Go's `TestExtremelyLongProcNetRoute` (903 lines are fine), plus the caps. + #[test] + fn extremely_long_proc_net_route() { + let mut text = HEADER.to_owned(); + for n in 0..=900 { + writeln!( + text, + "eth{n}\t8008FEA9\t00000000\t0001\t0\t0\t0\t01FFFFFF\t0\t0\t0" + ) + .unwrap(); + } + text.push_str(&row( + "tokenring1", + "0.0.0.0", + "192.168.1.1", + 0x3, + 0, + "0.0.0.0", + )); + let r = parse_proc_net_route(text.as_bytes()).unwrap(); + assert_eq!(r.len(), 1); + assert_eq!(r[0].if_name.as_deref(), Some("tokenring1")); + + // Over the line cap. + for n in 0..100 { + writeln!( + text, + "eth{n}\t8008FEA9\t00000000\t0001\t0\t0\t0\t01FFFFFF\t0\t0\t0" + ) + .unwrap(); + } + assert!(matches!( + parse_proc_net_route(text.as_bytes()), + Err(LookupError::TooLarge) + )); + // Over the byte cap. + let big = format!("{HEADER}{}", " ".repeat(MAX_BYTES)); + assert!(matches!( + parse_proc_net_route(big.as_bytes()), + Err(LookupError::TooLarge) + )); + } + + #[test] + fn garbled_input_is_a_parse_error() { + for bad in [ + "".to_owned(), + "Iface\tGateway\n".to_owned(), + format!("{HEADER}eth0\t00000000\n"), + format!("{HEADER}eth0\tZZZZZZZZ\t0101A8C0\t0003\t0\t0\t0\t00000000\t0\t0\t0\n"), + format!("{HEADER}eth0\t00000000\t0101A8C0\t0003\t0\t0\tx\t00000000\t0\t0\t0\n"), + format!("{HEADER}eth0\t000000000\t0101A8C0\t0003\t0\t0\t0\t00000000\t0\t0\t0\n"), + ] { + assert!( + matches!( + parse_proc_net_route(bad.as_bytes()), + Err(LookupError::Parse { .. }) + ), + "{bad:?}" + ); + } + assert!(matches!( + parse_proc_net_route(&[0xff, 0xfe]), + Err(LookupError::Parse { .. }) + )); + } + + #[test] + fn ipv6_default_detection() { + let zero = "0".repeat(32); + let nh = "fe800000000000000000000000000001"; + let def = format!("{zero} 00 {zero} 00 {nh} 00000400 00000001 00000000 00000003 eth0\n"); + assert!(parse_proc_net_ipv6_route_has_default(def.as_bytes()).unwrap()); + let link = format!( + "fe800000000000000000000000000000 40 {zero} 00 {zero} 00000100 00000001 00000000 00000001 eth0\n" + ); + assert!(!parse_proc_net_ipv6_route_has_default(link.as_bytes()).unwrap()); + assert!(parse_proc_net_ipv6_route_has_default(b"short line\n").is_err()); + assert!(!parse_proc_net_ipv6_route_has_default(b"").unwrap()); + } + + /// The IPv6 table is advisory: a huge one (hundreds of veth `fe80::/64` routes, over the cap) + /// never fails a lookup with a valid IPv4 default, and is not even read then. + #[test] + fn ipv6_route_is_advisory() { + let zero = "0".repeat(32); + let mut veths = String::new(); + for i in 0..450 { + writeln!( + veths, + "fe800000000000000000000000000000 40 {zero} 00 {zero} 00000100 00000001 00000000 00000001 veth{i:05x}" + ) + .unwrap(); + } + assert!(veths.len() > MAX_BYTES); + assert!(matches!( + parse_proc_net_ipv6_route_has_default(veths.as_bytes()), + Err(LookupError::TooLarge) + )); + + let v4 = format!( + "{HEADER}{}", + row("eth0", "0.0.0.0", "192.168.1.1", 0x3, 100, "0.0.0.0") + ); + let r = default_routes_from(Some(v4.as_bytes()), || { + panic!("ipv6_route read with an IPv4 default present") + }) + .unwrap(); + let got = select(&r, &[eth("eth0", "192.168.1.20")], &|_| None) + .unwrap() + .unwrap(); + assert_eq!(got.gw, Ipv4Addr::new(192, 168, 1, 1)); + + // No IPv4 default: an oversize or unreadable IPv6 table means "no IPv6 default". + let r = default_routes_from(Some(HEADER.as_bytes()), || Ok(Some(veths.into_bytes()))); + assert_eq!(r.unwrap(), DefaultRoutes::default()); + let r = default_routes_from(None, || Err(LookupError::TooLarge)); + assert_eq!(r.unwrap(), DefaultRoutes::default()); + // And a real IPv6 default is still reported. + let nh = "fe800000000000000000000000000001"; + let def = format!("{zero} 00 {zero} 00 {nh} 00000400 00000001 00000000 00000003 eth0\n"); + let r = default_routes_from(None, || Ok(Some(def.into_bytes()))).unwrap(); + assert!(r.has_v6_default && r.v4.is_empty()); + } + + proptest::proptest! { + #[test] + fn never_panics(b in proptest::collection::vec(proptest::prelude::any::(), 0..512)) { + drop(parse_proc_net_route(&b)); + drop(parse_proc_net_ipv6_route_has_default(&b)); + } + } +} diff --git a/ts_portmapper/src/gateway/mod.rs b/ts_portmapper/src/gateway/mod.rs new file mode 100644 index 00000000..f8c5bf36 --- /dev/null +++ b/ts_portmapper/src/gateway/mod.rs @@ -0,0 +1,750 @@ +//! Default-gateway lookup and validation. +//! +//! The port mapper only ever talks to one host: the gateway of the host's most preferred IPv4 +//! default route, and only when that gateway is private (RFC 1918) and on-link. This module finds +//! it and proves those properties; [`crate::GatewayCap`] re-checks them before any socket exists. + +use std::{ + fmt, io, + net::{Ipv4Addr, SocketAddr}, + str::FromStr, +}; + +pub mod darwin; +pub mod linux; +#[cfg(not(any(target_os = "macos", target_os = "linux")))] +mod other; + +#[cfg(target_os = "macos")] +use darwin as os; +#[cfg(target_os = "linux")] +use linux as os; +#[cfg(not(any(target_os = "macos", target_os = "linux")))] +use other as os; + +/// A validated LAN gateway: where PMP/PCP requests go, and the host address and interface they +/// are sent from. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct Gateway { + /// The gateway (router) address. Private (RFC 1918) and on-link. + pub gw: Ipv4Addr, + /// This host's address on `if_index`. Private, and its prefix contains `gw`. + pub self_ip: Ipv4Addr, + /// Prefix length of `self_ip`'s subnet. + pub prefix_len: u8, + /// Interface index the default route uses. + pub if_index: u32, + /// Interface name the default route uses. + pub if_name: String, +} + +impl Gateway { + /// Whether two gateways describe the same network: identical gateway, self address, + /// interface and prefix. Two networks with the same identity are indistinguishable. + pub fn same_network(&self, other: &Gateway) -> bool { + self == other + } +} + +/// Why no gateway could be determined. `Ok(None)` from a lookup means "no IPv4 default route", +/// which is the normal offline state and not an error. +#[derive(Debug, thiserror::Error)] +pub enum LookupError { + /// Gateway lookup is not implemented on this platform. + #[error("gateway lookup is not supported on this platform")] + Unsupported, + /// The OS refused the routing-table read (`EPERM`/`EACCES`); the error is kept verbatim. + #[error("routing table read not permitted: {0}")] + Permission(io::Error), + /// Another OS error from the routing-table read; kept verbatim. + #[error("routing table read failed: {0}")] + Os(io::Error), + /// The routing table could not be parsed. + #[error("routing table parse error at offset {offset}: {what}")] + Parse { + /// Byte (macOS) or line (Linux) offset of the problem. + offset: usize, + /// What was wrong. + what: &'static str, + }, + /// The routing table exceeded the size cap. + #[error("routing table too large")] + TooLarge, + /// The only default route is IPv6; the port mapper is IPv4-only. + #[error("the best default route is IPv6-only")] + GatewayIpv6, + /// The best default route's gateway (or this host's address) is not private. + #[error("the best default route's gateway or self address is not private (RFC 1918)")] + NotPrivate, + /// No address on the route's interface has a prefix containing the gateway. + #[error("no address on the default route's interface contains the gateway")] + NotOnLink, + /// The route's interface is down, loopback, point-to-point, or vanished. + #[error("the default route's interface is unusable (down, loopback, point-to-point or gone)")] + InterfaceUnusable, + /// Equally preferred default routes or self addresses; refusing to guess. + #[error("ambiguous default route or self address")] + Ambiguous, +} + +/// One usable IPv4 default route as the OS reports it, before any validation: destination +/// `0.0.0.0/0`, flags `UP` and `GATEWAY`, an IPv4 gateway, and (macOS) not interface-scoped. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DefaultRoute { + /// The route's gateway. + pub gw: Ipv4Addr, + /// Interface index, when the OS reports one (macOS). + pub if_index: Option, + /// Interface name, when the OS reports one (Linux). + pub if_name: Option, + /// Route metric; lower is preferred. Always 0 on macOS. + pub metric: u32, + /// The route's interface address (macOS `RTAX_IFA`), a hint for the self address. + pub ifa: Option, +} + +/// The default routes found in one routing-table read. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct DefaultRoutes { + /// Usable IPv4 default routes. + pub v4: Vec, + /// Whether a usable IPv6 default route exists (to tell "offline" from "IPv6-only"). + pub has_v6_default: bool, +} + +/// A source of the current [`Gateway`]. +pub trait GatewayLookup: Send + Sync { + /// Look up the current gateway. `Ok(None)` means there is no IPv4 default route. + fn lookup(&self) -> Result, LookupError>; +} + +impl fmt::Display for Gateway { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "gw={} self={}/{} if={}/{}", + self.gw, self.self_ip, self.prefix_len, self.if_name, self.if_index + ) + } +} + +/// One interface as the gateway validation needs it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct IfInfo { + /// Interface name. + pub name: String, + /// Interface index (0 if unknown). + pub index: u32, + /// Up and running. + pub up: bool, + /// A loopback interface. + pub loopback: bool, + /// A point-to-point interface (VPN tunnels such as utun or WireGuard). + pub point_to_point: bool, + /// IPv4 addresses with their prefix lengths. + pub v4: Vec<(Ipv4Addr, u8)>, +} + +impl IfInfo { + /// Up, not loopback and not point-to-point. + pub(crate) fn usable(&self) -> bool { + self.up && !self.loopback && !self.point_to_point + } +} + +/// A configured gateway (`--portmap-gateway GW[,SELF]`). It replaces discovery only: the result +/// goes through the same private, on-link and interface validation as a discovered route. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct GatewayOverride { + /// The gateway address. + pub gw: Ipv4Addr, + /// This host's address on the gateway's subnet, if it should not be inferred. + pub self_ip: Option, +} + +impl FromStr for GatewayOverride { + type Err = String; + + /// Parse `GW` or `GW,SELF`. + fn from_str(s: &str) -> Result { + let parse = |v: &str| { + v.trim() + .parse::() + .map_err(|e| format!("{v:?} is not an IPv4 address: {e}")) + }; + match s.split_once(',') { + None => Ok(Self { + gw: parse(s)?, + self_ip: None, + }), + Some((gw, me)) => Ok(Self { + gw: parse(gw)?, + self_ip: Some(parse(me)?), + }), + } + } +} + +/// Whether `ip` is in RFC 1918 space. Link-local, CGNAT, loopback, unspecified and public +/// addresses are not. +pub(crate) fn is_private(ip: Ipv4Addr) -> bool { + ip.is_private() +} + +/// The prefix lengths a LAN self address may have: a `/0`..`/7` "subnet" would make any private +/// gateway look on-link, and a `/32` has no neighbours. +pub(crate) const LAN_PREFIX_LENS: std::ops::RangeInclusive = 8..=31; + +/// Whether `net/prefix_len` contains `ip`. +pub(crate) fn prefix_contains(net: Ipv4Addr, prefix_len: u8, ip: Ipv4Addr) -> bool { + if prefix_len > 32 { + return false; + } + let mask = u32::MAX + .checked_shl(32 - u32::from(prefix_len)) + .unwrap_or(0); + u32::from(net) & mask == u32::from(ip) & mask +} + +/// The kernel's preferred source address for `dst`, from a UDP socket that is `connect()`ed but +/// never sends: a UDP connect only runs the route lookup. Used to break a tie between several +/// candidate self addresses. +pub(crate) fn kernel_source(dst: Ipv4Addr) -> Option { + let s = std::net::UdpSocket::bind((Ipv4Addr::UNSPECIFIED, 0)).ok()?; + s.connect((dst, crate::PXP_PORT)).ok()?; + match s.local_addr().ok()? { + SocketAddr::V4(a) => Some(*a.ip()), + SocketAddr::V6(_) => None, + } +} + +/// Pick this host's address on `iface` for talking to `gw`: an address whose prefix contains +/// `gw`, preferring the route's own interface address, then the kernel's source selection. +fn pick_self( + gw: Ipv4Addr, + iface: &IfInfo, + hint: Option, + kernel_source: &dyn Fn(Ipv4Addr) -> Option, +) -> Result<(Ipv4Addr, u8), LookupError> { + let candidates: Vec<(Ipv4Addr, u8)> = iface + .v4 + .iter() + .copied() + .filter(|&(a, p)| a != gw && LAN_PREFIX_LENS.contains(&p) && prefix_contains(a, p, gw)) + .collect(); + let chosen = match candidates.as_slice() { + [] => return Err(LookupError::NotOnLink), + [one] => *one, + many => { + let by_hint = hint.and_then(|h| many.iter().find(|(a, _)| *a == h)); + match by_hint { + Some(c) => *c, + None => { + let src = kernel_source(gw).ok_or(LookupError::Ambiguous)?; + *many + .iter() + .find(|(a, _)| *a == src) + .ok_or(LookupError::Ambiguous)? + } + } + } + }; + if !is_private(chosen.0) { + return Err(LookupError::NotPrivate); + } + Ok(chosen) +} + +/// Choose and validate the gateway from one routing-table read and the interface list. +/// +/// The single most preferred IPv4 default route wins (lowest metric; ties are `Ambiguous`). Its +/// gateway must be private: a public best route is `NotPrivate`, never a fall-through to a less +/// preferred private one. Its interface must be up, not loopback and not point-to-point, and hold +/// a private address whose prefix contains the gateway. +pub fn select( + routes: &DefaultRoutes, + ifs: &[IfInfo], + kernel_source: &dyn Fn(Ipv4Addr) -> Option, +) -> Result, LookupError> { + let Some(min_metric) = routes.v4.iter().map(|r| r.metric).min() else { + return if routes.has_v6_default { + Err(LookupError::GatewayIpv6) + } else { + Ok(None) + }; + }; + let mut best: Vec<&DefaultRoute> = Vec::new(); + for r in routes.v4.iter().filter(|r| r.metric == min_metric) { + // The same route reported twice is one route, not a tie. + if !best + .iter() + .any(|b| b.gw == r.gw && b.if_index == r.if_index && b.if_name == r.if_name) + { + best.push(r); + } + } + let [route] = best.as_slice() else { + return Err(LookupError::Ambiguous); + }; + if !is_private(route.gw) { + return Err(LookupError::NotPrivate); + } + let iface = ifs + .iter() + .find(|i| { + route.if_index.is_none_or(|x| x == i.index) + && route.if_name.as_deref().is_none_or(|n| n == i.name) + && (route.if_index.is_some() || route.if_name.is_some()) + }) + .ok_or(LookupError::InterfaceUnusable)?; + if !iface.usable() { + return Err(LookupError::InterfaceUnusable); + } + let (self_ip, prefix_len) = pick_self(route.gw, iface, route.ifa, kernel_source)?; + Ok(Some(Gateway { + gw: route.gw, + self_ip, + prefix_len, + if_index: iface.index, + if_name: iface.name.clone(), + })) +} + +/// Validate a configured gateway exactly like a discovered one: private, on-link on a usable +/// interface, with a private self address. +pub fn validate_override( + o: &GatewayOverride, + ifs: &[IfInfo], + kernel_source: &dyn Fn(Ipv4Addr) -> Option, +) -> Result { + if !is_private(o.gw) { + return Err(LookupError::NotPrivate); + } + let on_link = |i: &&IfInfo| { + i.v4.iter().any(|&(a, p)| { + a != o.gw && prefix_contains(a, p, o.gw) && o.self_ip.is_none_or(|s| s == a) + }) + }; + let matching: Vec<&IfInfo> = ifs.iter().filter(on_link).collect(); + let usable: Vec<&IfInfo> = matching.iter().copied().filter(|i| i.usable()).collect(); + let iface = match usable.as_slice() { + [] if matching.is_empty() => return Err(LookupError::NotOnLink), + [] => return Err(LookupError::InterfaceUnusable), + [one] => *one, + many => { + let src = kernel_source(o.gw).ok_or(LookupError::Ambiguous)?; + *many + .iter() + .find(|i| i.v4.iter().any(|&(a, _)| a == src)) + .ok_or(LookupError::Ambiguous)? + } + }; + let mut only_self = iface.clone(); + if let Some(s) = o.self_ip { + only_self.v4.retain(|&(a, _)| a == s); + } + let (self_ip, prefix_len) = pick_self(o.gw, &only_self, o.self_ip, kernel_source)?; + Ok(Gateway { + gw: o.gw, + self_ip, + prefix_len, + if_index: iface.index, + if_name: iface.name.clone(), + }) +} + +/// The OS gateway lookup: the routing table plus the interface list, or a validated override. +/// +/// Reading it sends no packet. On platforms other than macOS and Linux every lookup is +/// [`LookupError::Unsupported`], override or not. +#[derive(Debug, Clone, Default)] +pub struct SystemLookup { + gateway_override: Option, +} + +impl SystemLookup { + /// A lookup that discovers the gateway, or validates `gateway_override` instead. + pub fn new(gateway_override: Option) -> Self { + Self { gateway_override } + } + + /// Whether this lookup validates an override rather than discovering a route. + pub fn is_override(&self) -> bool { + self.gateway_override.is_some() + } +} + +impl GatewayLookup for SystemLookup { + fn lookup(&self) -> Result, LookupError> { + if cfg!(not(any(target_os = "macos", target_os = "linux"))) { + return Err(LookupError::Unsupported); + } + match &self.gateway_override { + // An override replaces discovery only: the routing table is not read, but the + // interface checks are the same. + Some(o) => validate_override(o, &interfaces()?, &kernel_source).map(Some), + None => { + let routes = os::default_routes()?; + select(&routes, &interfaces()?, &kernel_source) + } + } + } +} + +/// The host's interfaces, grouped by name, from `getifaddrs`. +pub fn interfaces() -> Result, LookupError> { + let raw = if_addrs::get_if_addrs().map_err(LookupError::Os)?; + let mut out: Vec = Vec::new(); + for a in raw { + let pos = match out.iter().position(|i| i.name == a.name) { + Some(p) => p, + None => { + out.push(IfInfo { + name: a.name.clone(), + index: a.index.unwrap_or(0), + up: true, + loopback: false, + point_to_point: false, + v4: Vec::new(), + }); + out.len() - 1 + } + }; + let i = &mut out[pos]; + i.up &= a.is_oper_up(); + i.loopback |= a.is_loopback(); + i.point_to_point |= a.is_p2p(); + if let if_addrs::IfAddr::V4(v4) = &a.addr { + i.v4.push((v4.ip, v4.prefixlen)); + } + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn ip(s: &str) -> Ipv4Addr { + s.parse().unwrap() + } + + fn iface(name: &str, index: u32, addrs: &[(&str, u8)]) -> IfInfo { + IfInfo { + name: name.to_owned(), + index, + up: true, + loopback: false, + point_to_point: false, + v4: addrs.iter().map(|(a, p)| (ip(a), *p)).collect(), + } + } + + fn route(gw: &str, index: u32, metric: u32) -> DefaultRoute { + DefaultRoute { + gw: ip(gw), + if_index: Some(index), + if_name: None, + metric, + ifa: None, + } + } + + fn routes(v4: Vec) -> DefaultRoutes { + DefaultRoutes { + v4, + has_v6_default: false, + } + } + + /// A kernel source lookup that must not be consulted. + fn no_kernel(_: Ipv4Addr) -> Option { + panic!("kernel source selection consulted unexpectedly") + } + + /// Adapted from Go's `TestLikelyHomeRouterIP` (`net/netmon/interfaces_test.go`). + #[test] + fn likely_home_router_ip() { + let mut down = iface("down0", 1, &[("10.0.0.100", 8)]); + down.up = false; + let ifs = vec![ + down, + iface("ipsixonly0", 2, &[]), + iface("fake0", 3, &[("192.168.7.100", 24)]), + ]; + let got = select(&routes(vec![route("192.168.7.1", 3, 0)]), &ifs, &no_kernel) + .unwrap() + .unwrap(); + assert_eq!(got.gw, ip("192.168.7.1")); + assert_eq!(got.self_ip, ip("192.168.7.100")); + assert_eq!( + (got.prefix_len, got.if_index, got.if_name.as_str()), + (24, 3, "fake0") + ); + + // NoIPv4Addrs: the route's interface has only IPv6 addresses. + let ifs = vec![iface("en0", 2, &[])]; + assert!(matches!( + select(&routes(vec![route("192.168.7.1", 2, 0)]), &ifs, &no_kernel), + Err(LookupError::NotOnLink) + )); + } + + /// Adapted from Go's `TestLikelyHomeRouterIP_Prefix` (tailscale/tailscale#10466): an earlier + /// interface without the gateway on-link is never picked. + #[test] + fn likely_home_router_ip_prefix() { + let ifs = vec![ + iface("docker0", 1, &[("172.17.0.0", 16)]), + iface("fake0", 2, &[("192.168.7.100", 24)]), + ]; + let got = select(&routes(vec![route("192.168.7.1", 2, 0)]), &ifs, &no_kernel) + .unwrap() + .unwrap(); + assert_eq!(got.self_ip, ip("192.168.7.100")); + assert_eq!(got.if_name, "fake0"); + } + + #[test] + fn no_default_route_is_offline() { + assert!(matches!(select(&routes(vec![]), &[], &no_kernel), Ok(None))); + } + + /// An IPv6-only default is a typed error. + #[test] + fn ipv6_only_default() { + let r = DefaultRoutes { + v4: vec![], + has_v6_default: true, + }; + assert!(matches!( + select(&r, &[], &no_kernel), + Err(LookupError::GatewayIpv6) + )); + } + + /// A public best default is `NotPrivate`, never a fall-through to the private one. + #[test] + fn public_best_route_does_not_fall_through() { + let ifs = vec![ + iface("en0", 1, &[("203.0.113.5", 24)]), + iface("en1", 2, &[("192.168.1.20", 24)]), + ]; + let r = routes(vec![ + route("203.0.113.1", 1, 0), + route("192.168.1.1", 2, 100), + ]); + assert!(matches!( + select(&r, &ifs, &no_kernel), + Err(LookupError::NotPrivate) + )); + // CGNAT and link-local gateways are not private either. + for gw in ["100.64.0.1", "169.254.1.1"] { + let r = routes(vec![route(gw, 2, 0)]); + assert!(matches!( + select(&r, &ifs, &no_kernel), + Err(LookupError::NotPrivate) + )); + } + } + + #[test] + fn lowest_metric_wins_and_ties_are_ambiguous() { + let ifs = vec![ + iface("eth0", 1, &[("192.168.1.20", 24)]), + iface("eth1", 2, &[("10.0.0.20", 24)]), + ]; + let r = routes(vec![ + route("10.0.0.1", 2, 600), + route("192.168.1.1", 1, 100), + ]); + let got = select(&r, &ifs, &no_kernel).unwrap().unwrap(); + assert_eq!(got.gw, ip("192.168.1.1")); + + let tie = routes(vec![ + route("10.0.0.1", 2, 100), + route("192.168.1.1", 1, 100), + ]); + assert!(matches!( + select(&tie, &ifs, &no_kernel), + Err(LookupError::Ambiguous) + )); + // The same route listed twice is not a tie. + let dup = routes(vec![ + route("192.168.1.1", 1, 100), + route("192.168.1.1", 1, 100), + ]); + assert!(select(&dup, &ifs, &no_kernel).unwrap().is_some()); + } + + /// Unusable interfaces are typed errors. + #[test] + fn unusable_interfaces() { + let r = routes(vec![route("192.168.1.1", 1, 0)]); + let mut p2p = iface("utun3", 1, &[("192.168.1.20", 24)]); + p2p.point_to_point = true; + let mut down = iface("en0", 1, &[("192.168.1.20", 24)]); + down.up = false; + let mut lo = iface("lo0", 1, &[("192.168.1.20", 24)]); + lo.loopback = true; + for i in [p2p, down, lo] { + assert!(matches!( + select(&r, &[i], &no_kernel), + Err(LookupError::InterfaceUnusable) + )); + } + // The interface vanished between the dump and the address scan. + let gone = vec![iface("en1", 2, &[("192.168.1.20", 24)])]; + assert!(matches!( + select(&r, &gone, &no_kernel), + Err(LookupError::InterfaceUnusable) + )); + // A by-name route (Linux) resolves by name. + let mut by_name = route("192.168.1.1", 0, 0); + by_name.if_index = None; + by_name.if_name = Some("en1".to_owned()); + assert!( + select(&routes(vec![by_name]), &gone, &no_kernel) + .unwrap() + .is_some() + ); + } + + #[test] + fn overlapping_subnets_use_the_route_interface() { + let ifs = vec![ + iface("en0", 1, &[("192.168.1.10", 24)]), + iface("en1", 2, &[("192.168.1.20", 24)]), + ]; + let got = select(&routes(vec![route("192.168.1.1", 2, 0)]), &ifs, &no_kernel) + .unwrap() + .unwrap(); + assert_eq!((got.self_ip, got.if_index), (ip("192.168.1.20"), 2)); + } + + /// Self-IP selection with several candidates. + #[test] + fn self_ip_selection() { + let ifs = vec![iface( + "en0", + 1, + &[("192.168.1.20", 24), ("192.168.1.21", 24), ("10.9.0.2", 16)], + )]; + // The route's RTAX_IFA wins when it is a candidate. + let mut r = route("192.168.1.1", 1, 0); + r.ifa = Some(ip("192.168.1.21")); + let got = select(&routes(vec![r.clone()]), &ifs, &no_kernel) + .unwrap() + .unwrap(); + assert_eq!(got.self_ip, ip("192.168.1.21")); + // An RTAX_IFA that is not on the gateway's prefix is ignored; the kernel decides. + r.ifa = Some(ip("10.9.0.2")); + let kernel = |_| Some(ip("192.168.1.20")); + let got = select(&routes(vec![r.clone()]), &ifs, &kernel) + .unwrap() + .unwrap(); + assert_eq!(got.self_ip, ip("192.168.1.20")); + // The kernel picks an address that is not a candidate: ambiguous. + let disagree = |_| Some(ip("10.9.0.2")); + assert!(matches!( + select(&routes(vec![r.clone()]), &ifs, &disagree), + Err(LookupError::Ambiguous) + )); + assert!(matches!( + select(&routes(vec![r]), &ifs, &|_| None), + Err(LookupError::Ambiguous) + )); + } + + #[test] + fn self_ip_must_be_private() { + // 192.0.2.5/8 contains the private gateway 192.168.1.1, but 192.0.2.5 is not private. + let ifs = vec![iface("en0", 1, &[("192.0.2.5", 8)])]; + assert!(matches!( + select(&routes(vec![route("192.168.1.1", 1, 0)]), &ifs, &no_kernel), + Err(LookupError::NotPrivate) + )); + } + + /// A self address with a /0../7 or /32 prefix never makes a gateway "on-link". + #[test] + fn implausible_prefixes_are_not_on_link() { + for p in [0, 1, 7, 32] { + let ifs = vec![iface("en0", 1, &[("192.168.1.20", p)])]; + assert!( + matches!( + select(&routes(vec![route("10.0.0.1", 1, 0)]), &ifs, &no_kernel), + Err(LookupError::NotOnLink) + ), + "/{p}" + ); + } + let ifs = vec![iface("en0", 1, &[("10.1.2.3", 8)])]; + assert!(select(&routes(vec![route("10.0.0.1", 1, 0)]), &ifs, &no_kernel).is_ok()); + } + + /// The override goes through the same validation. + #[test] + fn override_validation() { + let ifs = vec![ + iface("en0", 4, &[("192.168.1.20", 24)]), + iface("en1", 5, &[("10.0.0.20", 8)]), + ]; + let o = |s: &str| s.parse::().unwrap(); + let got = validate_override(&o("192.168.1.1"), &ifs, &no_kernel).unwrap(); + assert_eq!((got.self_ip, got.if_index), (ip("192.168.1.20"), 4)); + let got = validate_override(&o("10.0.0.1,10.0.0.20"), &ifs, &no_kernel).unwrap(); + assert_eq!(got.if_name, "en1"); + // Public gateway: rejected. + assert!(matches!( + validate_override(&o("203.0.113.1"), &ifs, &no_kernel), + Err(LookupError::NotPrivate) + )); + // Not on any interface, or the given self address does not hold it. + assert!(matches!( + validate_override(&o("172.16.0.1"), &ifs, &no_kernel), + Err(LookupError::NotOnLink) + )); + assert!(matches!( + validate_override(&o("192.168.1.1,10.0.0.20"), &ifs, &no_kernel), + Err(LookupError::NotOnLink) + )); + // On-link only through a VPN interface. + let mut vpn = iface("utun3", 9, &[("172.16.0.2", 24)]); + vpn.point_to_point = true; + assert!(matches!( + validate_override(&o("172.16.0.1"), &[vpn], &no_kernel), + Err(LookupError::InterfaceUnusable) + )); + assert!("192.168.1.1,".parse::().is_err()); + assert!("x".parse::().is_err()); + } + + #[test] + fn prefix_math() { + assert!(prefix_contains(ip("192.168.1.20"), 24, ip("192.168.1.1"))); + assert!(!prefix_contains(ip("192.168.1.20"), 24, ip("192.168.2.1"))); + assert!(prefix_contains(ip("192.168.1.20"), 0, ip("203.0.113.9"))); + assert!(prefix_contains(ip("192.168.1.20"), 32, ip("192.168.1.20"))); + assert!(!prefix_contains(ip("192.168.1.20"), 33, ip("192.168.1.20"))); + } + + /// Go's `TestLikelyHomeRouterIP_NoMocks`: the real interface scan works. A local read only; + /// no packet is sent. + #[test] + fn interfaces_no_mocks() { + let ifs = interfaces().unwrap(); + assert!(ifs.iter().any(|i| i.loopback), "{ifs:?}"); + } + + /// The real lookup runs end to end (routing-table read, interface scan, and at most a + /// never-sending `connect()`); its result depends on the host, so it is only printed. + #[test] + fn system_lookup_no_mocks() { + let got = SystemLookup::new(None).lookup(); + println!("SystemLookup: {got:?}"); + if cfg!(any(target_os = "macos", target_os = "linux")) { + assert!(!matches!(got, Err(LookupError::Unsupported))); + } + } +} diff --git a/ts_portmapper/src/gateway/other.rs b/ts_portmapper/src/gateway/other.rs new file mode 100644 index 00000000..72c0dbfc --- /dev/null +++ b/ts_portmapper/src/gateway/other.rs @@ -0,0 +1,9 @@ +//! Platforms without a gateway lookup: every lookup is `Unsupported`, so the port mapper never +//! sends anything there, but the crate still builds. + +use super::{DefaultRoutes, LookupError}; + +/// Always [`LookupError::Unsupported`]. +pub(super) fn default_routes() -> Result { + Err(LookupError::Unsupported) +} diff --git a/ts_portmapper/src/lib.rs b/ts_portmapper/src/lib.rs new file mode 100644 index 00000000..6ccf1bc4 --- /dev/null +++ b/ts_portmapper/src/lib.rs @@ -0,0 +1,116 @@ +//! NAT port mapping for the node's underlay UDP socket: NAT-PMP (RFC 6886) and PCP (RFC 6887). +//! +//! The port mapper asks the LAN gateway for an inbound UDP mapping so that a peer behind a hard +//! NAT can reach this node directly instead of through DERP. It is modelled on Go Tailscale's +//! `net/portmapper`. UPnP-IGD is not implemented. +//! +//! # LAN boundary +//! +//! Every packet this crate sends goes through a [`GatewayCap`], a capability built from a +//! validated [`Gateway`]: the gateway must be private (RFC 1918) and on-link, sockets are bound +//! to the host's own address and, where the OS allows it, to its interface (see [`BindMode`]), the +//! only destination is `gateway:5351` ([`LanDest::Pxp`]), and a [`PolicyHandle`] is checked before +//! every send. No API in this crate takes an arbitrary destination address. (The gateway lookup's +//! kernel source-address check connects a UDP socket but never sends on it.) +//! +//! # Layout +//! +//! - [`gateway`]: default-route lookup and validation (macOS routing-socket dump, Linux +//! `/proc/net/route`, `Unsupported` elsewhere). +//! - [`pmp`] and [`pcp`]: pure, strictly validating codecs. +//! - [`oneshot`]: single-attempt probe, map and delete. +//! - [`probe_tool`]: the `--portmap-probe` diagnostic used to validate a deployment. +//! - `testing` (feature `test-harness`): a fake NAT-PMP/PCP router and `TestCap`, a loopback +//! transport that production code cannot construct. + +mod blocking; +mod cap; +pub mod gateway; +pub mod oneshot; +pub mod pcp; +pub mod pmp; +pub mod probe_tool; +#[cfg(any(test, feature = "test-harness"))] +pub mod testing; + +/// Without the `test-harness` feature the test transport does not exist, so production code +/// cannot construct it: +#[cfg_attr( + not(feature = "test-harness"), + doc = "```compile_fail\nlet _ = ts_portmapper::testing::TestCap::new;\n```" +)] +#[cfg_attr( + feature = "test-harness", + doc = "(The compile-fail check runs in builds without the feature.)" +)] +pub const TEST_HARNESS_ENABLED: bool = cfg!(feature = "test-harness"); + +pub use cap::{ + BindMode, CapError, DenyReason, EXCHANGE_TIMEOUT, ExchangeStats, Flow, GatewayCap, LanDest, + MAX_EXAMINED, NetGen, PolicyHandle, ProbeListener, PxpTransport, RECV_BUF, errno_name, +}; +pub use gateway::{Gateway, GatewayLookup, LookupError}; + +/// The UDP port NAT-PMP and PCP servers listen on (RFC 6886 §3, RFC 6887 §19.1). +pub const PXP_PORT: u16 = 5351; + +/// Whether `ip` can be a router's external address at all. The codecs reject anything else as +/// malformed: unspecified, `0.0.0.0/8`, loopback, link-local, multicast, and `240.0.0.0/4` +/// (which includes the limited broadcast address). Private (RFC 1918) and shared (`100.64.0.0/10`) +/// addresses are plausible: they are a double NAT's honest answer and worth reporting. +pub fn is_plausible_external(ip: std::net::Ipv4Addr) -> bool { + let [a, ..] = ip.octets(); + !(a == 0 || ip.is_loopback() || ip.is_link_local() || ip.is_multicast() || a >= 240) +} + +/// Whether `ip` may be advertised to peers as a reachable endpoint: plausible +/// ([`is_plausible_external`]) and neither private (RFC 1918, a double NAT) nor shared +/// (`100.64.0.0/10`: carrier-grade NAT, and the tailnet's own range). +pub fn is_advertisable(ip: std::net::Ipv4Addr) -> bool { + let [a, b, ..] = ip.octets(); + let shared = a == 100 && (64..128).contains(&b); + is_plausible_external(ip) && !ip.is_private() && !shared +} + +#[cfg(test)] +mod tests { + use std::net::Ipv4Addr; + + use super::*; + + #[test] + fn external_address_classes() { + let ip = |s: &str| s.parse::().unwrap(); + for bad in [ + "0.0.0.0", + "0.1.2.3", + "127.0.0.1", + "169.254.1.1", + "224.0.0.1", + "239.255.255.250", + "240.0.0.1", + "255.255.255.255", + ] { + assert!(!is_plausible_external(ip(bad)), "{bad}"); + assert!(!is_advertisable(ip(bad)), "{bad}"); + } + for double_nat in [ + "10.0.0.1", + "172.16.0.1", + "192.168.1.1", + "100.64.0.0", + "100.127.255.255", + ] { + assert!(is_plausible_external(ip(double_nat)), "{double_nat}"); + assert!(!is_advertisable(ip(double_nat)), "{double_nat}"); + } + for public in ["203.0.113.7", "198.51.100.9", "192.0.2.1"] { + assert!(is_advertisable(ip(public)), "{public}"); + } + // The shared range is exactly 100.64.0.0/10. + for b in 0..=255u8 { + let shared = (64..128).contains(&b); + assert_eq!(!is_advertisable(Ipv4Addr::new(100, b, 0, 1)), shared, "{b}"); + } + } +} diff --git a/ts_portmapper/src/oneshot.rs b/ts_portmapper/src/oneshot.rs new file mode 100644 index 00000000..694e2acf --- /dev/null +++ b/ts_portmapper/src/oneshot.rs @@ -0,0 +1,1124 @@ +//! The minimal one-shot client: a single probe, map or delete over a [`PxpTransport`], with no +//! state machine, no retries and no caching. +//! +//! The lifecycle client performs its I/O through these same functions, so there is only +//! one implementation of the wire exchanges. Each call is one exchange against an absolute +//! [`EXCHANGE_TIMEOUT`] deadline. Replies that do not validate are ignored (and remembered for +//! diagnostics) until the deadline. A router error ends the exchange at once only when it is tied +//! to our request: a NAT-PMP MAP reply for our internal port, or a PCP response that echoes our +//! nonce. A nonce-less PCP error (or a NAT-PMP "unsupported version" answer to PCP) is only a +//! fallback, reported if nothing better arrives by the deadline. + +use std::{ + fmt, + net::{Ipv4Addr, SocketAddrV4}, + num::{NonZeroU16, NonZeroU32}, + time::Duration, +}; + +use tokio::time::Instant; + +use crate::{ + cap::{CapError, EXCHANGE_TIMEOUT, ExchangeStats, Flow, LanDest, PxpTransport}, + pcp, pmp, +}; + +/// At most this many distinct malformed-reply reasons are kept per exchange. +const MAX_NOTES: usize = 4; +/// The longest error lifetime taken from a PCP error that carries no nonce (anyone on the LAN +/// could have sent it), in seconds. +pub const MAX_UNVERIFIED_ERROR_LIFETIME: u32 = 1800; + +/// A port-mapping protocol. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Proto { + /// NAT-PMP (RFC 6886). + Pmp, + /// PCP (RFC 6887). + Pcp, +} + +impl fmt::Display for Proto { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::Pmp => "pmp", + Self::Pcp => "pcp", + }) + } +} + +/// Which protocols to exercise. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ProtoSet { + /// NAT-PMP. + pub pmp: bool, + /// PCP. + pub pcp: bool, +} + +impl ProtoSet { + /// Both protocols. + pub const BOTH: Self = Self { + pmp: true, + pcp: true, + }; + + /// The selected protocols, NAT-PMP first. + pub fn iter(self) -> impl Iterator { + [(self.pmp, Proto::Pmp), (self.pcp, Proto::Pcp)] + .into_iter() + .filter_map(|(on, p)| on.then_some(p)) + } +} + +/// What a probe learned. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ProbeReport { + /// A NAT-PMP server answered the public-address request with success. + pub pmp: bool, + /// The NAT-PMP epoch from the reply (success or error). + pub pmp_epoch: Option, + /// The public address NAT-PMP reported. + pub pmp_public_ip: Option, + /// The NAT-PMP result code, if a reply arrived. + pub pmp_result: Option, + /// A PCP server answered ANNOUNCE with success. `NOT_AUTHORIZED`, `ADDRESS_MISMATCH` and + /// other codes mean a PCP server exists but will not map for us. + pub pcp: bool, + /// The PCP ANNOUNCE result code, if a reply arrived. + pub pcp_result: Option, + /// The PCP epoch from the reply. + pub pcp_epoch: Option, + /// Malformed replies and receive errors, verbatim. + pub errors: Vec, +} + +/// A mapping the router granted. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Mapping { + /// The protocol that created it. + pub proto: Proto, + /// Our internal port. + pub internal: NonZeroU16, + /// The router's public address and assigned external port: never port 0, and always a + /// plausible external address ([`crate::is_plausible_external`]). It may still be private or + /// shared (a double NAT): check [`crate::is_advertisable`] before offering it to peers. + pub external: SocketAddrV4, + /// The granted lifetime in seconds (never 0). May differ from the requested lifetime. + pub lifetime: u32, + /// The router's epoch in the reply. + pub epoch: u32, + /// The PCP nonce that owns the mapping (renewals and the delete must reuse it). + pub nonce: Option, +} + +impl Mapping { + /// What it takes to delete this mapping. + pub fn rule(&self) -> Rule { + match &self.nonce { + Some(nonce) if self.proto == Proto::Pcp => Rule::Pcp { + internal: self.internal, + nonce: nonce.clone(), + }, + _ => Rule::Pmp { + internal: self.internal, + }, + } + } + + /// When the router's grant ends, counted from `granted_at`. + pub fn expires_at(&self, granted_at: Instant) -> Instant { + lease_deadline(granted_at, self.lifetime) + } +} + +/// `now + secs`, covering the full wire lifetime range without arithmetic overflow. +pub fn lease_deadline(now: Instant, secs: u32) -> Instant { + let d = Duration::from_secs(u64::from(secs)); + now.checked_add(d).unwrap_or(now) +} + +/// Enough to delete a rule, even one whose create reply was lost: NAT-PMP needs only the +/// internal port, PCP also the nonce the create was sent with. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Rule { + /// A NAT-PMP rule. + Pmp { + /// Internal port. + internal: NonZeroU16, + }, + /// A PCP rule. + Pcp { + /// Internal port. + internal: NonZeroU16, + /// The nonce it was created with. + nonce: pcp::Nonce, + }, +} + +impl Rule { + /// A new rule for `internal`; PCP gets a fresh nonce from the OS CSPRNG. + pub fn new(proto: Proto, internal: NonZeroU16) -> Result { + Ok(match proto { + Proto::Pmp => Self::Pmp { internal }, + Proto::Pcp => Self::Pcp { + internal, + nonce: pcp::Nonce::random()?, + }, + }) + } + + /// The rule's internal port. + pub fn internal(&self) -> NonZeroU16 { + match self { + Self::Pmp { internal } | Self::Pcp { internal, .. } => *internal, + } + } + + /// The rule's protocol. + pub fn proto(&self) -> Proto { + match self { + Self::Pmp { .. } => Proto::Pmp, + Self::Pcp { .. } => Proto::Pcp, + } + } +} + +/// Why a map or delete failed. +#[derive(Debug, thiserror::Error)] +pub enum MapError { + /// The capability refused or the OS failed (verbatim). + #[error("{0}")] + Cap(CapError), + /// No acceptable reply before the deadline. + #[error("no port-mapping service answered{}", .malformed.as_ref().map(|m| format!(" (ignored: {m})")).unwrap_or_default())] + NoServices { + /// The last malformed or receive problem seen, if any. + malformed: Option, + }, + /// A NAT-PMP error reply to the MAP request itself (or, for a PCP rule, a NAT-PMP server's + /// "unsupported version" answer). + #[error("NAT-PMP {code} (epoch {epoch})")] + Pmp { + /// Result code. + code: pmp::ResultCode, + /// Router epoch. + epoch: u32, + }, + /// An identity-free NAT-PMP error; it does not prove a MAP was refused. + #[error("NAT-PMP {code} without request identity (epoch {epoch})")] + PmpUnverified { + /// Result code. + code: pmp::ResultCode, + /// Claimed router epoch. + epoch: u32, + }, + /// A PCP error reply; `lifetime` is how long the error stays current. + #[error("PCP {code} (error lifetime {lifetime} s, epoch {epoch})")] + Pcp { + /// Result code. + code: pcp::ResultCode, + /// Error lifetime in seconds. + lifetime: u32, + /// Router epoch. + epoch: u32, + }, + /// A NAT-PMP error reply to the public-address request sent alongside the MAP. The MAP was + /// sent too and may have been committed (MiniUPnPd adds the redirect without checking the WAN + /// address), so the rule must be treated as possibly existing. `map` is the MAP's success + /// reply, if one arrived. + #[error("NAT-PMP public address: {code} (epoch {epoch}); the MAP may exist")] + PmpAddr { + /// Result code of the public-address reply. + code: pmp::ResultCode, + /// Router epoch. + epoch: u32, + /// The MAP's success reply, if it arrived before the deadline. + map: Option, + }, + /// A header-only PCP error that arrived with no nonce-matched reply by the deadline. It is not + /// tied to our request (anyone on the LAN could have sent it), so the MAP may still have been + /// committed. `lifetime` is capped at [`MAX_UNVERIFIED_ERROR_LIFETIME`]. + #[error("PCP {code} without a nonce (error lifetime {lifetime} s, epoch {epoch})")] + PcpUnverified { + /// Result code. + code: pcp::ResultCode, + /// Error lifetime in seconds, capped. + lifetime: u32, + /// Router epoch. + epoch: u32, + }, + /// The OS random source failed while making a PCP nonce. + #[error("{0}")] + Nonce(pcp::NonceError), +} + +impl From for MapError { + fn from(e: CapError) -> Self { + Self::Cap(e) + } +} + +/// Remember a malformed reply or receive problem, bounded. +fn note(notes: &mut Vec, what: String) { + if notes.len() < MAX_NOTES && !notes.contains(&what) { + notes.push(what); + } +} + +fn finish_notes(stats: &ExchangeStats, notes: &mut Vec) { + if let Some(e) = &stats.recv_error { + note(notes, e.to_string()); + } + if stats.oversize > 0 { + note(notes, format!("{} oversize datagram(s)", stats.oversize)); + } + if stats.wrong_source > 0 { + note( + notes, + format!( + "{} datagram(s) from a source other than the gateway", + stats.wrong_source + ), + ); + } + if stats.hit_cap { + note(notes, "too many datagrams; stopped reading".to_owned()); + } +} + +fn deadline() -> Instant { + Instant::now() + EXCHANGE_TIMEOUT +} + +/// The outcome of one datagram in a PCP MAP or delete exchange: an authoritative answer, a +/// fallback, or nothing. +enum PcpSeen { + Answer(Result), + Fallback(MapError), + Ignored, +} + +/// Classify one reply to a PCP MAP or delete sent with `nonce` for `internal`. +fn pcp_seen( + b: &[u8], + nonce: &pcp::Nonce, + internal: NonZeroU16, + kind: pcp::MapKind, + ok: impl FnOnce(pcp::MapReply) -> T, + notes: &mut Vec, +) -> PcpSeen { + match b.first() { + Some(&pcp::VERSION) => match pcp::parse_map_reply(b, nonce, internal, kind) { + Ok(r) => PcpSeen::Answer(Ok(ok(r))), + Err(pcp::ReplyError::Router { + code, + lifetime, + epoch, + }) => PcpSeen::Answer(Err(MapError::Pcp { + code, + lifetime, + epoch, + })), + Err(pcp::ReplyError::Unmatched { + code, + lifetime, + epoch, + }) => PcpSeen::Fallback(MapError::PcpUnverified { + code, + lifetime: lifetime.min(MAX_UNVERIFIED_ERROR_LIFETIME), + epoch, + }), + Err(pcp::ReplyError::Malformed(m)) => { + note(notes, format!("pcp: {m}")); + PcpSeen::Ignored + } + }, + // A NAT-PMP-only router answers a PCP request with "unsupported version". It carries no + // nonce either, so it is a fallback too. + Some(&pmp::VERSION) => match pmp::reply_result(b) { + Some((code @ pmp::ResultCode::UnsupportedVersion, epoch)) => { + PcpSeen::Fallback(MapError::PmpUnverified { code, epoch }) + } + _ => { + note(notes, "pcp: unexpected NAT-PMP reply".to_owned()); + PcpSeen::Ignored + } + }, + _ => { + note(notes, "pcp: not a PCP reply".to_owned()); + PcpSeen::Ignored + } + } +} + +/// Probe for NAT-PMP (public-address request) and PCP (ANNOUNCE) in one exchange from one +/// socket. Returns early once every requested protocol has a definitive answer. +pub async fn probe(t: &T, protos: ProtoSet) -> Result { + let mut report = ProbeReport::default(); + let pmp_req = pmp::public_address_request(); + let pcp_req = pcp::announce_request(t.gateway().self_ip); + let mut reqs: Vec<&[u8]> = Vec::new(); + if protos.pmp { + reqs.push(&pmp_req); + } + if protos.pcp { + reqs.push(&pcp_req); + } + if reqs.is_empty() { + return Ok(report); + } + let mut pmp_decided = !protos.pmp; + let mut pcp_decided = !protos.pcp; + let mut notes = Vec::new(); + let mut on_reply = |b: &[u8]| { + match b.first() { + Some(&pmp::VERSION) if protos.pmp => match pmp::parse_public_addr_reply(b) { + Ok(r) => { + report.pmp = true; + report.pmp_epoch = Some(r.epoch); + report.pmp_public_ip = Some(r.public_ip); + report.pmp_result = Some(pmp::ResultCode::Success); + pmp_decided = true; + } + Err( + pmp::ReplyError::Router { code, epoch } + | pmp::ReplyError::Unmatched { code, epoch }, + ) => { + if !report.pmp { + report.pmp_result = Some(code); + report.pmp_epoch = Some(epoch); + } + // A NAT-PMP-only router answers our PCP request with "unsupported + // version"; that says nothing about NAT-PMP itself. + if code != pmp::ResultCode::UnsupportedVersion { + pmp_decided = true; + } + } + Err(pmp::ReplyError::Malformed(m)) => note(&mut notes, format!("pmp: {m}")), + }, + Some(&pcp::VERSION) if protos.pcp => match pcp::parse_announce_reply(b) { + Ok(a) => { + report.pcp = a.result == pcp::ResultCode::Success; + report.pcp_result = Some(a.result); + report.pcp_epoch = Some(a.epoch); + // Any ANNOUNCE reply decides PCP (Go's early-exit rule). + pcp_decided = true; + } + Err(m) => note(&mut notes, format!("pcp: {m}")), + }, + Some(v) => note(&mut notes, format!("unexpected version {v}")), + None => note(&mut notes, "empty datagram".to_owned()), + } + if pmp_decided && pcp_decided { + Flow::Done + } else { + Flow::Continue + } + }; + let stats = t + .exchange(LanDest::Pxp, &reqs, deadline(), &mut on_reply) + .await?; + finish_notes(&stats, &mut notes); + report.errors = notes; + Ok(report) +} + +/// Ask for a UDP mapping for `rule` (its protocol, internal port and, for PCP, nonce) lasting +/// `lifetime` seconds. The rule is chosen before sending, so it can be deleted even if the reply +/// is lost. `prev`, a mapping this client holds for the same rule, makes it a renewal: its external +/// port is suggested. NAT-PMP always asks for the public address in the same exchange (one extra +/// datagram), so a renewed mapping never carries a stale WAN address. +pub async fn map( + t: &T, + rule: &Rule, + lifetime: NonZeroU32, + prev: Option<&Mapping>, +) -> Result { + let prev = prev.filter(|p| p.rule() == *rule); + match rule { + Rule::Pmp { internal } => map_pmp(t, *internal, lifetime, prev).await, + Rule::Pcp { internal, nonce } => map_pcp(t, *internal, lifetime, prev, nonce.clone()).await, + } +} + +async fn map_pmp( + t: &T, + internal: NonZeroU16, + lifetime: NonZeroU32, + prev: Option<&Mapping>, +) -> Result { + let suggested = prev.map_or(0, |p| p.external.port()); + let addr_req = pmp::public_address_request(); + let map_req = pmp::map_request(internal, suggested, lifetime); + let reqs: [&[u8]; 2] = [&addr_req, &map_req]; + + let mut public_ip: Option = None; + let mut granted: Option = None; + // An error to the MAP itself ends the exchange; an error to the address request does not, + // since the MAP was sent too and its reply says whether it was committed. + let mut map_refused: Option = None; + let mut fallback: Option = None; + let mut addr_error: Option<(pmp::ResultCode, u32)> = None; + let mut notes = Vec::new(); + let mut on_reply = |b: &[u8]| { + match b.get(..2) { + Some(&[pmp::VERSION, op]) if op == pmp::OP_PUBLIC_ADDR | pmp::OP_REPLY => { + match pmp::parse_public_addr_reply(b) { + Ok(r) => public_ip = Some(r.public_ip), + Err( + pmp::ReplyError::Router { code, epoch } + | pmp::ReplyError::Unmatched { code, epoch }, + ) => { + addr_error.get_or_insert((code, epoch)); + } + Err(pmp::ReplyError::Malformed(m)) => note(&mut notes, format!("pmp: {m}")), + } + } + Some(&[pmp::VERSION, _]) => { + match pmp::parse_map_reply(b, internal, pmp::MapKind::Map) { + Ok(r) => granted = Some(r), + Err(pmp::ReplyError::Router { code, epoch }) => { + map_refused = Some(MapError::Pmp { code, epoch }); + } + Err(pmp::ReplyError::Unmatched { code, epoch }) => { + fallback.get_or_insert(MapError::PmpUnverified { code, epoch }); + } + Err(pmp::ReplyError::Malformed(m)) => note(&mut notes, format!("pmp: {m}")), + } + } + _ => note(&mut notes, "pmp: not a NAT-PMP reply".to_owned()), + } + let answered = granted.is_some() && (public_ip.is_some() || addr_error.is_some()); + if map_refused.is_some() || answered { + Flow::Done + } else { + Flow::Continue + } + }; + let stats = t + .exchange(LanDest::Pxp, &reqs, deadline(), &mut on_reply) + .await?; + if let Some(f) = map_refused { + return Err(f); + } + finish_notes(&stats, &mut notes); + if let (Some((code, epoch)), None) = (addr_error, public_ip) { + return Err(MapError::PmpAddr { + code, + epoch, + map: granted, + }); + } + match (granted, public_ip) { + (Some(r), Some(ip)) => Ok(Mapping { + proto: Proto::Pmp, + internal, + external: SocketAddrV4::new(ip, r.external), + lifetime: r.lifetime, + epoch: r.epoch, + nonce: None, + }), + _ => Err(fallback.unwrap_or(MapError::NoServices { + malformed: notes.pop(), + })), + } +} + +async fn map_pcp( + t: &T, + internal: NonZeroU16, + lifetime: NonZeroU32, + prev: Option<&Mapping>, + nonce: pcp::Nonce, +) -> Result { + let suggested = prev.map_or(0, |p| p.external.port()); + let req = pcp::map_request(t.gateway().self_ip, internal, suggested, lifetime, &nonce); + let mut outcome: Option> = None; + let mut fallback: Option = None; + let mut notes = Vec::new(); + let mut on_reply = |b: &[u8]| { + match pcp_seen(b, &nonce, internal, pcp::MapKind::Map, |r| r, &mut notes) { + PcpSeen::Answer(a) => outcome = Some(a), + PcpSeen::Fallback(e) => { + fallback.get_or_insert(e); + } + PcpSeen::Ignored => {} + } + if outcome.is_some() { + Flow::Done + } else { + Flow::Continue + } + }; + let stats = t + .exchange(LanDest::Pxp, &[&req], deadline(), &mut on_reply) + .await?; + finish_notes(&stats, &mut notes); + match outcome { + Some(Ok(r)) => Ok(Mapping { + proto: Proto::Pcp, + internal, + external: r.external, + lifetime: r.lifetime, + epoch: r.epoch, + nonce: Some(nonce), + }), + Some(Err(e)) => Err(e), + None => Err(fallback.unwrap_or(MapError::NoServices { + malformed: notes.pop(), + })), + } +} + +/// The delete request for `rule`, sent from `self_ip`: NAT-PMP lifetime 0 with suggested port 0; +/// PCP lifetime 0 with the rule's original nonce. +pub fn delete_request_bytes(rule: &Rule, self_ip: Ipv4Addr) -> Vec { + match rule { + Rule::Pmp { internal } => pmp::delete_request(*internal).to_vec(), + Rule::Pcp { internal, nonce } => pcp::delete_request(self_ip, *internal, nonce).to_vec(), + } +} + +/// Delete `rule` and wait for the router's acknowledgement (lifetime 0). NAT-PMP sends lifetime 0 +/// with suggested port 0; PCP also sends the rule's original nonce. +pub async fn delete(t: &T, rule: &Rule) -> Result<(), MapError> { + let req = delete_request_bytes(rule, t.gateway().self_ip); + let internal = rule.internal(); + let mut outcome: Option> = None; + let mut fallback: Option = None; + let mut notes = Vec::new(); + let mut on_reply = |b: &[u8]| { + match rule { + Rule::Pmp { .. } => match pmp::parse_map_reply(b, internal, pmp::MapKind::Delete) { + Ok(_) => outcome = Some(Ok(())), + Err(pmp::ReplyError::Router { code, epoch }) => { + outcome = Some(Err(MapError::Pmp { code, epoch })); + } + Err(pmp::ReplyError::Unmatched { code, epoch }) => { + fallback.get_or_insert(MapError::PmpUnverified { code, epoch }); + } + Err(pmp::ReplyError::Malformed(m)) => note(&mut notes, format!("pmp: {m}")), + }, + Rule::Pcp { nonce, .. } => { + match pcp_seen(b, nonce, internal, pcp::MapKind::Delete, |_| (), &mut notes) { + PcpSeen::Answer(a) => outcome = Some(a), + PcpSeen::Fallback(e) => { + fallback.get_or_insert(e); + } + PcpSeen::Ignored => {} + } + } + } + if outcome.is_some() { + Flow::Done + } else { + Flow::Continue + } + }; + let stats = t + .exchange(LanDest::Pxp, &[&req], deadline(), &mut on_reply) + .await?; + finish_notes(&stats, &mut notes); + outcome.unwrap_or_else(|| { + Err(fallback.unwrap_or(MapError::NoServices { + malformed: notes.pop(), + })) + }) +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use super::*; + use crate::{ + Gateway, GatewayCap, PolicyHandle, + testing::{ + AssignPort, Corruption, FakeRouter, RequestKind, ResultOverride, RouterOptions, TestCap, + }, + }; + + const P1234: NonZeroU16 = NonZeroU16::new(1234).unwrap(); + const LIFE: NonZeroU32 = NonZeroU32::new(7200).unwrap(); + + fn gateway(self_ip: Ipv4Addr) -> Gateway { + Gateway { + gw: Ipv4Addr::new(192, 168, 1, 1), + self_ip, + prefix_len: 24, + if_index: 4, + if_name: "en0".to_owned(), + } + } + + fn rule(proto: Proto, internal: NonZeroU16) -> Rule { + Rule::new(proto, internal).unwrap() + } + + async fn setup(opts: RouterOptions) -> (FakeRouter, Arc) { + let router = FakeRouter::start(opts).await.unwrap(); + let cap = TestCap::new( + gateway(Ipv4Addr::LOCALHOST), + &router, + PolicyHandle::new(true), + ); + (router, cap) + } + + /// Port of Go's `TestProbeIntegration` (PxP half): one request of each kind, and both seen + /// (Rust's harness answers NAT-PMP; Go's does not). + #[tokio::test] + async fn probe_integration() { + let (router, cap) = setup(RouterOptions::default()).await; + let r = probe(&*cap, ProtoSet::BOTH).await.unwrap(); + assert!(r.pmp && r.pcp, "{r:?}"); + assert_eq!(r.pmp_public_ip, Some(Ipv4Addr::new(203, 0, 113, 7))); + assert_eq!(r.pmp_epoch, Some(3600)); + assert_eq!(r.pcp_result, Some(pcp::ResultCode::Success)); + let c = router.counters(); + assert_eq!( + ( + c.num_pmp_recv, + c.num_pmp_public_addr_recv, + c.num_pcp_recv, + c.num_pcp_disco_recv + ), + (1, 1, 1, 1) + ); + router.shutdown().await; + } + + /// Port of Go's `TestPCPIntegration` (the one-shot part): a PCP-only router; the probe sees + /// only PCP and a PCP map yields a valid external address. + #[tokio::test] + async fn pcp_integration() { + let (router, cap) = setup(RouterOptions { + pmp: false, + ..RouterOptions::default() + }) + .await; + let r = probe(&*cap, ProtoSet::BOTH).await.unwrap(); + assert!(!r.pmp && r.pcp, "{r:?}"); + let m = map(&*cap, &rule(Proto::Pcp, P1234), LIFE, None) + .await + .unwrap(); + assert!(!m.external.ip().is_unspecified() && m.external.port() != 0); + assert!(m.nonce.is_some()); + router.shutdown().await; + } + + /// A NAT-PMP-only router: its "unsupported version" answer to ANNOUNCE does not spoil PMP. + #[tokio::test] + async fn pmp_only_router() { + let (router, cap) = setup(RouterOptions { + pcp: false, + ..RouterOptions::default() + }) + .await; + let r = probe(&*cap, ProtoSet::BOTH).await.unwrap(); + assert!(r.pmp && !r.pcp, "{r:?}"); + assert_eq!(r.pmp_result, Some(pmp::ResultCode::Success)); + assert!( + map(&*cap, &rule(Proto::Pmp, P1234), LIFE, None) + .await + .is_ok() + ); + assert!(matches!( + map(&*cap, &rule(Proto::Pcp, P1234), LIFE, None).await, + Err(MapError::PmpUnverified { + code: pmp::ResultCode::UnsupportedVersion, + .. + }) + )); + router.shutdown().await; + } + + /// No responder: the probe returns all false within the exchange budget and a map + /// gives `NoServices`. + #[tokio::test] + async fn no_responder() { + let (router, cap) = setup(RouterOptions { + pmp: false, + pcp: false, + ..RouterOptions::default() + }) + .await; + let start = std::time::Instant::now(); + let r = probe(&*cap, ProtoSet::BOTH).await.unwrap(); + assert!(!r.pmp && !r.pcp && r.pmp_result.is_none() && r.pcp_result.is_none()); + assert!(start.elapsed() >= EXCHANGE_TIMEOUT); + assert!(start.elapsed() < Duration::from_secs(2)); + for proto in [Proto::Pmp, Proto::Pcp] { + assert!(matches!( + map(&*cap, &rule(proto, P1234), LIFE, None).await, + Err(MapError::NoServices { .. }) + )); + } + router.shutdown().await; + } + + #[tokio::test] + async fn pmp_map_renew_delete() { + let (router, cap) = setup(RouterOptions::default()).await; + router.set_assign_port(AssignPort::InternalPlus(1000)); + let m = map(&*cap, &rule(Proto::Pmp, P1234), LIFE, None) + .await + .unwrap(); + // External ≠ internal flows through. + assert_eq!(m.external, "203.0.113.7:2234".parse().unwrap()); + assert_eq!((m.lifetime, m.nonce.clone()), (7200, None)); + let c = router.counters(); + assert_eq!((c.num_pmp_public_addr_recv, c.num_pmp_map_recv), (1, 1)); + + // Renewal: the previous external port is suggested and the address is asked again. + let r = map(&*cap, &m.rule(), LIFE, Some(&m)).await.unwrap(); + assert_eq!(r.external, m.external); + let c = router.counters(); + assert_eq!((c.num_pmp_public_addr_recv, c.num_pmp_map_recv), (2, 2)); + assert_eq!( + router.requests().last().unwrap().suggested_external, + Some(2234) + ); + + delete(&*cap, &m.rule()).await.unwrap(); + let del = router.requests().last().unwrap().clone(); + assert_eq!( + (del.internal, del.suggested_external, del.lifetime), + (Some(1234), Some(0), Some(0)) + ); + assert!(router.mappings().is_empty()); + router.shutdown().await; + } + + #[tokio::test] + async fn pcp_map_renew_delete_reuse_the_nonce() { + let (router, cap) = setup(RouterOptions { + enforce_pcp_client_ip: true, + ..RouterOptions::default() + }) + .await; + let m = map(&*cap, &rule(Proto::Pcp, P1234), LIFE, None) + .await + .unwrap(); + let nonce = *m.nonce.as_ref().unwrap().as_bytes(); + let r = map(&*cap, &m.rule(), LIFE, Some(&m)).await.unwrap(); + assert_eq!(r.external, m.external); + let renew = router.requests().last().unwrap().clone(); + assert_eq!(renew.nonce, Some(nonce)); + assert_eq!(renew.suggested_external, Some(m.external.port())); + // The client IP field is self_ip (source-consistent loopback mode). + assert_eq!(renew.pcp_client_ip, Some(Ipv4Addr::LOCALHOST)); + + delete(&*cap, &m.rule()).await.unwrap(); + let del = router.requests().last().unwrap().clone(); + assert_eq!((del.nonce, del.lifetime), (Some(nonce), Some(0))); + assert!(router.mappings().is_empty()); + router.shutdown().await; + } + + /// A client IP field that differs from the source gets `ADDRESS_MISMATCH`. + #[tokio::test] + async fn pcp_address_mismatch() { + let router = FakeRouter::start(RouterOptions { + enforce_pcp_client_ip: true, + ..RouterOptions::default() + }) + .await + .unwrap(); + let cap = TestCap::new( + gateway(Ipv4Addr::new(198, 51, 100, 4)), + &router, + PolicyHandle::new(true), + ); + let r = probe(&*cap, ProtoSet::BOTH).await.unwrap(); + assert!(!r.pcp); + assert_eq!(r.pcp_result, Some(pcp::ResultCode::AddressMismatch)); + assert!(matches!( + map(&*cap, &rule(Proto::Pcp, P1234), LIFE, None).await, + Err(MapError::Pcp { + code: pcp::ResultCode::AddressMismatch, + .. + }) + )); + router.shutdown().await; + } + + /// A PCP reply with a wrong nonce followed by the correct one: the correct one wins. + #[tokio::test] + async fn wrong_nonce_then_correct_reply() { + let (router, cap) = setup(RouterOptions::default()).await; + router.corrupt_next(Corruption::WrongNonce); + router.duplicate_next(1); + let m = map(&*cap, &rule(Proto::Pcp, P1234), LIFE, None) + .await + .unwrap(); + assert_eq!(m.external.port(), 4242); + router.shutdown().await; + } + + /// Hostile replies end in `NoServices` with the reason kept, never a mapping. + #[tokio::test] + async fn hostile_replies() { + let (router, cap) = setup(RouterOptions::default()).await; + for (i, c) in [ + Corruption::Truncate, + Corruption::BadVersion, + Corruption::WrongOpcode, + Corruption::NoRBit, + Corruption::WrongNonce, + Corruption::WrongProto, + Corruption::WrongInternalPort, + Corruption::Garbage, + Corruption::Oversize, + ] + .into_iter() + .enumerate() + { + // A fresh internal port each time: the router committed the previous rule under + // another nonce. + let port = NonZeroU16::new(2000 + u16::try_from(i).unwrap()).unwrap(); + router.corrupt_next(c); + let r = map(&*cap, &rule(Proto::Pcp, port), LIFE, None).await; + assert!( + matches!(r, Err(MapError::NoServices { malformed: Some(_) })), + "{c:?}: {r:?}" + ); + } + router.reply_from_wrong_port_next(1); + let r = map(&*cap, &rule(Proto::Pmp, P1234), LIFE, None).await; + assert!(matches!(r, Err(MapError::NoServices { .. })), "{r:?}"); + router.shutdown().await; + } + + /// A NAT-PMP renewal carries the router's current WAN address, not the one from the create. + #[tokio::test] + async fn pmp_renewal_refreshes_the_public_ip() { + let (router, cap) = setup(RouterOptions::default()).await; + let mut m = map(&*cap, &rule(Proto::Pmp, P1234), LIFE, None) + .await + .unwrap(); + assert_eq!(m.external, "203.0.113.7:4242".parse().unwrap()); + router.set_public_ip(Ipv4Addr::new(198, 51, 100, 9)); + for _ in 0..3 { + m = map(&*cap, &m.rule(), LIFE, Some(&m)).await.unwrap(); + assert_eq!(m.external, "198.51.100.9:4242".parse().unwrap()); + } + router.shutdown().await; + } + + /// An error to the public-address request (here NETWORK_FAILURE, the WAN is down) is not a + /// refusal of the MAP: the MAP was sent alongside and committed, so the error says so. + #[tokio::test] + async fn pmp_address_error_is_not_a_map_refusal() { + let (router, cap) = setup(RouterOptions::default()).await; + router.set_result( + RequestKind::PmpPublicAddr, + Some(ResultOverride { + code: 3, + lifetime: 0, + }), + ); + let r = map(&*cap, &rule(Proto::Pmp, P1234), LIFE, None).await; + assert!( + matches!( + r, + Err(MapError::PmpAddr { + code: pmp::ResultCode::NetworkFailure, + map: Some(pmp::MapReply { external: 4242, .. }), + .. + }) + ), + "{r:?}" + ); + assert_eq!(router.mappings().len(), 1, "the MAP was committed"); + // The MAP's own refusal is still a plain `Pmp` error. + router.set_result(RequestKind::PmpPublicAddr, None); + router.set_result( + RequestKind::PmpMap, + Some(ResultOverride { + code: 2, + lifetime: 0, + }), + ); + let r = map(&*cap, &rule(Proto::Pmp, P1234), LIFE, None).await; + assert!(matches!(r, Err(MapError::Pmp { .. })), "{r:?}"); + router.shutdown().await; + } + + /// At the one-shot level, router errors are typed, with PCP error lifetimes. + #[tokio::test] + async fn router_errors_are_typed() { + let (router, cap) = setup(RouterOptions::default()).await; + for code in 1..=5u16 { + router.set_result( + RequestKind::PmpMap, + Some(ResultOverride { code, lifetime: 0 }), + ); + let r = map(&*cap, &rule(Proto::Pmp, P1234), LIFE, None).await; + assert!( + matches!(r, Err(MapError::Pmp { code: c, .. }) if u16::from(c) == code), + "{r:?}" + ); + } + // A full-length error echoing our nonce is authoritative and keeps its lifetime. + router.set_result( + RequestKind::PcpMap, + Some(ResultOverride { + code: 2, + lifetime: 86_400, + }), + ); + let r = map(&*cap, &rule(Proto::Pcp, P1234), LIFE, None).await; + assert!( + matches!( + r, + Err(MapError::Pcp { + code: pcp::ResultCode::NotAuthorized, + lifetime: 86_400, + .. + }) + ), + "{r:?}" + ); + // A header-only error has no nonce: reported only after the deadline, as unverified, + // with its lifetime capped. + router.set_short_pcp_errors(true); + let start = std::time::Instant::now(); + let r = map(&*cap, &rule(Proto::Pcp, P1234), LIFE, None).await; + assert!(start.elapsed() >= EXCHANGE_TIMEOUT); + assert!( + matches!( + r, + Err(MapError::PcpUnverified { + code: pcp::ResultCode::NotAuthorized, + lifetime: MAX_UNVERIFIED_ERROR_LIFETIME, + .. + }) + ), + "{r:?}" + ); + router.shutdown().await; + } + + /// A spoofed header-only PCP error from `gateway:5351` (here 24 bytes, NOT_AUTHORIZED, error + /// lifetime `u32::MAX`) that beats the real reply cannot end the exchange: the nonce-matched + /// reply that follows wins, for a map and for a delete. + #[tokio::test] + async fn spoofed_short_pcp_error_does_not_win() { + let (router, cap) = setup(RouterOptions::default()).await; + let mut spoof = vec![0u8; 24]; + spoof[0] = pcp::VERSION; + spoof[1] = pcp::R_BIT | pcp::OP_MAP; + spoof[3] = 2; + spoof[4..8].copy_from_slice(&u32::MAX.to_be_bytes()); + router.spoof_before_next_reply(spoof.clone()); + let rule = rule(Proto::Pcp, P1234); + let m = map(&*cap, &rule, LIFE, None).await.unwrap(); + assert_eq!(m.external.port(), 4242); + router.spoof_before_next_reply(spoof); + delete(&*cap, &rule).await.unwrap(); + assert!(router.mappings().is_empty()); + router.shutdown().await; + } + + /// The granted lifetime can differ from the requested one, both ways. + #[tokio::test] + async fn granted_lifetime_is_reported() { + let (router, cap) = setup(RouterOptions::default()).await; + for granted in [60, 86_400] { + router.set_lifetime_override(Some(granted)); + for proto in [Proto::Pmp, Proto::Pcp] { + let m = map(&*cap, &rule(proto, P1234), LIFE, None).await.unwrap(); + assert_eq!(m.lifetime, granted); + delete(&*cap, &m.rule()).await.unwrap(); + } + } + router.shutdown().await; + } + + /// Commit-then-lost-reply: the rule is still deletable from what the client sent. + #[tokio::test] + async fn lost_reply_rule_is_deletable() { + let (router, cap) = setup(RouterOptions::default()).await; + router.commit_then_drop_next(1); + let rule = rule(Proto::Pcp, P1234); + let r = map(&*cap, &rule, LIFE, None).await; + assert!(matches!(r, Err(MapError::NoServices { .. }))); + assert_eq!(router.mappings().len(), 1); + delete(&*cap, &rule).await.unwrap(); + assert!(router.mappings().is_empty()); + router.shutdown().await; + } + + /// Lease arithmetic never panics and is clamped. + #[test] + fn lease_deadlines() { + let now = Instant::now(); + assert_eq!(lease_deadline(now, 0), now); + assert_eq!(lease_deadline(now, 1), now + Duration::from_secs(1)); + assert_eq!( + lease_deadline(now, u32::MAX), + now + Duration::from_secs(u64::from(u32::MAX)) + ); + } + + /// The production `GatewayCap` runs the same probe/map/delete against a loopback router: the + /// real socket, interface binding and gate code, with no packet leaving the host. + #[tokio::test] + async fn real_gateway_cap_against_loopback_router() { + let router = FakeRouter::start(RouterOptions { + enforce_pcp_client_ip: true, + ..RouterOptions::default() + }) + .await + .unwrap(); + let cap = GatewayCap::loopback_for_test(router.addr().port(), PolicyHandle::new(true)) + .expect("loopback capability construction must succeed"); + let r = probe(&*cap, ProtoSet::BOTH).await.unwrap(); + assert!(r.pmp && r.pcp, "{r:?}"); + for proto in [Proto::Pmp, Proto::Pcp] { + let m = map(&*cap, &rule(proto, P1234), LIFE, None).await.unwrap(); + delete(&*cap, &m.rule()).await.unwrap(); + } + assert!(router.mappings().is_empty()); + router.shutdown().await; + } +} + +#[cfg(test)] +mod identity_error_tests { + use super::*; + use crate::testing::{FakeRouter, RequestKind, RouterOptions, TestCap}; + use crate::{Gateway, PolicyHandle}; + + #[tokio::test] + async fn pmp_unmatched_errors_do_not_beat_success() { + let router = FakeRouter::start(RouterOptions::default()).await.unwrap(); + let cap = TestCap::new( + Gateway { + gw: "192.168.1.1".parse().unwrap(), + self_ip: Ipv4Addr::LOCALHOST, + prefix_len: 24, + if_index: 4, + if_name: "en0".to_owned(), + }, + &router, + PolicyHandle::new(true), + ); + let internal = NonZeroU16::new(1234).unwrap(); + for short in [false, true] { + let mut error = vec![0, 0x81, 0, 2, 0, 0, 0, 1]; + if !short { + error.extend_from_slice(&[0x12, 0x34, 0, 0, 0, 0, 0, 0]); + } + router.knobs_for(Some(RequestKind::PmpMap)); + router.spoof_before_next_reply(error); + let rule = Rule::new(Proto::Pmp, internal).unwrap(); + let m = map(&*cap, &rule, NonZeroU32::new(7200).unwrap(), None).await; + assert!(m.is_ok(), "unmatched error beat success: {m:?}"); + delete(&*cap, &rule).await.unwrap(); + } + router.shutdown().await; + } +} diff --git a/ts_portmapper/src/pcp.rs b/ts_portmapper/src/pcp.rs new file mode 100644 index 00000000..d4df4f88 --- /dev/null +++ b/ts_portmapper/src/pcp.rs @@ -0,0 +1,1005 @@ +//! PCP codec (RFC 6887), IPv4 only. +//! +//! Pure functions over byte slices: no I/O, no panics (fuzzed). Every reply must carry version 2, +//! the R (response) bit, the request's opcode and a length that is at least 24, at most 1100 and +//! a multiple of 4. A MAP success must echo our nonce, protocol 17 and internal port, and assign +//! an IPv4-mapped, nonzero external address and port with a nonzero lifetime. A nonzero result +//! code in a full-length response that echoes our nonce is a typed [`ReplyError::Router`] that +//! carries the error lifetime (how long the client should treat the error as current, RFC 6887 +//! §7.4). A header-only (shorter than 60 bytes) error carries no nonce, so it cannot be tied to +//! our request: it is [`ReplyError::Unmatched`], which callers treat as a fallback only, never as +//! an answer that ends an exchange. +//! +//! ```compile_fail +//! // The internal port is a `NonZeroU16`: a request for internal port 0 ("all ports") cannot be +//! // built. +//! let nonce = ts_portmapper::pcp::Nonce::from_bytes([0; 12]); +//! let _ = ts_portmapper::pcp::map_request( +//! std::net::Ipv4Addr::LOCALHOST, 0, 0, std::num::NonZeroU32::MIN, &nonce); +//! ``` + +use std::{ + fmt, + net::{Ipv4Addr, SocketAddrV4}, + num::{NonZeroU16, NonZeroU32}, +}; + +/// PCP protocol version. +pub const VERSION: u8 = 2; +/// The R bit: set in responses, clear in requests. +pub const R_BIT: u8 = 0x80; +/// Opcode ANNOUNCE. +pub const OP_ANNOUNCE: u8 = 0; +/// Opcode MAP. +pub const OP_MAP: u8 = 1; +/// IANA protocol number for UDP, the only protocol we map. +pub const PROTO_UDP: u8 = 17; +/// Length of the common header. +pub const HEADER_LEN: usize = 24; +/// Length of a MAP request or response without options. +pub const MAP_LEN: usize = 60; +/// Largest PCP message (RFC 6887 §7). +pub const MAX_LEN: usize = 1100; + +/// A PCP result code (RFC 6887 §7.4). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ResultCode { + /// 0 + Success, + /// 1 + UnsuppVersion, + /// 2: PCP is implemented but refuses this client (mapping turned off). + NotAuthorized, + /// 3 + MalformedRequest, + /// 4 + UnsuppOpcode, + /// 5 + UnsuppOption, + /// 6 + MalformedOption, + /// 7 + NetworkFailure, + /// 8 + NoResources, + /// 9 + UnsuppProtocol, + /// 10 + UserExQuota, + /// 11 + CannotProvideExternal, + /// 12: the request's source differs from its client IP field (a NAT in between). + AddressMismatch, + /// 13 + ExcessiveRemotePeers, + /// Any other value. + Unknown(u8), +} + +impl From for ResultCode { + fn from(v: u8) -> Self { + match v { + 0 => Self::Success, + 1 => Self::UnsuppVersion, + 2 => Self::NotAuthorized, + 3 => Self::MalformedRequest, + 4 => Self::UnsuppOpcode, + 5 => Self::UnsuppOption, + 6 => Self::MalformedOption, + 7 => Self::NetworkFailure, + 8 => Self::NoResources, + 9 => Self::UnsuppProtocol, + 10 => Self::UserExQuota, + 11 => Self::CannotProvideExternal, + 12 => Self::AddressMismatch, + 13 => Self::ExcessiveRemotePeers, + n => Self::Unknown(n), + } + } +} + +impl From for u8 { + fn from(c: ResultCode) -> u8 { + match c { + ResultCode::Success => 0, + ResultCode::UnsuppVersion => 1, + ResultCode::NotAuthorized => 2, + ResultCode::MalformedRequest => 3, + ResultCode::UnsuppOpcode => 4, + ResultCode::UnsuppOption => 5, + ResultCode::MalformedOption => 6, + ResultCode::NetworkFailure => 7, + ResultCode::NoResources => 8, + ResultCode::UnsuppProtocol => 9, + ResultCode::UserExQuota => 10, + ResultCode::CannotProvideExternal => 11, + ResultCode::AddressMismatch => 12, + ResultCode::ExcessiveRemotePeers => 13, + ResultCode::Unknown(n) => n, + } + } +} + +impl fmt::Display for ResultCode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let name = match self { + Self::Success => "success", + Self::UnsuppVersion => "unsupp-version", + Self::NotAuthorized => "not-authorized", + Self::MalformedRequest => "malformed-request", + Self::UnsuppOpcode => "unsupp-opcode", + Self::UnsuppOption => "unsupp-option", + Self::MalformedOption => "malformed-option", + Self::NetworkFailure => "network-failure", + Self::NoResources => "no-resources", + Self::UnsuppProtocol => "unsupp-protocol", + Self::UserExQuota => "user-ex-quota", + Self::CannotProvideExternal => "cannot-provide-external", + Self::AddressMismatch => "address-mismatch", + Self::ExcessiveRemotePeers => "excessive-remote-peers", + Self::Unknown(_) => "unknown", + }; + write!(f, "{name}({})", u8::from(*self)) + } +} + +/// A MAP nonce (RFC 6887 §11.1). Its `Debug` output is redacted: nonces are never logged. +#[derive(Clone, PartialEq, Eq)] +pub struct Nonce([u8; 12]); + +/// The OS random source failed. +#[derive(Debug, thiserror::Error)] +#[error("OS random number generator failed: {0}")] +pub struct NonceError(getrandom::Error); + +impl Nonce { + /// A fresh nonce from the OS CSPRNG. + pub fn random() -> Result { + let mut b = [0u8; 12]; + getrandom::fill(&mut b).map_err(NonceError)?; + Ok(Self(b)) + } + + /// A nonce with the given bytes (tests, fuzzing and parsing only; use [`Nonce::random`] for + /// requests). + pub const fn from_bytes(b: [u8; 12]) -> Self { + Self(b) + } + + /// The nonce bytes. + pub const fn as_bytes(&self) -> &[u8; 12] { + &self.0 + } +} + +impl fmt::Debug for Nonce { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("Nonce(..)") + } +} + +/// Why a datagram is not an acceptable reply to the request it was matched against. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Malformed { + /// Shorter than required. + TooShort(usize), + /// Longer than 1100 bytes. + TooLong(usize), + /// Length not a multiple of 4. + NotMultipleOf4(usize), + /// Not version 2. + BadVersion(u8), + /// The R bit is clear (a request, not a response). + NoRBit, + /// Not the request's opcode. + WrongOpcode(u8), + /// A MAP response with another nonce (another client's or a spoofer's). + WrongNonce, + /// A MAP response for another protocol. + WrongProtocol(u8), + /// A MAP response for another internal port. + InternalPortMismatch { + /// Internal port in the reply. + got: u16, + /// Internal port we asked for. + want: u16, + }, + /// The assigned external address is not IPv4-mapped. + NotIpv4Mapped, + /// The assigned external address is `0.0.0.0`. + ZeroAddress, + /// An external address no router can have (see [`crate::is_plausible_external`]). + ImplausibleAddress(Ipv4Addr), + /// The assigned external port is 0. + ZeroExternalPort, + /// A success with lifetime 0 for a create or renew. + ZeroLifetime, + /// A reply to a delete that still carries a lifetime. + NonZeroLifetimeOnDelete(u32), + /// An option runs past the end of the message. + BadOption(usize), +} + +impl fmt::Display for Malformed { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::TooShort(n) => write!(f, "too short ({n} bytes)"), + Self::TooLong(n) => write!(f, "too long ({n} bytes)"), + Self::NotMultipleOf4(n) => write!(f, "length {n} is not a multiple of 4"), + Self::BadVersion(v) => write!(f, "version {v}, not 2"), + Self::NoRBit => f.write_str("R bit clear"), + Self::WrongOpcode(o) => write!(f, "unexpected opcode {o}"), + Self::WrongNonce => f.write_str("nonce mismatch"), + Self::WrongProtocol(p) => write!(f, "protocol {p}, not 17"), + Self::InternalPortMismatch { got, want } => { + write!(f, "internal port {got}, expected {want}") + } + Self::NotIpv4Mapped => f.write_str("external address not IPv4-mapped"), + Self::ZeroAddress => f.write_str("external address 0.0.0.0"), + Self::ImplausibleAddress(ip) => write!(f, "implausible external address {ip}"), + Self::ZeroExternalPort => f.write_str("external port 0"), + Self::ZeroLifetime => f.write_str("lifetime 0 on a create"), + Self::NonZeroLifetimeOnDelete(l) => write!(f, "lifetime {l} on a delete"), + Self::BadOption(at) => write!(f, "option at offset {at} overruns the message"), + } + } +} + +/// A MAP reply that is not a success. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +pub enum ReplyError { + /// A well-formed response with a nonzero result code. + #[error("PCP {code} (error lifetime {lifetime} s, epoch {epoch})")] + Router { + /// The result code. + code: ResultCode, + /// For how long the error should be considered current, in seconds. + lifetime: u32, + /// The server's epoch time. + epoch: u32, + }, + /// A header-only error response (shorter than a MAP response). It carries no nonce, so + /// anyone on the LAN who can send from `gateway:5351` could have forged it: callers must not + /// end an exchange on it, and must cap the error lifetime they act on. + #[error("PCP {code} without a nonce (error lifetime {lifetime} s, epoch {epoch})")] + Unmatched { + /// The result code. + code: ResultCode, + /// The claimed error lifetime, in seconds (unverified). + lifetime: u32, + /// The claimed epoch. + epoch: u32, + }, + /// Not an acceptable reply. + #[error("malformed PCP reply: {0}")] + Malformed(Malformed), +} + +impl From for ReplyError { + fn from(m: Malformed) -> Self { + Self::Malformed(m) + } +} + +/// A validated response common header. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Header { + /// The opcode without the R bit. + pub opcode: u8, + /// The result code. + pub result: ResultCode, + /// Lifetime in seconds (mapping lifetime, or error lifetime on failure). + pub lifetime: u32, + /// The server's epoch time. + pub epoch: u32, +} + +/// An ANNOUNCE response. Any result code is returned as data: `Success` means PCP is usable, +/// `NotAuthorized` and `AddressMismatch` mean a PCP server exists but won't help us. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct AnnounceReply { + /// The result code. + pub result: ResultCode, + /// The server's epoch time. + pub epoch: u32, +} + +/// A successful MAP response. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MapReply { + /// The server's epoch time. + pub epoch: u32, + /// Granted lifetime in seconds: nonzero for a create, 0 for a delete. + pub lifetime: u32, + /// The internal port; always the one requested. + pub internal: u16, + /// The assigned external address. For a delete reply it may be `0.0.0.0:0`. + pub external: SocketAddrV4, +} + +/// Whether a MAP request creates/renews a mapping or deletes it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MapKind { + /// Create or renew (lifetime > 0). + Map, + /// Delete (lifetime 0). + Delete, +} + +/// The IPv4-mapped IPv6 form of `ip` (`::ffff:a.b.c.d`). +const fn mapped(ip: Ipv4Addr) -> [u8; 16] { + let o = ip.octets(); + [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, o[0], o[1], o[2], o[3], + ] +} + +fn header_bytes(op: u8, lifetime: u32, client: Ipv4Addr) -> [u8; HEADER_LEN] { + let mut p = [0u8; HEADER_LEN]; + p[0] = VERSION; + p[1] = op; + p[4..8].copy_from_slice(&lifetime.to_be_bytes()); + p[8..24].copy_from_slice(&mapped(client)); + p +} + +/// An ANNOUNCE request from `client` (this host's own LAN address). +pub fn announce_request(client: Ipv4Addr) -> [u8; HEADER_LEN] { + header_bytes(OP_ANNOUNCE, 0, client) +} + +/// A MAP request for UDP `internal` on `client`, suggesting `suggested_external_port` (0 for +/// none) and the all-zeros external address `::ffff:0.0.0.0`. +pub fn map_request( + client: Ipv4Addr, + internal: NonZeroU16, + suggested_external_port: u16, + lifetime: NonZeroU32, + nonce: &Nonce, +) -> [u8; MAP_LEN] { + build_map( + client, + internal.get(), + suggested_external_port, + lifetime.get(), + nonce, + ) +} + +/// A MAP delete for UDP `internal` on `client`: lifetime 0, suggested port 0, suggested address +/// `::ffff:0.0.0.0`, and the mapping's **original** nonce (RFC 6887 §15.1, Errata 3621). +pub fn delete_request(client: Ipv4Addr, internal: NonZeroU16, nonce: &Nonce) -> [u8; MAP_LEN] { + build_map(client, internal.get(), 0, 0, nonce) +} + +fn build_map( + client: Ipv4Addr, + internal: u16, + suggested_port: u16, + lifetime: u32, + nonce: &Nonce, +) -> [u8; MAP_LEN] { + let mut p = [0u8; MAP_LEN]; + p[..HEADER_LEN].copy_from_slice(&header_bytes(OP_MAP, lifetime, client)); + p[24..36].copy_from_slice(nonce.as_bytes()); + p[36] = PROTO_UDP; + p[40..42].copy_from_slice(&internal.to_be_bytes()); + p[42..44].copy_from_slice(&suggested_port.to_be_bytes()); + p[44..60].copy_from_slice(&mapped(Ipv4Addr::UNSPECIFIED)); + p +} + +fn be16(b: &[u8], at: usize) -> u16 { + match b.get(at..at + 2) { + Some(&[x, y]) => u16::from_be_bytes([x, y]), + _ => 0, + } +} + +fn be32(b: &[u8], at: usize) -> u32 { + match b.get(at..at + 4) { + Some(&[w, x, y, z]) => u32::from_be_bytes([w, x, y, z]), + _ => 0, + } +} + +/// Validate a response's common header: length, version and R bit. +pub fn parse_header(b: &[u8]) -> Result { + if b.len() < HEADER_LEN { + return Err(Malformed::TooShort(b.len())); + } + if b.len() > MAX_LEN { + return Err(Malformed::TooLong(b.len())); + } + if !b.len().is_multiple_of(4) { + return Err(Malformed::NotMultipleOf4(b.len())); + } + if b[0] != VERSION { + return Err(Malformed::BadVersion(b[0])); + } + if b[1] & R_BIT == 0 { + return Err(Malformed::NoRBit); + } + Ok(Header { + opcode: b[1] & !R_BIT, + result: ResultCode::from(b[3]), + lifetime: be32(b, 4), + epoch: be32(b, 8), + }) +} + +/// Parse a response to [`announce_request`]. +pub fn parse_announce_reply(b: &[u8]) -> Result { + let h = parse_header(b)?; + if h.opcode != OP_ANNOUNCE { + return Err(Malformed::WrongOpcode(h.opcode)); + } + Ok(AnnounceReply { + result: h.result, + epoch: h.epoch, + }) +} + +/// Check that the options after the MAP payload stay inside the message. We send none, so their +/// content is ignored. +fn check_options(b: &[u8]) -> Result<(), Malformed> { + let mut off = MAP_LEN; + while off < b.len() { + if b.len() - off < 4 { + return Err(Malformed::BadOption(off)); + } + let len = usize::from(be16(b, off + 2)); + // Option data is padded to a multiple of 4 (RFC 6887 §7.3). + let total = 4 + len.div_ceil(4) * 4; + if total > b.len() - off { + return Err(Malformed::BadOption(off)); + } + off += total; + } + Ok(()) +} + +/// Parse a response to a MAP request for UDP `internal` sent with `nonce`. +pub fn parse_map_reply( + b: &[u8], + nonce: &Nonce, + internal: NonZeroU16, + kind: MapKind, +) -> Result { + let h = parse_header(b)?; + if h.opcode != OP_MAP { + return Err(Malformed::WrongOpcode(h.opcode).into()); + } + if b.len() < MAP_LEN { + if h.result == ResultCode::Success { + return Err(Malformed::TooShort(b.len()).into()); + } + // A header-only error cannot be matched by nonce (RFC 6887 §7.2 has a server that parsed + // our request copy its fields): not authoritative. + return Err(ReplyError::Unmatched { + code: h.result, + lifetime: h.lifetime, + epoch: h.epoch, + }); + } + if b.get(24..36) != Some(nonce.as_bytes().as_slice()) { + return Err(Malformed::WrongNonce.into()); + } + if b[36] != PROTO_UDP { + return Err(Malformed::WrongProtocol(b[36]).into()); + } + let got_internal = be16(b, 40); + if got_internal != internal.get() { + return Err(Malformed::InternalPortMismatch { + got: got_internal, + want: internal.get(), + } + .into()); + } + if h.result != ResultCode::Success { + return Err(ReplyError::Router { + code: h.result, + lifetime: h.lifetime, + epoch: h.epoch, + }); + } + check_options(b)?; + let ext_port = be16(b, 42); + let ext = &b[44..60]; + let ext_ip = match ext { + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, a, b2, c, d] => Ipv4Addr::new(*a, *b2, *c, *d), + _ if kind == MapKind::Delete => Ipv4Addr::UNSPECIFIED, + _ => return Err(Malformed::NotIpv4Mapped.into()), + }; + match kind { + MapKind::Map => { + if h.lifetime == 0 { + return Err(Malformed::ZeroLifetime.into()); + } + if ext_ip.is_unspecified() { + return Err(Malformed::ZeroAddress.into()); + } + if !crate::is_plausible_external(ext_ip) { + return Err(Malformed::ImplausibleAddress(ext_ip).into()); + } + if ext_port == 0 { + return Err(Malformed::ZeroExternalPort.into()); + } + } + MapKind::Delete if h.lifetime != 0 => { + return Err(Malformed::NonZeroLifetimeOnDelete(h.lifetime).into()); + } + MapKind::Delete => {} + } + Ok(MapReply { + epoch: h.epoch, + lifetime: h.lifetime, + internal: got_internal, + external: SocketAddrV4::new(ext_ip, ext_port), + }) +} + +#[cfg(test)] +mod tests { + use proptest::prelude::*; + + use super::*; + + const P1234: NonZeroU16 = NonZeroU16::new(1234).unwrap(); + const L7200: NonZeroU32 = NonZeroU32::new(7200).unwrap(); + const CLIENT: Ipv4Addr = Ipv4Addr::new(192, 168, 1, 20); + + /// Go's `examplePCPMapResponse` (`net/portmapper/pcp_test.go`) with its public external + /// address rewritten into 203.0.113.0/24. PCP has no checksum, so the rewrite is the last + /// four bytes only. Nonce = bytes 24..36, internal = external = 1234, lifetime 7200. + const GOLDEN_MAP_RESPONSE: [u8; 60] = [ + 2, 129, 0, 0, 0, 0, 28, 32, 0, 2, 155, 237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 112, + 9, 24, 241, 208, 251, 45, 157, 76, 10, 188, 17, 0, 0, 0, 4, 210, 4, 210, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 255, 255, 203, 0, 113, 246, + ]; + + fn golden_nonce() -> Nonce { + let mut n = [0u8; 12]; + n.copy_from_slice(&GOLDEN_MAP_RESPONSE[24..36]); + Nonce::from_bytes(n) + } + + /// Port of Go's `TestParsePCPMapResponse`. + #[test] + fn golden_map_response() { + let r = + parse_map_reply(&GOLDEN_MAP_RESPONSE, &golden_nonce(), P1234, MapKind::Map).unwrap(); + assert_eq!(r.external, "203.0.113.246:1234".parse().unwrap()); + assert_eq!(r.lifetime, 7200); + assert_eq!(r.epoch, 0x29bed); + assert_eq!(r.internal, 1234); + } + + /// Golden ANNOUNCE, MAP and delete requests. + #[test] + fn golden_requests() { + let mut want = [0u8; 24]; + want[0] = 2; + want[18..24].copy_from_slice(&[0xff, 0xff, 192, 168, 1, 20]); + assert_eq!(announce_request(CLIENT), want); + + let nonce = golden_nonce(); + let m = map_request(CLIENT, P1234, 41641, L7200, &nonce); + let mut want = [0u8; 60]; + want[0] = 2; + want[1] = 1; + want[4..8].copy_from_slice(&7200u32.to_be_bytes()); + want[18..24].copy_from_slice(&[0xff, 0xff, 192, 168, 1, 20]); + want[24..36].copy_from_slice(nonce.as_bytes()); + want[36] = 17; + want[40..42].copy_from_slice(&1234u16.to_be_bytes()); + want[42..44].copy_from_slice(&41641u16.to_be_bytes()); + want[54..56].copy_from_slice(&[0xff, 0xff]); + assert_eq!(m, want); + + // Delete (Errata 3621): lifetime 0, port 0, ::ffff:0.0.0.0, the ORIGINAL nonce. + let d = delete_request(CLIENT, P1234, &nonce); + want[4..8].copy_from_slice(&[0; 4]); + want[42..44].copy_from_slice(&[0, 0]); + assert_eq!(d, want); + } + + fn response(op: u8, result: u8, lifetime: u32, epoch: u32) -> Vec { + let mut b = vec![0u8; 24]; + b[0] = 2; + b[1] = op | R_BIT; + b[3] = result; + b[4..8].copy_from_slice(&lifetime.to_be_bytes()); + b[8..12].copy_from_slice(&epoch.to_be_bytes()); + b + } + + fn map_response(result: u8, lifetime: u32, nonce: &Nonce, ext: [u8; 16], port: u16) -> Vec { + let mut b = response(OP_MAP, result, lifetime, 3); + b.extend_from_slice(nonce.as_bytes()); + b.extend_from_slice(&[17, 0, 0, 0]); + b.extend_from_slice(&1234u16.to_be_bytes()); + b.extend_from_slice(&port.to_be_bytes()); + b.extend_from_slice(&ext); + b + } + + const EXT: [u8; 16] = mapped(Ipv4Addr::new(203, 0, 113, 7)); + + #[test] + fn announce_results_are_data() { + for code in [0u8, 2, 12, 99] { + let r = parse_announce_reply(&response(OP_ANNOUNCE, code, 0, 7)).unwrap(); + assert_eq!(r.result, ResultCode::from(code)); + assert_eq!(r.epoch, 7); + } + // A MAP response is not an ANNOUNCE response. + assert_eq!( + parse_announce_reply(&response(OP_MAP, 0, 0, 7)), + Err(Malformed::WrongOpcode(OP_MAP)) + ); + } + + /// Header validation. + #[test] + fn header_rejects() { + let ok = response(OP_ANNOUNCE, 0, 0, 1); + assert_eq!(parse_header(&ok[..23]), Err(Malformed::TooShort(23))); + let mut v = ok.clone(); + v[0] = 1; + assert_eq!(parse_header(&v), Err(Malformed::BadVersion(1))); + let mut r = ok.clone(); + r[1] = OP_ANNOUNCE; + assert_eq!(parse_header(&r), Err(Malformed::NoRBit)); + let mut odd = ok.clone(); + odd.extend_from_slice(&[0, 0]); + assert_eq!(parse_header(&odd), Err(Malformed::NotMultipleOf4(26))); + let mut long = ok.clone(); + long.resize(1104, 0); + assert_eq!(parse_header(&long), Err(Malformed::TooLong(1104))); + let mut max = ok; + max.resize(1100, 0); + assert!(parse_header(&max).is_ok()); + } + + /// MAP validation of nonce, protocol, internal port, external address and port. + #[test] + fn map_rejects() { + let n = golden_nonce(); + let other = Nonce::from_bytes([7; 12]); + let parse = |b: &[u8]| parse_map_reply(b, &n, P1234, MapKind::Map); + assert!(parse(&map_response(0, 60, &n, EXT, 4242)).is_ok()); + assert_eq!( + parse(&map_response(0, 60, &other, EXT, 4242)), + Err(Malformed::WrongNonce.into()) + ); + let mut proto = map_response(0, 60, &n, EXT, 4242); + proto[36] = 6; + assert_eq!(parse(&proto), Err(Malformed::WrongProtocol(6).into())); + let mut port = map_response(0, 60, &n, EXT, 4242); + port[40..42].copy_from_slice(&1235u16.to_be_bytes()); + assert_eq!( + parse(&port), + Err(Malformed::InternalPortMismatch { + got: 1235, + want: 1234 + } + .into()) + ); + let v6 = [0x20, 0x01, 0x0d, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]; + assert_eq!( + parse(&map_response(0, 60, &n, v6, 4242)), + Err(Malformed::NotIpv4Mapped.into()) + ); + assert_eq!( + parse(&map_response( + 0, + 60, + &n, + mapped(Ipv4Addr::UNSPECIFIED), + 4242 + )), + Err(Malformed::ZeroAddress.into()) + ); + let lo = Ipv4Addr::LOCALHOST; + assert_eq!( + parse(&map_response(0, 60, &n, mapped(lo), 4242)), + Err(Malformed::ImplausibleAddress(lo).into()) + ); + assert_eq!( + parse(&map_response(0, 60, &n, EXT, 0)), + Err(Malformed::ZeroExternalPort.into()) + ); + assert_eq!( + parse(&map_response(0, 0, &n, EXT, 4242)), + Err(Malformed::ZeroLifetime.into()) + ); + let mut no_r = map_response(0, 60, &n, EXT, 4242); + no_r[1] = OP_MAP; + assert_eq!(parse(&no_r), Err(Malformed::NoRBit.into())); + // A MAP parser rejects an ANNOUNCE opcode (stronger than Go, which ignores the opcode). + let mut announce = map_response(0, 60, &n, EXT, 4242); + announce[1] = R_BIT | OP_ANNOUNCE; + assert_eq!(parse(&announce), Err(Malformed::WrongOpcode(0).into())); + } + + /// A short (header-only) error carries no nonce, so it is `Unmatched`, never an + /// authoritative `Router` error, whatever its code or lifetime. + #[test] + fn short_error_is_unmatched() { + let n = golden_nonce(); + let short = response(OP_MAP, 2, 1800, 9); + assert_eq!( + parse_map_reply(&short, &n, P1234, MapKind::Map), + Err(ReplyError::Unmatched { + code: ResultCode::NotAuthorized, + lifetime: 1800, + epoch: 9 + }) + ); + // Any length from 24 to 56 bytes, any code, any lifetime: never `Router`. + for len in (HEADER_LEN..MAP_LEN).step_by(4) { + let mut b = response(OP_MAP, 8, u32::MAX, 1); + b.resize(len, 0); + for kind in [MapKind::Map, MapKind::Delete] { + assert!( + matches!( + parse_map_reply(&b, &n, P1234, kind), + Err(ReplyError::Unmatched { + code: ResultCode::NoResources, + lifetime: u32::MAX, + .. + }) + ), + "{len}" + ); + } + } + // A full-length error with our nonce is also typed; with another nonce it is not ours. + let full = map_response(8, 30, &n, EXT, 0); + assert!(matches!( + parse_map_reply(&full, &n, P1234, MapKind::Map), + Err(ReplyError::Router { + code: ResultCode::NoResources, + lifetime: 30, + .. + }) + )); + let theirs = map_response(8, 30, &Nonce::from_bytes([1; 12]), EXT, 0); + assert_eq!( + parse_map_reply(&theirs, &n, P1234, MapKind::Map), + Err(Malformed::WrongNonce.into()) + ); + // A header-only success is not a mapping. + assert_eq!( + parse_map_reply(&response(OP_MAP, 0, 60, 1), &n, P1234, MapKind::Map), + Err(Malformed::TooShort(24).into()) + ); + } + + #[test] + fn delete_reply() { + let n = golden_nonce(); + let zero = mapped(Ipv4Addr::UNSPECIFIED); + let r = parse_map_reply(&map_response(0, 0, &n, zero, 0), &n, P1234, MapKind::Delete); + assert_eq!(r.unwrap().lifetime, 0); + assert_eq!( + parse_map_reply(&map_response(0, 5, &n, EXT, 1), &n, P1234, MapKind::Delete), + Err(Malformed::NonZeroLifetimeOnDelete(5).into()) + ); + } + + #[test] + fn options_are_bounds_checked() { + let n = golden_nonce(); + let mut b = map_response(0, 60, &n, EXT, 4242); + // One well-formed option: code 1, length 4, 4 bytes of data. + b.extend_from_slice(&[1, 0, 0, 4, 9, 9, 9, 9]); + assert!(parse_map_reply(&b, &n, P1234, MapKind::Map).is_ok()); + // An option claiming 8 data bytes with only 4 present. + let len = b.len(); + b[len - 6..len - 4].copy_from_slice(&8u16.to_be_bytes()); + assert_eq!( + parse_map_reply(&b, &n, P1234, MapKind::Map), + Err(Malformed::BadOption(60).into()) + ); + } + + /// Lifetimes 1 and `u32::MAX` parse without panicking. + #[test] + fn extreme_lifetimes() { + let n = golden_nonce(); + for l in [1, u32::MAX] { + let r = parse_map_reply(&map_response(0, l, &n, EXT, 1), &n, P1234, MapKind::Map); + assert_eq!(r.unwrap().lifetime, l); + } + } + + #[test] + fn nonces_are_random_and_redacted() { + let a = Nonce::random().unwrap(); + let b = Nonce::random().unwrap(); + assert_ne!(a, b); + assert_eq!(format!("{a:?}"), "Nonce(..)"); + } + + /// The seeds of Go's `FuzzParsePCPResponse` and `FuzzParsePCPMapResponse` + /// (`net/portmapper/fuzz_test.go`), with stronger expectations: a MAP parser rejects the + /// ANNOUNCE-opcode seed and the IPv6 external address, and the result is always exactly one of + /// a mapping or an error (guaranteed by `Result`). + #[test] + fn go_fuzz_seeds() { + let mut header_seeds: Vec> = vec![ + [ + &[ + 2, 129, 0, 7, 28, 32, 155, 237, 10, 188, 17, 255, 135, 180, 175, 246, + ][..], + &[0; 8], + ] + .concat(), + [ + &[ + 2, 129, 0, 0, 210, 3, 241, 208, 251, 45, 157, 76, 10, 188, 17, 255, + ][..], + &[0; 8], + ] + .concat(), + vec![ + 2, 129, 0, 7, 28, 32, 155, 237, 10, 188, 17, 255, 135, 180, 175, 246, + ], + vec![2], + vec![ + 1, 129, 0, 0, 0, 0, 0x30, 0x39, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, + ], + vec![ + 2, 129, 0, 0, 0, 0, 0x30, 0x39, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, + ], + vec![ + 2, 0xff, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ], + vec![ + 2, 0x80, 0, 12, 0, 0, 0x30, 0x39, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, + ], + vec![ + 2, 129, 0, 1, 0, 0, 0x30, 0x39, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, + ], + vec![ + 2, 129, 0, 0xff, 0, 0, 0x30, 0x39, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, + ], + vec![ + 2, 129, 0, 0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, + ], + ]; + // Go's `{2, 0, ...}` seed has no R bit; ours rejects it. + let no_r = vec![ + 2, 0, 0, 0, 0, 0, 0x30, 0x39, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, + ]; + assert_eq!(parse_header(&no_r), Err(Malformed::NoRBit)); + header_seeds.push(no_r); + for s in &header_seeds { + let _ = parse_header(s); + let _ = parse_announce_reply(s); + } + // Announce reply opcode, accepted by ANNOUNCE and rejected by MAP. + let announce = vec![ + 2, 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]; + assert!(parse_announce_reply(&announce).is_ok()); + + let nonce = Nonce::from_bytes([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]); + let tail = |result: u8, op: u8, lifetime: [u8; 4], ext_port: [u8; 2]| -> Vec { + let mut b = vec![2, op, 0, result]; + b.extend_from_slice(&lifetime); + b.extend_from_slice(&[0, 0, 0, 1]); + b.extend_from_slice(&[0; 12]); + b.extend_from_slice(nonce.as_bytes()); + b.extend_from_slice(&[17, 0, 0, 0, 0x04, 0xd2]); + b.extend_from_slice(&ext_port); + b.extend_from_slice(&[0x20, 0x01, 0x0d, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]); + b + }; + let p = |b: &[u8]| parse_map_reply(b, &nonce, P1234, MapKind::Map); + let ok = tail(0, 129, [0, 0, 0x1c, 0x20], [0x12, 0x34]); + // Go accepts this seed's IPv6 external address; the port mapper is IPv4-only. + assert_eq!(p(&ok), Err(Malformed::NotIpv4Mapped.into())); + assert!(matches!( + p(&tail(2, 129, [0, 0, 0x1c, 0x20], [0x12, 0x34])), + Err(ReplyError::Router { + code: ResultCode::NotAuthorized, + .. + }) + )); + assert!(matches!( + p(&tail(12, 129, [0, 0, 0x1c, 0x20], [0x12, 0x34])), + Err(ReplyError::Router { + code: ResultCode::AddressMismatch, + .. + }) + )); + assert_eq!(p(&ok[..59]), Err(Malformed::NotMultipleOf4(59).into())); + let mut v1 = ok.clone(); + v1[0] = 1; + assert_eq!(p(&v1), Err(Malformed::BadVersion(1).into())); + assert_eq!( + p(&tail(0, 0x80, [0, 0, 0x1c, 0x20], [0x12, 0x34])), + Err(Malformed::WrongOpcode(0).into()) + ); + let _ = p(&tail(0, 129, [0; 4], [0x12, 0x34])); + let _ = p(&tail(0, 129, [0xff; 4], [0xff, 0xff])); + let _ = p(&GOLDEN_MAP_RESPONSE); + } + + proptest! { + #[test] + fn parsers_never_panic(b in proptest::collection::vec(any::(), 0..1200), port in 1u16..) { + let port = NonZeroU16::new(port).unwrap(); + let n = Nonce::from_bytes([0; 12]); + let _ = parse_header(&b); + let _ = parse_announce_reply(&b); + for kind in [MapKind::Map, MapKind::Delete] { + let r = parse_map_reply(&b, &n, port, kind); + // Only a full-length response with our nonce is a success or a router error. + if matches!(r, Ok(_) | Err(ReplyError::Router { .. })) { + prop_assert!(b.len() >= MAP_LEN && b[24..36] == [0; 12]); + } + } + } + + /// The same invariant on inputs shaped like MAP responses (version 2, MAP opcode with the + /// R bit, a valid length), so the error paths are actually reached. + #[test] + fn only_nonce_matched_replies_are_authoritative( + words in 6usize..20, code in any::(), lifetime in any::(), + body in proptest::collection::vec(any::(), 80), ours in any::() + ) { + let n = Nonce::from_bytes([9; 12]); + let mut b = response(OP_MAP, code, lifetime, 1); + b.extend_from_slice(&body); + b.truncate(words * 4); + if ours && b.len() >= MAP_LEN { + b[24..36].copy_from_slice(n.as_bytes()); + } + for kind in [MapKind::Map, MapKind::Delete] { + match parse_map_reply(&b, &n, P1234, kind) { + Ok(_) | Err(ReplyError::Router { .. }) => { + prop_assert!(b.len() >= MAP_LEN && b[24..36] == *n.as_bytes()); + } + Err(ReplyError::Unmatched { .. }) => { + prop_assert!(b.len() < MAP_LEN && code != 0); + } + Err(ReplyError::Malformed(_)) => {} + } + } + } + + #[test] + fn map_roundtrip( + nonce in any::<[u8; 12]>(), ip in any::<[u8; 4]>(), port in 1u16.., lifetime in 1u32.. + ) { + let ip = Ipv4Addr::from(ip); + prop_assume!(crate::is_plausible_external(ip)); + let n = Nonce::from_bytes(nonce); + let b = map_response(0, lifetime, &n, mapped(ip), port); + let r = parse_map_reply(&b, &n, P1234, MapKind::Map).unwrap(); + prop_assert_eq!(r.external, SocketAddrV4::new(ip, port)); + prop_assert_eq!(r.lifetime, lifetime); + } + } + /// Even a nonce-matched error must identify our protocol/internal port. + #[test] + fn error_replies_validate_full_request_identity() { + for at in [36, 41] { + let mut packet = GOLDEN_MAP_RESPONSE; + packet[3] = 2; + packet[at] ^= 1; + assert!( + matches!( + parse_map_reply(&packet, &golden_nonce(), P1234, MapKind::Map), + Err(ReplyError::Malformed(_)) + ), + "wrong field {at} was authoritative" + ); + } + } +} diff --git a/ts_portmapper/src/pmp.rs b/ts_portmapper/src/pmp.rs new file mode 100644 index 00000000..690847ab --- /dev/null +++ b/ts_portmapper/src/pmp.rs @@ -0,0 +1,610 @@ +//! NAT-PMP codec (RFC 6886). +//! +//! Pure functions over byte slices: no I/O, no allocation, no panics (fuzzed). Parsing is strict: +//! a reply must carry version 0, the request's opcode plus 128, and for success the exact RFC +//! length; anything else is [`Malformed`]. A nonzero result code is a typed +//! [`ReplyError::Router`], never a success. +//! +//! ```compile_fail +//! // The internal port is a `NonZeroU16`: a request for internal port 0 ("all ports") cannot be +//! // built. +//! let _ = ts_portmapper::pmp::map_request(0, 0, std::num::NonZeroU32::MIN); +//! ``` + +use std::{ + fmt, + net::Ipv4Addr, + num::{NonZeroU16, NonZeroU32}, +}; + +/// NAT-PMP protocol version. +pub const VERSION: u8 = 0; +/// Opcode: request the gateway's public address. +pub const OP_PUBLIC_ADDR: u8 = 0; +/// Opcode: map a UDP port. +pub const OP_MAP_UDP: u8 = 1; +/// Added to a request's opcode in its reply. +pub const OP_REPLY: u8 = 0x80; +/// Exact length of a successful public-address reply. +pub const PUBLIC_ADDR_REPLY_LEN: usize = 12; +/// Exact length of a successful MAP reply. +pub const MAP_REPLY_LEN: usize = 16; +/// Length of a MAP request. +pub const MAP_REQUEST_LEN: usize = 12; +/// Shortest reply that carries a result code and epoch. +pub const MIN_REPLY_LEN: usize = 8; + +/// A NAT-PMP result code (RFC 6886 §3.5). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ResultCode { + /// 0: success. + Success, + /// 1: unsupported version. + UnsupportedVersion, + /// 2: not authorized/refused (for example, mapping turned off on the router). + NotAuthorized, + /// 3: network failure (for example, the router has no DHCP lease). + NetworkFailure, + /// 4: out of resources. + OutOfResources, + /// 5: unsupported opcode. + UnsupportedOpcode, + /// Any other value. + Unknown(u16), +} + +impl From for ResultCode { + fn from(v: u16) -> Self { + match v { + 0 => Self::Success, + 1 => Self::UnsupportedVersion, + 2 => Self::NotAuthorized, + 3 => Self::NetworkFailure, + 4 => Self::OutOfResources, + 5 => Self::UnsupportedOpcode, + n => Self::Unknown(n), + } + } +} + +impl From for u16 { + fn from(c: ResultCode) -> u16 { + match c { + ResultCode::Success => 0, + ResultCode::UnsupportedVersion => 1, + ResultCode::NotAuthorized => 2, + ResultCode::NetworkFailure => 3, + ResultCode::OutOfResources => 4, + ResultCode::UnsupportedOpcode => 5, + ResultCode::Unknown(n) => n, + } + } +} + +impl fmt::Display for ResultCode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let name = match self { + Self::Success => "success", + Self::UnsupportedVersion => "unsupported-version", + Self::NotAuthorized => "not-authorized", + Self::NetworkFailure => "network-failure", + Self::OutOfResources => "out-of-resources", + Self::UnsupportedOpcode => "unsupported-opcode", + Self::Unknown(_) => "unknown", + }; + write!(f, "{name}({})", u16::from(*self)) + } +} + +/// Why a datagram is not an acceptable reply to the request it was matched against. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Malformed { + /// Shorter than the 8-byte common reply header. + TooShort(usize), + /// Not NAT-PMP version 0. + BadVersion(u8), + /// Not the request's opcode plus 128. + WrongOpcode(u8), + /// A success reply with the wrong length. + BadLength(usize), + /// A public-address reply of `0.0.0.0`. + ZeroAddress, + /// A public address no router can have (see [`crate::is_plausible_external`]). + ImplausibleAddress(Ipv4Addr), + /// A MAP reply for a different internal port. + InternalPortMismatch { + /// Internal port in the reply. + got: u16, + /// Internal port we asked for. + want: u16, + }, + /// A MAP success with external port 0. + ZeroExternalPort, + /// A MAP success with lifetime 0 for a create or renew. + ZeroLifetime, + /// A reply to a delete that still carries a lifetime. + NonZeroLifetimeOnDelete(u32), +} + +impl fmt::Display for Malformed { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::TooShort(n) => write!(f, "too short ({n} bytes)"), + Self::BadVersion(v) => write!(f, "version {v}, not 0"), + Self::WrongOpcode(o) => write!(f, "unexpected opcode {o:#04x}"), + Self::BadLength(n) => write!(f, "success reply of {n} bytes"), + Self::ZeroAddress => f.write_str("public address 0.0.0.0"), + Self::ImplausibleAddress(ip) => write!(f, "implausible public address {ip}"), + Self::InternalPortMismatch { got, want } => { + write!(f, "internal port {got}, expected {want}") + } + Self::ZeroExternalPort => f.write_str("external port 0"), + Self::ZeroLifetime => f.write_str("lifetime 0 on a create"), + Self::NonZeroLifetimeOnDelete(l) => write!(f, "lifetime {l} on a delete"), + } + } +} + +/// A reply that is not a success. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +pub enum ReplyError { + /// A well-formed reply with a nonzero result code. + #[error("NAT-PMP {code} (epoch {epoch})")] + Router { + /// The result code. + code: ResultCode, + /// Seconds since the router's start of epoch. + epoch: u32, + }, + /// A header-only error has no internal port and cannot identify a MAP request. + #[error("NAT-PMP {code} without an internal port (epoch {epoch})")] + Unmatched { + /// Result code. + code: ResultCode, + /// Router epoch. + epoch: u32, + }, + /// Not an acceptable reply. + #[error("malformed NAT-PMP reply: {0}")] + Malformed(Malformed), +} + +impl From for ReplyError { + fn from(m: Malformed) -> Self { + Self::Malformed(m) + } +} + +/// A successful public-address reply. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PublicAddrReply { + /// Seconds since the router's start of epoch. + pub epoch: u32, + /// The router's public address; never `0.0.0.0`. + pub public_ip: Ipv4Addr, +} + +/// A successful MAP reply. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MapReply { + /// Seconds since the router's start of epoch. + pub epoch: u32, + /// The internal port; always the one requested. + pub internal: u16, + /// The assigned external port. Nonzero for a create; 0 is allowed in a delete reply. + pub external: u16, + /// Granted lifetime in seconds. Nonzero for a create; 0 for a delete. + pub lifetime: u32, +} + +/// Whether a MAP request creates/renews a mapping or deletes it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MapKind { + /// Create or renew (lifetime > 0). + Map, + /// Delete (lifetime 0). + Delete, +} + +/// The public-address request: `{version 0, opcode 0}`. +pub const fn public_address_request() -> [u8; 2] { + [VERSION, OP_PUBLIC_ADDR] +} + +/// A MAP-UDP request for `internal`, suggesting `suggested_external` (0 for none), with a +/// nonzero lifetime. +pub fn map_request( + internal: NonZeroU16, + suggested_external: u16, + lifetime: NonZeroU32, +) -> [u8; MAP_REQUEST_LEN] { + build_map(internal.get(), suggested_external, lifetime.get()) +} + +/// A delete request for `internal`: lifetime 0 and suggested external port 0, as RFC 6886 §3.4 +/// requires. (Go sends the current external port instead; routers ignore it.) +pub fn delete_request(internal: NonZeroU16) -> [u8; MAP_REQUEST_LEN] { + build_map(internal.get(), 0, 0) +} + +fn build_map(internal: u16, suggested_external: u16, lifetime: u32) -> [u8; MAP_REQUEST_LEN] { + let mut p = [0u8; MAP_REQUEST_LEN]; + p[0] = VERSION; + p[1] = OP_MAP_UDP; + p[4..6].copy_from_slice(&internal.to_be_bytes()); + p[6..8].copy_from_slice(&suggested_external.to_be_bytes()); + p[8..12].copy_from_slice(&lifetime.to_be_bytes()); + p +} + +/// Read a big-endian u16 at `at`; the caller has checked the length. +fn be16(b: &[u8], at: usize) -> u16 { + match b.get(at..at + 2) { + Some(&[x, y]) => u16::from_be_bytes([x, y]), + _ => 0, + } +} + +/// Read a big-endian u32 at `at`; the caller has checked the length. +fn be32(b: &[u8], at: usize) -> u32 { + match b.get(at..at + 4) { + Some(&[w, x, y, z]) => u32::from_be_bytes([w, x, y, z]), + _ => 0, + } +} + +/// Check the common header against `op` and return the epoch, or the router's error. +fn header(b: &[u8], op: u8) -> Result { + if b.len() < MIN_REPLY_LEN { + return Err(Malformed::TooShort(b.len()).into()); + } + if b[0] != VERSION { + return Err(Malformed::BadVersion(b[0]).into()); + } + if b[1] != op | OP_REPLY { + return Err(Malformed::WrongOpcode(b[1]).into()); + } + let code = ResultCode::from(be16(b, 2)); + let epoch = be32(b, 4); + if code != ResultCode::Success { + return Err(ReplyError::Router { code, epoch }); + } + Ok(epoch) +} + +/// The result code and epoch of any well-formed NAT-PMP reply header (version 0, reply bit set, +/// at least 8 bytes), whatever its opcode. Used to recognise a NAT-PMP server's "unsupported +/// version" answer to a PCP request (RFC 6887 Appendix A). +pub fn reply_result(b: &[u8]) -> Option<(ResultCode, u32)> { + match b { + [VERSION, op, ..] if b.len() >= MIN_REPLY_LEN && op & OP_REPLY != 0 => { + Some((ResultCode::from(be16(b, 2)), be32(b, 4))) + } + _ => None, + } +} + +/// Parse a reply to [`public_address_request`]. +pub fn parse_public_addr_reply(b: &[u8]) -> Result { + let epoch = match header(b, OP_PUBLIC_ADDR) { + Err(e @ ReplyError::Router { .. }) + if matches!(b.len(), MIN_REPLY_LEN | PUBLIC_ADDR_REPLY_LEN) => + { + return Err(e); + } + Err(ReplyError::Router { .. }) => return Err(Malformed::BadLength(b.len()).into()), + other => other?, + }; + if b.len() != PUBLIC_ADDR_REPLY_LEN { + return Err(Malformed::BadLength(b.len()).into()); + } + let public_ip = Ipv4Addr::from(be32(b, 8)); + if public_ip.is_unspecified() { + return Err(Malformed::ZeroAddress.into()); + } + if !crate::is_plausible_external(public_ip) { + return Err(Malformed::ImplausibleAddress(public_ip).into()); + } + Ok(PublicAddrReply { epoch, public_ip }) +} + +/// Parse a reply to a MAP request for `internal` (see [`map_request`] and [`delete_request`]). +pub fn parse_map_reply( + b: &[u8], + internal: NonZeroU16, + kind: MapKind, +) -> Result { + let (epoch, refusal) = match header(b, OP_MAP_UDP) { + Ok(epoch) => (epoch, None), + Err(ReplyError::Router { code, epoch }) => (epoch, Some(code)), + Err(e) => return Err(e), + }; + if b.len() == MIN_REPLY_LEN + && let Some(code) = refusal + { + return Err(ReplyError::Unmatched { code, epoch }); + } + if b.len() != MAP_REPLY_LEN { + return Err(Malformed::BadLength(b.len()).into()); + } + let reply = MapReply { + epoch, + internal: be16(b, 8), + external: be16(b, 10), + lifetime: be32(b, 12), + }; + if reply.internal != internal.get() { + return Err(Malformed::InternalPortMismatch { + got: reply.internal, + want: internal.get(), + } + .into()); + } + if let Some(code) = refusal { + return Err(ReplyError::Router { code, epoch }); + } + match kind { + MapKind::Map if reply.lifetime == 0 => Err(Malformed::ZeroLifetime.into()), + MapKind::Map if reply.external == 0 => Err(Malformed::ZeroExternalPort.into()), + MapKind::Delete if reply.lifetime != 0 => { + Err(Malformed::NonZeroLifetimeOnDelete(reply.lifetime).into()) + } + _ => Ok(reply), + } +} + +#[cfg(test)] +mod tests { + use proptest::prelude::*; + + use super::*; + + const P1234: NonZeroU16 = NonZeroU16::new(1234).unwrap(); + const L7200: NonZeroU32 = NonZeroU32::new(7200).unwrap(); + + fn map_reply(code: u16, epoch: u32, internal: u16, external: u16, lifetime: u32) -> Vec { + let mut b = vec![VERSION, OP_MAP_UDP | OP_REPLY]; + b.extend_from_slice(&code.to_be_bytes()); + b.extend_from_slice(&epoch.to_be_bytes()); + b.extend_from_slice(&internal.to_be_bytes()); + b.extend_from_slice(&external.to_be_bytes()); + b.extend_from_slice(&lifetime.to_be_bytes()); + b + } + + fn addr_reply(code: u16, epoch: u32, ip: [u8; 4]) -> Vec { + let mut b = vec![VERSION, OP_PUBLIC_ADDR | OP_REPLY]; + b.extend_from_slice(&code.to_be_bytes()); + b.extend_from_slice(&epoch.to_be_bytes()); + b.extend_from_slice(&ip); + b + } + + /// Golden request bytes. + #[test] + fn build_golden() { + assert_eq!(public_address_request(), [0, 0]); + assert_eq!( + map_request(P1234, 0, L7200), + [0, 1, 0, 0, 0x04, 0xd2, 0, 0, 0, 0, 0x1c, 0x20] + ); + assert_eq!( + map_request(P1234, 41641, L7200), + [0, 1, 0, 0, 0x04, 0xd2, 0xa2, 0xa9, 0, 0, 0x1c, 0x20] + ); + // Delete: lifetime 0 AND suggested external port 0 (RFC 6886 §3.4). + assert_eq!( + delete_request(P1234), + [0, 1, 0, 0, 0x04, 0xd2, 0, 0, 0, 0, 0, 0] + ); + } + + #[test] + fn public_addr_ok() { + let r = parse_public_addr_reply(&addr_reply(0, 77, [203, 0, 113, 7])).unwrap(); + assert_eq!(r.epoch, 77); + assert_eq!(r.public_ip, Ipv4Addr::new(203, 0, 113, 7)); + } + + /// Header and length validation. + #[test] + fn public_addr_rejects() { + let good = addr_reply(0, 1, [203, 0, 113, 7]); + let mut bad_ver = good.clone(); + bad_ver[0] = 2; + assert_eq!( + parse_public_addr_reply(&bad_ver), + Err(Malformed::BadVersion(2).into()) + ); + let mut bad_op = good.clone(); + bad_op[1] = OP_MAP_UDP | OP_REPLY; + assert_eq!( + parse_public_addr_reply(&bad_op), + Err(Malformed::WrongOpcode(0x81).into()) + ); + // A request echoed back (no reply bit) is not a reply. + let mut no_reply_bit = good.clone(); + no_reply_bit[1] = OP_PUBLIC_ADDR; + assert!(matches!( + parse_public_addr_reply(&no_reply_bit), + Err(ReplyError::Malformed(Malformed::WrongOpcode(0))) + )); + assert_eq!( + parse_public_addr_reply(&good[..11]), + Err(Malformed::BadLength(11).into()) + ); + let mut long = good.clone(); + long.push(0); + assert_eq!( + parse_public_addr_reply(&long), + Err(Malformed::BadLength(13).into()) + ); + assert_eq!( + parse_public_addr_reply(&good[..7]), + Err(Malformed::TooShort(7).into()) + ); + assert_eq!( + parse_public_addr_reply(&addr_reply(0, 1, [0, 0, 0, 0])), + Err(Malformed::ZeroAddress.into()) + ); + for bad in [ + [127, 0, 0, 1], + [255, 255, 255, 255], + [224, 0, 0, 1], + [169, 254, 0, 1], + ] { + assert_eq!( + parse_public_addr_reply(&addr_reply(0, 1, bad)), + Err(Malformed::ImplausibleAddress(bad.into()).into()) + ); + } + // A double NAT's private or shared WAN address is reported (diagnostics), not rejected. + for ok in [[192, 168, 0, 2], [100, 64, 0, 9]] { + assert!(parse_public_addr_reply(&addr_reply(0, 1, ok)).is_ok()); + } + } + + /// Result codes 1–5 and an unknown code are typed errors, never a success, even in a + /// short 8-byte reply. + #[test] + fn result_codes_are_errors() { + for (raw, code) in [ + (1, ResultCode::UnsupportedVersion), + (2, ResultCode::NotAuthorized), + (3, ResultCode::NetworkFailure), + (4, ResultCode::OutOfResources), + (5, ResultCode::UnsupportedOpcode), + (99, ResultCode::Unknown(99)), + ] { + let full = addr_reply(raw, 9, [203, 0, 113, 7]); + let want = ReplyError::Router { code, epoch: 9 }; + assert_eq!(parse_public_addr_reply(&full), Err(want)); + assert_eq!(parse_public_addr_reply(&full[..8]), Err(want)); + let m = map_reply(raw, 9, 1234, 0, 0); + assert_eq!(parse_map_reply(&m, P1234, MapKind::Map), Err(want)); + assert_eq!( + parse_map_reply(&m[..8], P1234, MapKind::Map), + Err(ReplyError::Unmatched { code, epoch: 9 }) + ); + assert_eq!(u16::from(code), raw); + } + } + + #[test] + fn map_ok_and_renew_with_new_port() { + let r = parse_map_reply(&map_reply(0, 5, 1234, 4242, 3600), P1234, MapKind::Map).unwrap(); + assert_eq!( + r, + MapReply { + epoch: 5, + internal: 1234, + external: 4242, + lifetime: 3600 + } + ); + } + + /// MAP-specific rejections. + #[test] + fn map_rejects() { + assert_eq!( + parse_map_reply(&map_reply(0, 5, 1235, 4242, 7200), P1234, MapKind::Map), + Err(Malformed::InternalPortMismatch { + got: 1235, + want: 1234 + } + .into()) + ); + assert_eq!( + parse_map_reply(&map_reply(0, 5, 1234, 0, 7200), P1234, MapKind::Map), + Err(Malformed::ZeroExternalPort.into()) + ); + // Lifetime 0 on a create is a failure. + assert_eq!( + parse_map_reply(&map_reply(0, 5, 1234, 4242, 0), P1234, MapKind::Map), + Err(Malformed::ZeroLifetime.into()) + ); + let good = map_reply(0, 5, 1234, 4242, 7200); + assert_eq!( + parse_map_reply(&good[..15], P1234, MapKind::Map), + Err(Malformed::BadLength(15).into()) + ); + // A public-address reply is not a MAP reply. + assert_eq!( + parse_map_reply(&addr_reply(0, 1, [203, 0, 113, 7]), P1234, MapKind::Map), + Err(Malformed::WrongOpcode(0x80).into()) + ); + } + + #[test] + fn delete_reply() { + let r = parse_map_reply(&map_reply(0, 5, 1234, 0, 0), P1234, MapKind::Delete).unwrap(); + assert_eq!((r.external, r.lifetime), (0, 0)); + assert_eq!( + parse_map_reply(&map_reply(0, 5, 1234, 4242, 60), P1234, MapKind::Delete), + Err(Malformed::NonZeroLifetimeOnDelete(60).into()) + ); + } + + #[test] + fn reply_result_any_opcode() { + assert_eq!( + reply_result(&[0, 0x81, 0, 1, 0, 0, 0, 9]), + Some((ResultCode::UnsupportedVersion, 9)) + ); + assert_eq!(reply_result(&[0, 0x01, 0, 1, 0, 0, 0, 9]), None); + assert_eq!(reply_result(&[0, 0x81, 0, 1]), None); + assert_eq!(reply_result(&[2, 0x81, 0, 1, 0, 0, 0, 9]), None); + } + + /// Lifetimes 0, 1 and `u32::MAX` parse without panicking. + #[test] + fn extreme_lifetimes() { + for l in [1, u32::MAX] { + let r = parse_map_reply(&map_reply(0, u32::MAX, 1234, 1, l), P1234, MapKind::Map); + assert_eq!(r.unwrap().lifetime, l); + } + assert!(parse_map_reply(&map_reply(0, 0, 1234, 1, 0), P1234, MapKind::Delete).is_ok()); + } + + proptest! { + #[test] + fn parsers_never_panic(b in proptest::collection::vec(any::(), 0..64), port in 1u16..) { + let port = NonZeroU16::new(port).unwrap(); + let _ = parse_public_addr_reply(&b); + let _ = parse_map_reply(&b, port, MapKind::Map); + let _ = parse_map_reply(&b, port, MapKind::Delete); + } + + #[test] + fn map_reply_roundtrip( + epoch in any::(), internal in 1u16.., external in 1u16.., lifetime in 1u32.. + ) { + let b = map_reply(0, epoch, internal, external, lifetime); + let r = parse_map_reply(&b, NonZeroU16::new(internal).unwrap(), MapKind::Map).unwrap(); + prop_assert_eq!(r, MapReply { epoch, internal, external, lifetime }); + } + + #[test] + fn map_request_layout(internal in 1u16.., ext in any::(), lifetime in 1u32..) { + let p = map_request( + NonZeroU16::new(internal).unwrap(), ext, NonZeroU32::new(lifetime).unwrap()); + prop_assert_eq!(&p[..4], &[0, 1, 0, 0]); + prop_assert_eq!(be16(&p, 4), internal); + prop_assert_eq!(be16(&p, 6), ext); + prop_assert_eq!(be32(&p, 8), lifetime); + } + } + #[test] + fn public_error_lengths_are_validated() { + let packet = addr_reply(2, 1, [203, 0, 113, 7]); + for len in [9, 11, 13] { + let mut error = packet.clone(); + error.resize(len, 0); + assert!(matches!( + parse_public_addr_reply(&error), + Err(ReplyError::Malformed(Malformed::BadLength(_))) + )); + } + } +} diff --git a/ts_portmapper/src/probe_tool.rs b/ts_portmapper/src/probe_tool.rs new file mode 100644 index 00000000..7b6c4d77 --- /dev/null +++ b/ts_portmapper/src/probe_tool.rs @@ -0,0 +1,1198 @@ +//! The `--portmap-probe` diagnostic: validate NAT-PMP/PCP in a deployment's real launch context +//! without touching the node. +//! +//! It binds a dedicated, receive-only test socket on the gateway's self address (never the node's +//! port, never 41641/41642), probes both protocols, then for each requested protocol in turn maps +//! the test port, optionally holds the mapping open while counting inbound datagrams (length and +//! source only, never the payload, never a reply), and deletes it. If the router grants a lease no +//! longer than the hold, the mapping is renewed every half lease during the hold. Total time is +//! bounded by the gateway wait (at most 120 s) and the hold (at most 600 s) plus a few 250 ms +//! exchanges. An interrupt is checked before each map and during the hold: it skips the remaining +//! maps, and the delete for anything sent still runs. The default lease is 120 s, so even a +//! crashed probe leaves at most a 120 s request behind. +//! +//! Output is stable `key=value` lines on the given writer. It may contain the router's public +//! address: never commit it. + +use std::{ + future::Future, + io::Write, + net::SocketAddr, + num::{NonZeroU16, NonZeroU32}, + pin::Pin, + sync::Arc, + time::Duration, +}; + +use crate::{ + Gateway, GatewayCap, LookupError, NetGen, PolicyHandle, PxpTransport, + gateway::{GatewayLookup, GatewayOverride, SystemLookup}, + oneshot::{self, MapError, Mapping, Proto, ProtoSet, Rule}, +}; + +/// Ports the test socket may never use: Go's and this runner's default node ports. +pub const RESERVED_PORTS: [u16; 2] = [41641, 41642]; +/// Default requested lease, in seconds. +pub const DEFAULT_LIFETIME: u32 = 120; +/// Longest lease the probe may request, in seconds. +pub const MAX_LIFETIME: u32 = 7200; +/// Longest hold window. +pub const MAX_HOLD: Duration = Duration::from_secs(600); +/// Longest wait for a usable gateway (a launchd job started before the network is up). +pub const MAX_WAIT: Duration = Duration::from_secs(120); +/// How often the gateway lookup is retried while waiting. +const WAIT_POLL: Duration = Duration::from_secs(1); +/// At most this many per-datagram `inbound` lines are printed per hold. +const MAX_INBOUND_LINES: usize = 1000; +/// At most this many distinct sources are listed per hold. +const MAX_SOURCES: usize = 32; + +/// What to probe. +#[derive(Debug, Clone)] +pub struct ProbeOptions { + /// Protocols to map, each separately. + pub protos: ProtoSet, + /// The test socket's port; 0 for an ephemeral one. + pub test_port: u16, + /// Requested lease in seconds (1..=7200). + pub lifetime: u32, + /// How long to hold each mapping open (at most 600 s, shorter than the lease). + pub hold: Duration, + /// Use this gateway instead of discovering one (still fully validated). + pub gateway_override: Option, + /// The node's own UDP port, which the test socket must not use. + pub node_port: u16, + /// How long to keep retrying the gateway lookup (routing-table reads only, no packets) + /// until it finds a gateway; at most 120 s. + pub wait: Duration, +} + +impl Default for ProbeOptions { + fn default() -> Self { + Self { + protos: ProtoSet::BOTH, + test_port: 0, + lifetime: DEFAULT_LIFETIME, + hold: Duration::ZERO, + gateway_override: None, + node_port: 41642, + wait: Duration::ZERO, + } + } +} + +impl ProbeOptions { + fn forbidden(&self, port: u16) -> bool { + port != 0 && (RESERVED_PORTS.contains(&port) || port == self.node_port) + } + + /// Check the options; the error says what is wrong. + pub fn validate(&self) -> Result<(), String> { + if !self.protos.pmp && !self.protos.pcp { + return Err("no protocol selected".to_owned()); + } + if self.forbidden(self.test_port) { + return Err(format!( + "test port {} is a node port (41641, 41642 and --port are reserved)", + self.test_port + )); + } + if !(1..=MAX_LIFETIME).contains(&self.lifetime) { + return Err(format!("lifetime must be 1..={MAX_LIFETIME} seconds")); + } + if self.hold > MAX_HOLD { + return Err(format!( + "hold must be at most {} seconds", + MAX_HOLD.as_secs() + )); + } + if self.hold >= Duration::from_secs(u64::from(self.lifetime)) && !self.hold.is_zero() { + return Err("hold must be shorter than the lifetime".to_owned()); + } + if self.wait > MAX_WAIT { + return Err(format!( + "wait must be at most {} seconds", + MAX_WAIT.as_secs() + )); + } + Ok(()) + } +} + +/// Quote a value if it contains anything that would break `key=value` parsing. +fn v(s: impl ToString) -> String { + let s = s.to_string(); + if s.is_empty() { + "-".to_owned() + } else if s.chars().any(|c| c.is_whitespace() || c == '"' || c == '=') { + format!("{s:?}") + } else { + s + } +} + +fn opt(o: Option) -> String { + o.map_or_else(|| "-".to_owned(), v) +} + +/// Whether `cancel` has fired, without waiting. Never call it again once it returned `true`. +async fn fired(cancel: &mut Pin<&mut impl Future>) -> bool { + tokio::select! { + biased; + () = cancel.as_mut() => true, + () = std::future::ready(()) => false, + } +} + +/// Look up the gateway, retrying every second for up to `wait` while there is none (or the +/// lookup fails), e.g. a launchd job started before the network is up. Reads the routing table +/// and interface list only; sends nothing. `Err(None)` means `cancel` fired. +async fn wait_for_gateway( + lookup: Arc, + wait: Duration, + cancel: &mut Pin<&mut impl Future>, + out: &mut dyn Write, +) -> Result, LookupError>>> { + let start = tokio::time::Instant::now(); + let mut attempts = 0u32; + let result = loop { + attempts += 1; + let os_lookup = lookup.clone(); + let work = crate::blocking::run(move || os_lookup.lookup()); + let r = tokio::select! { + biased; + () = cancel.as_mut() => return Err(None), + r = work => r.unwrap_or(Ok(None)), + }; + if matches!(r, Ok(Some(_))) || start.elapsed() >= wait { + break r; + } + tokio::select! { + () = tokio::time::sleep(WAIT_POLL.min(wait.saturating_sub(start.elapsed()))) => {} + () = cancel.as_mut() => { + writeln!(out, "gateway wait={}s attempts={attempts} interrupted=true", start.elapsed().as_secs()).ok(); + return Err(None); + } + } + }; + if !wait.is_zero() { + writeln!( + out, + "gateway wait={}s attempts={attempts}", + start.elapsed().as_secs() + ) + .ok(); + } + match result { + Ok(Some(g)) => Ok(g), + other => Err(Some(other)), + } +} + +/// The process context the probe runs in (the launch context is what is being validated). +fn env_line() -> String { + let os = os_version(); + #[cfg(unix)] + let (uid, euid, ppid) = { + // SAFETY: getuid and geteuid take no arguments, cannot fail and touch no memory. + let (uid, euid) = unsafe { (libc::getuid(), libc::geteuid()) }; + ( + uid.to_string(), + euid.to_string(), + std::os::unix::process::parent_id().to_string(), + ) + }; + #[cfg(not(unix))] + let (uid, euid, ppid) = ("-".to_owned(), "-".to_owned(), "-".to_owned()); + let exe = std::env::current_exe().map_or_else(|_| "-".to_owned(), |p| v(p.display())); + let xpc = std::env::var("XPC_SERVICE_NAME").unwrap_or_default(); + format!( + "env os={} uid={uid} euid={euid} exe={exe} ppid={ppid} xpc_service={}", + v(os), + v(xpc) + ) +} + +/// `/` on macOS, the kernel release elsewhere. +fn os_version() -> String { + #[cfg(target_os = "macos")] + { + let plist = std::fs::read_to_string("/System/Library/CoreServices/SystemVersion.plist") + .unwrap_or_default(); + let key = |k: &str| { + let after = plist.split(&format!("{k}")).nth(1)?; + let start = after.find("")? + "".len(); + let end = after[start..].find("")? + start; + Some(after[start..end].to_owned()) + }; + format!( + "macos-{}/{}", + key("ProductVersion").unwrap_or_default(), + key("ProductBuildVersion").unwrap_or_default() + ) + } + #[cfg(not(target_os = "macos"))] + { + let release = std::fs::read_to_string("/proc/sys/kernel/osrelease").unwrap_or_default(); + format!("{}-{}", std::env::consts::OS, release.trim()) + } +} + +/// Run the probe against the real gateway: lookup (or validated override), [`GatewayCap`], then +/// the steps of [`run_with`]. Returns the exit code: 0 if every requested protocol was mapped and +/// deleted, 2 if some were, 1 if none were or the lookup or capability failed. +pub async fn run( + opts: &ProbeOptions, + cancel: impl Future, + out: &mut dyn Write, +) -> i32 { + if let Err(e) = opts.validate() { + writeln!(out, "error msg={}", v(e)).ok(); + return 1; + } + let mut cancel = std::pin::pin!(cancel); + let lookup: Arc = Arc::new(SystemLookup::new(opts.gateway_override)); + let source = if opts.gateway_override.is_some() { + "override" + } else { + "discovered" + }; + let gateway = match wait_for_gateway(lookup.clone(), opts.wait, &mut cancel, out).await { + Ok(g) => g, + Err(None) => { + writeln!(out, "{}", env_line()).ok(); + writeln!(out, "gateway source={source} error=interrupted").ok(); + return 1; + } + Err(Some(Ok(Some(_)))) => unreachable!("a found gateway is returned as Ok"), + Err(Some(Ok(None))) => { + writeln!(out, "{}", env_line()).ok(); + writeln!(out, "gateway source={source} error=no-ipv4-default-route").ok(); + return 1; + } + Err(Some(Err(e))) => { + writeln!(out, "{}", env_line()).ok(); + writeln!(out, "gateway source={source} error={}", v(e)).ok(); + return 1; + } + }; + let validate_gateway = gateway.clone(); + let cap = match crate::blocking::run(move || { + GatewayCap::validate( + validate_gateway, + NetGen(1), + PolicyHandle::new(true), + Some(lookup), + ) + }) + .await + .unwrap_or(Err(crate::CapError::InvalidGateway( + "validation worker failed", + ))) { + Ok(c) => c, + Err(e) => { + writeln!(out, "{}", env_line()).ok(); + writeln!( + out, + "gateway gw={} self={} if={}/{} source={source} error={}", + gateway.gw, + gateway.self_ip, + v(&gateway.if_name), + gateway.if_index, + v(e) + ) + .ok(); + return 1; + } + }; + run_with(opts, &*cap, source, cancel, out).await +} + +/// The probe steps over any transport (the real cap, or the test transport in tests). +pub async fn run_with( + opts: &ProbeOptions, + t: &T, + source: &str, + cancel: impl Future, + out: &mut dyn Write, +) -> i32 { + writeln!(out, "{}", env_line()).ok(); + if let Err(e) = opts.validate() { + writeln!(out, "error msg={}", v(e)).ok(); + return 1; + } + let g = t.gateway(); + writeln!( + out, + "gateway gw={} self={} if={}/{} prefix=/{} source={source} bind={}", + g.gw, + g.self_ip, + v(&g.if_name), + g.if_index, + g.prefix_len, + t.bind_mode() + ) + .ok(); + + // The dedicated test socket. An ephemeral bind can land on a reserved port (Linux's + // ephemeral range includes 41641/41642), so retry a few times. + let mut listener = None; + for _ in 0..5 { + match t.bind_probe_listener(opts.test_port) { + Ok(l) if opts.forbidden(l.port()) => continue, + Ok(l) => { + listener = Some(l); + break; + } + Err(e) => { + writeln!(out, "listen port={} error={}", opts.test_port, v(e)).ok(); + return 1; + } + } + } + let Some(mut listener) = listener else { + writeln!(out, "listen error=only-reserved-ports").ok(); + return 1; + }; + let Some(internal) = NonZeroU16::new(listener.port()) else { + writeln!(out, "listen error=port-0").ok(); + return 1; + }; + writeln!(out, "listen port={internal} self={}", g.self_ip).ok(); + + match oneshot::probe(t, opts.protos).await { + Ok(r) => { + writeln!( + out, + "probe pmp={} pmp_epoch={} pmp_ext_ip={} pmp_result={} pcp={} pcp_result={} pcp_epoch={} errors={}", + r.pmp, + opt(r.pmp_epoch), + opt(r.pmp_public_ip), + opt(r.pmp_result), + r.pcp, + opt(r.pcp_result), + opt(r.pcp_epoch), + v(r.errors.join("; ")) + ) + .ok(); + } + Err(e) => { + writeln!(out, "probe error={}", v(e)).ok(); + return 1; + } + } + + let lifetime = NonZeroU32::new(opts.lifetime).unwrap_or(NonZeroU32::MIN); + let mut cancel = std::pin::pin!(cancel); + let mut interrupted = false; + let mut requested = 0; + let mut succeeded = 0; + for proto in opts.protos.iter() { + requested += 1; + // An interrupt during the probe or an earlier map/delete skips the remaining maps. + if !interrupted { + interrupted = fired(&mut cancel).await; + } + if interrupted { + writeln!(out, "map proto={proto} result=skipped err=interrupted").ok(); + continue; + } + let rule = match Rule::new(proto, internal) { + Ok(r) => r, + Err(e) => { + writeln!(out, "map proto={proto} result=error err={}", v(e)).ok(); + continue; + } + }; + let mapped = oneshot::map(t, &rule, lifetime, None).await; + let (result, err) = match &mapped { + Ok(_) => ("ok".to_owned(), None), + Err(MapError::Pmp { code, .. }) => (code.to_string(), None), + Err(MapError::Pcp { code, lifetime, .. }) => ( + code.to_string(), + Some(format!("error lifetime {lifetime} s")), + ), + Err(MapError::PmpAddr { code, .. }) => ( + code.to_string(), + Some("public-address request failed; the MAP may exist".to_owned()), + ), + Err(MapError::PcpUnverified { code, lifetime, .. }) => ( + code.to_string(), + Some(format!("no nonce, unverified; error lifetime {lifetime} s")), + ), + Err(MapError::NoServices { malformed }) => ("timeout".to_owned(), malformed.clone()), + Err(e) => ("error".to_owned(), Some(e.to_string())), + }; + match &mapped { + Ok(m) => writeln!( + out, + "map proto={proto} internal={internal} requested_lifetime={} result=ok external={} lease={} epoch={} err=-", + opts.lifetime, m.external, m.lifetime, m.epoch + ), + Err( + MapError::Pmp { epoch, .. } + | MapError::PmpAddr { epoch, .. } + | MapError::Pcp { epoch, .. } + | MapError::PcpUnverified { epoch, .. }, + ) => writeln!( + out, + "map proto={proto} internal={internal} requested_lifetime={} result={} external=- lease=- epoch={epoch} err={}", + opts.lifetime, + v(result), + opt(err) + ), + Err(_) => writeln!( + out, + "map proto={proto} internal={internal} requested_lifetime={} result={} external=- lease=- epoch=- err={}", + opts.lifetime, + v(result), + opt(err) + ), + } + .ok(); + + if let (Ok(m), false) = (&mapped, opts.hold.is_zero()) { + let mut h = Hold { + t, + rule: &rule, + lifetime, + mapping: m.clone(), + renewals: 0, + }; + interrupted = h.run(&mut listener, opts.hold, &mut cancel, out).await; + } + + // Delete whenever the router may hold the rule. Only a refusal tied to our MAP request + // proves it created nothing: a NAT-PMP MAP error for our port, or a PCP error that echoes + // our nonce. A timeout, a nonce-less PCP error, a capability error (the send may have + // happened) or anything else may hide a committed MAP; deleting a rule that does not + // exist is harmless. + let may_exist = !matches!( + (proto, &mapped), + (Proto::Pmp, Err(MapError::Pmp { .. })) + | (Proto::Pcp, Err(MapError::Pcp { .. })) + | (_, Err(MapError::Nonce(_))) + ); + if !may_exist { + writeln!(out, "delete proto={proto} sent=skipped reply=-").ok(); + continue; + } + let deleted = oneshot::delete(t, &rule).await; + let line = match &deleted { + Ok(()) => "sent=ok reply=ok lifetime=0".to_owned(), + Err(MapError::Cap(e)) => format!("sent=err err={} reply=-", v(e)), + Err(MapError::NoServices { .. }) => "sent=ok reply=timeout".to_owned(), + Err(MapError::Pmp { code, .. }) => format!("sent=ok reply={code}"), + Err(MapError::Pcp { code, .. }) => format!("sent=ok reply={code}"), + Err(MapError::PcpUnverified { code, .. }) => { + format!("sent=ok reply={code} err=unverified") + } + Err(e) => format!("sent=err err={} reply=-", v(e)), + }; + writeln!(out, "delete proto={proto} {line}").ok(); + if mapped.is_ok() && deleted.is_ok() { + succeeded += 1; + } + } + + let code = if succeeded == requested { + 0 + } else if succeeded > 0 { + 2 + } else { + 1 + }; + writeln!( + out, + "done exit={code} mapped_and_deleted={succeeded}/{requested}" + ) + .ok(); + code +} + +/// A mapping held open during the hold window. +struct Hold<'a, T> { + t: &'a T, + rule: &'a Rule, + lifetime: NonZeroU32, + mapping: Mapping, + renewals: u32, +} + +impl Hold<'_, T> { + /// When to renew next: every half lease, if the granted lease is no longer than the hold + /// (the router may grant less than requested). + fn renew_every(&self, dur: Duration) -> Option { + let lease = Duration::from_secs(u64::from(self.mapping.lifetime)); + (lease <= dur).then(|| (lease / 2).max(Duration::from_secs(1))) + } + + /// Renew the mapping with the same rule and report it. + async fn renew(&mut self, out: &mut dyn Write) { + self.renewals += 1; + let proto = self.rule.proto(); + let n = self.renewals; + match oneshot::map(self.t, self.rule, self.lifetime, Some(&self.mapping)).await { + Ok(m) => { + writeln!( + out, + "renew proto={proto} n={n} result=ok external={} lease={} epoch={}", + m.external, m.lifetime, m.epoch + ) + .ok(); + self.mapping = m; + } + Err(e) => { + writeln!(out, "renew proto={proto} n={n} result=error err={}", v(e)).ok(); + } + } + } + + /// Keep the mapping open for `dur`, reporting each inbound datagram's length and source. + /// Returns whether `cancel` fired. + async fn run( + &mut self, + listener: &mut crate::ProbeListener, + dur: Duration, + cancel: &mut Pin<&mut impl Future>, + out: &mut dyn Write, + ) -> bool { + let proto = self.rule.proto(); + let start = tokio::time::Instant::now(); + let end = start + dur; + let mut next_renew = self.renew_every(dur).map(|d| start + d); + if let Some(at) = next_renew { + writeln!( + out, + "lease proto={proto} warning=not-longer-than-hold lease={} renew_every={}s", + self.mapping.lifetime, + (at - start).as_secs() + ) + .ok(); + } + let mut count = 0usize; + let mut sources: Vec = Vec::new(); + let mut interrupted = false; + loop { + tokio::select! { + () = tokio::time::sleep_until(end) => break, + () = cancel.as_mut() => { + interrupted = true; + break; + } + () = tokio::time::sleep_until(next_renew.unwrap_or(end)), if next_renew.is_some_and(|r| r < end) => { + self.renew(out).await; + next_renew = self + .renew_every(dur) + .map(|d| tokio::time::Instant::now() + d); + } + r = listener.recv() => match r { + Ok((len, src)) => { + count += 1; + if count <= MAX_INBOUND_LINES { + writeln!(out, "inbound proto={proto} n={count} src={src} len={len}").ok(); + } + if sources.len() < MAX_SOURCES && !sources.contains(&src) { + sources.push(src); + } + } + Err(e) => { + writeln!(out, "inbound proto={proto} error={}", v(e)).ok(); + tokio::time::sleep(Duration::from_millis(100)).await; + } + }, + } + } + let list: Vec = sources.iter().map(ToString::to_string).collect(); + writeln!( + out, + "hold proto={proto} seconds={} inbound_datagrams={count} sources={} interrupted={interrupted} renewals={}", + dur.as_secs(), + v(list.join(",")), + self.renewals + ) + .ok(); + interrupted + } +} + +#[cfg(test)] +mod tests { + use std::net::Ipv4Addr; + + use super::*; + use crate::{ + Gateway, + testing::{ + FakeRouter, RequestKind, ResultOverride, RouterOptions, TestCap, send_datagrams, + }, + }; + + fn gateway() -> Gateway { + Gateway { + gw: Ipv4Addr::new(192, 168, 1, 1), + self_ip: Ipv4Addr::LOCALHOST, + prefix_len: 24, + if_index: 4, + if_name: "en0".to_owned(), + } + } + + async fn run_probe( + router: &FakeRouter, + opts: &ProbeOptions, + cancel: impl Future, + ) -> (i32, String) { + let cap = TestCap::new(gateway(), router, PolicyHandle::new(true)); + let mut out = Vec::new(); + let code = run_with(opts, &*cap, "discovered", cancel, &mut out).await; + (code, String::from_utf8(out).unwrap()) + } + + /// Parse `key=value` pairs of the first line starting with `tag` (and containing `filter`). + fn kv(text: &str, tag: &str, filter: &str) -> Vec<(String, String)> { + let line = text + .lines() + .find(|l| l.starts_with(&format!("{tag} ")) && l.contains(filter)) + .unwrap_or_else(|| panic!("no {tag} line with {filter:?} in:\n{text}")); + let mut pairs = Vec::new(); + let mut rest = line[tag.len() + 1..].trim(); + while !rest.is_empty() { + let (k, after) = rest.split_once('=').unwrap(); + let (val, next) = if let Some(q) = after.strip_prefix('"') { + let end = q.find('"').unwrap(); + (q[..end].to_owned(), q[end + 1..].trim_start()) + } else { + let end = after.find(' ').unwrap_or(after.len()); + (after[..end].to_owned(), after[end..].trim_start()) + }; + pairs.push((k.to_owned(), val)); + rest = next; + } + pairs + } + + fn keys(pairs: &[(String, String)]) -> Vec<&str> { + pairs.iter().map(|(k, _)| k.as_str()).collect() + } + + fn get<'a>(pairs: &'a [(String, String)], k: &str) -> &'a str { + &pairs.iter().find(|(key, _)| key == k).unwrap().1 + } + + /// The full probe against the harness: each protocol mapped and deleted separately, on a + /// nonzero ephemeral test port, with the expected output keys. + #[tokio::test] + async fn probe_maps_and_deletes_each_protocol() { + let router = FakeRouter::start(RouterOptions::default()).await.unwrap(); + let (code, out) = + run_probe(&router, &ProbeOptions::default(), std::future::pending()).await; + assert_eq!(code, 0, "{out}"); + + let env = kv(&out, "env", ""); + assert_eq!( + keys(&env), + ["os", "uid", "euid", "exe", "ppid", "xpc_service"] + ); + let gw = kv(&out, "gateway", ""); + assert_eq!(keys(&gw), ["gw", "self", "if", "prefix", "source", "bind"]); + assert_eq!(get(&gw, "if"), "en0/4"); + let probe = kv(&out, "probe", ""); + for k in [ + "pmp", + "pmp_epoch", + "pmp_ext_ip", + "pcp", + "pcp_result", + "pcp_epoch", + "errors", + ] { + assert!(keys(&probe).contains(&k), "{k} missing: {out}"); + } + assert_eq!(get(&probe, "pmp"), "true"); + assert_eq!(get(&probe, "pcp"), "true"); + + let port: u16 = get(&kv(&out, "listen", ""), "port").parse().unwrap(); + assert_ne!(port, 0); + for proto in ["pmp", "pcp"] { + let m = kv(&out, "map", &format!("proto={proto}")); + assert_eq!( + keys(&m), + [ + "proto", + "internal", + "requested_lifetime", + "result", + "external", + "lease", + "epoch", + "err" + ] + ); + assert_eq!(get(&m, "result"), "ok"); + assert_eq!(get(&m, "internal"), port.to_string()); + assert_eq!(get(&m, "requested_lifetime"), "120"); + assert_eq!(get(&m, "lease"), "120"); + let d = kv(&out, "delete", &format!("proto={proto}")); + assert_eq!(get(&d, "sent"), "ok"); + assert_eq!(get(&d, "reply"), "ok"); + } + // The deletes reached the router with the right identity: our internal port, lifetime 0. + let deletes: Vec<_> = router + .requests() + .into_iter() + .filter(|r| r.lifetime == Some(0) && r.internal.is_some()) + .collect(); + assert_eq!(deletes.len(), 2); + assert!(deletes.iter().all(|r| r.internal == Some(port))); + assert!(router.mappings().is_empty()); + let c = router.counters(); + assert_eq!( + (c.num_pmp_map_recv, c.num_pcp_map_recv, c.num_deletes), + (2, 2, 2) + ); + assert!( + out.ends_with("done exit=0 mapped_and_deleted=2/2\n"), + "{out}" + ); + router.shutdown().await; + } + + /// Hold mode reports inbound datagrams by count, source and length only. + #[tokio::test] + async fn hold_reports_inbound_without_payload() { + let router = FakeRouter::start(RouterOptions::default()).await.unwrap(); + let cap = TestCap::new(gateway(), &router, PolicyHandle::new(true)); + let opts = ProbeOptions { + protos: ProtoSet { + pmp: true, + pcp: false, + }, + test_port: 0, + hold: Duration::from_secs(1), + ..ProbeOptions::default() + }; + // Learn the test port from the router's MAP request, then send three datagrams to it. + let router_ref = &router; + let sender = async { + loop { + if let Some(p) = router_ref + .requests() + .iter() + .find(|r| r.kind == RequestKind::PmpMap) + .and_then(|r| r.internal) + { + let to = SocketAddr::from((Ipv4Addr::LOCALHOST, p)); + return send_datagrams(to, 3, 77).await.unwrap(); + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }; + let mut out = Vec::new(); + let run = run_with(&opts, &*cap, "override", std::future::pending(), &mut out); + let (code, from) = tokio::join!(run, sender); + let out = String::from_utf8(out).unwrap(); + assert_eq!(code, 0, "{out}"); + let h = kv(&out, "hold", "proto=pmp"); + assert_eq!(get(&h, "inbound_datagrams"), "3"); + assert_eq!(get(&h, "sources"), from.to_string()); + assert_eq!(get(&h, "interrupted"), "false"); + assert_eq!(out.matches("inbound proto=pmp").count(), 3); + assert!(out.contains(&format!("src={from} len=77"))); + assert!(!out.contains("ZZZZ") && !out.contains("5a5a")); + assert!(router.mappings().is_empty()); + router.shutdown().await; + } + + /// An interrupt ends the hold early, and the delete still runs. + #[tokio::test] + async fn interrupt_still_deletes() { + let router = FakeRouter::start(RouterOptions::default()).await.unwrap(); + let opts = ProbeOptions { + hold: Duration::from_secs(60), + ..ProbeOptions::default() + }; + let cancel = tokio::time::sleep(Duration::from_millis(200)); + let start = std::time::Instant::now(); + let (code, out) = run_probe(&router, &opts, cancel).await; + assert!(start.elapsed() < Duration::from_secs(10)); + assert_eq!(code, 2, "{out}"); + assert_eq!(get(&kv(&out, "hold", "proto=pmp"), "interrupted"), "true"); + assert_eq!(get(&kv(&out, "delete", "proto=pmp"), "reply"), "ok"); + assert_eq!(get(&kv(&out, "map", "proto=pcp"), "result"), "skipped"); + assert!(router.mappings().is_empty()); + router.shutdown().await; + } + + /// An interrupt during the first map (no hold) skips the second map; the first is still + /// deleted. + #[tokio::test] + async fn interrupt_without_hold_skips_later_maps() { + let router = FakeRouter::start(RouterOptions::default()).await.unwrap(); + router.set_delay(Duration::from_millis(150)); + let cancel = async { + // Fire while the NAT-PMP map is in flight (after the probe exchange). + while !router + .requests() + .iter() + .any(|r| r.kind == RequestKind::PmpMap) + { + tokio::time::sleep(Duration::from_millis(5)).await; + } + }; + let (code, out) = run_probe(&router, &ProbeOptions::default(), cancel).await; + assert_eq!(code, 2, "{out}"); + assert_eq!(get(&kv(&out, "map", "proto=pmp"), "result"), "ok"); + assert_eq!(get(&kv(&out, "delete", "proto=pmp"), "reply"), "ok"); + assert_eq!(get(&kv(&out, "map", "proto=pcp"), "result"), "skipped"); + assert_eq!(router.counters().num_pcp_map_recv, 0); + assert!(router.mappings().is_empty()); + router.shutdown().await; + } + + /// A lease the router grants no longer than the hold is renewed every half lease, so the + /// rule does not expire mid-hold. + #[tokio::test] + async fn short_lease_is_renewed_during_the_hold() { + let router = FakeRouter::start(RouterOptions::default()).await.unwrap(); + router.set_lifetime_override(Some(2)); + let opts = ProbeOptions { + protos: ProtoSet { + pmp: true, + pcp: false, + }, + hold: Duration::from_secs(3), + ..ProbeOptions::default() + }; + let (code, out) = run_probe(&router, &opts, std::future::pending()).await; + assert_eq!(code, 0, "{out}"); + let l = kv(&out, "lease", "proto=pmp"); + assert_eq!(get(&l, "warning"), "not-longer-than-hold"); + assert_eq!(get(&l, "renew_every"), "1s"); + assert_eq!(get(&kv(&out, "renew", "n=1"), "result"), "ok"); + let renewals: u32 = get(&kv(&out, "hold", "seconds="), "renewals") + .parse() + .unwrap(); + assert!(renewals >= 2, "{out}"); + // Create + renewals + delete, all for the test port. + let maps = router + .requests() + .iter() + .filter(|r| r.kind == RequestKind::PmpMap) + .count(); + assert_eq!(maps, 2 + usize::try_from(renewals).unwrap()); + assert!(router.mappings().is_empty()); + router.shutdown().await; + } + + /// A lease longer than the hold is not renewed. + #[tokio::test] + async fn long_lease_is_not_renewed() { + let router = FakeRouter::start(RouterOptions::default()).await.unwrap(); + let opts = ProbeOptions { + protos: ProtoSet { + pmp: true, + pcp: false, + }, + hold: Duration::from_secs(1), + ..ProbeOptions::default() + }; + let (code, out) = run_probe(&router, &opts, std::future::pending()).await; + assert_eq!(code, 0, "{out}"); + assert!( + !out.contains("lease proto=") && !out.contains("renew "), + "{out}" + ); + assert_eq!(get(&kv(&out, "hold", "seconds="), "renewals"), "0"); + router.shutdown().await; + } + + /// A lookup that finds no gateway for its first `misses` calls. + struct LateLookup { + misses: std::sync::atomic::AtomicU32, + } + + impl GatewayLookup for LateLookup { + fn lookup(&self) -> Result, LookupError> { + use std::sync::atomic::Ordering; + let left = self.misses.load(Ordering::SeqCst); + if left == 0 { + return Ok(Some(gateway())); + } + self.misses.store(left - 1, Ordering::SeqCst); + Ok(None) + } + } + + /// `--portmap-wait`: the lookup is retried every second until a gateway appears or the wait + /// ends, and the wait is reported. + #[tokio::test] + async fn gateway_wait_retries_until_routes_appear() { + let late = |misses| LateLookup { + misses: std::sync::atomic::AtomicU32::new(misses), + }; + let mut never = std::pin::pin!(std::future::pending::<()>()); + let mut out = Vec::new(); + let g = wait_for_gateway( + Arc::new(late(2)), + Duration::from_secs(10), + &mut never, + &mut out, + ) + .await; + assert_eq!(g.unwrap(), gateway()); + assert_eq!( + String::from_utf8(out).unwrap(), + "gateway wait=2s attempts=3\n" + ); + + let mut out = Vec::new(); + let g = wait_for_gateway( + Arc::new(late(100)), + Duration::from_secs(3), + &mut never, + &mut out, + ) + .await; + assert!(matches!(g, Err(Some(Ok(None))))); + assert_eq!( + String::from_utf8(out).unwrap(), + "gateway wait=3s attempts=4\n" + ); + + // No wait: one attempt, no wait line. + let mut out = Vec::new(); + let g = wait_for_gateway(Arc::new(late(1)), Duration::ZERO, &mut never, &mut out).await; + assert!(matches!(g, Err(Some(Ok(None))))); + assert!(out.is_empty()); + + // An interrupt ends the wait. + let mut soon = std::pin::pin!(tokio::time::sleep(Duration::from_millis(1500))); + let mut out = Vec::new(); + let g = wait_for_gateway( + Arc::new(late(100)), + Duration::from_secs(60), + &mut soon, + &mut out, + ) + .await; + assert!(matches!(g, Err(None))); + assert!(String::from_utf8(out).unwrap().contains("interrupted=true")); + } + + /// Partial success exits 2; a refused map skips its delete. + #[tokio::test] + async fn partial_and_refused() { + let router = FakeRouter::start(RouterOptions::default()).await.unwrap(); + router.set_result( + RequestKind::PcpMap, + Some(ResultOverride { + code: 2, + lifetime: 600, + }), + ); + let (code, out) = + run_probe(&router, &ProbeOptions::default(), std::future::pending()).await; + assert_eq!(code, 2, "{out}"); + let m = kv(&out, "map", "proto=pcp"); + assert_eq!(get(&m, "result"), "not-authorized(2)"); + assert_eq!(get(&kv(&out, "delete", "proto=pcp"), "sent"), "skipped"); + router.shutdown().await; + } + + /// A NAT-PMP error to the public-address request (the WAN is down) is not a refusal of the + /// MAP sent alongside it: the delete runs and the committed rule is gone. + #[tokio::test] + async fn pmp_address_error_still_deletes() { + let router = FakeRouter::start(RouterOptions::default()).await.unwrap(); + let opts = ProbeOptions { + protos: ProtoSet { + pmp: true, + pcp: false, + }, + ..ProbeOptions::default() + }; + let cap = TestCap::new(gateway(), &router, PolicyHandle::new(true)); + let mut out = Vec::new(); + let b = router.barrier(RequestKind::PmpPublicAddr); + let run = run_with(&opts, &*cap, "discovered", std::future::pending(), &mut out); + let arm = async { + // The probe's address request is paused; fail every later one. + b.reached().await; + router.set_result( + RequestKind::PmpPublicAddr, + Some(ResultOverride { + code: 3, + lifetime: 0, + }), + ); + b.release(); + }; + let (code, ()) = tokio::join!(run, arm); + let out = String::from_utf8(out).unwrap(); + assert_eq!(code, 1, "{out}"); + assert_eq!(get(&kv(&out, "probe", ""), "pmp"), "true", "{out}"); + assert_eq!( + get(&kv(&out, "map", "proto=pmp"), "result"), + "network-failure(3)" + ); + assert_eq!(get(&kv(&out, "delete", "proto=pmp"), "reply"), "ok"); + assert!(router.mappings().is_empty(), "{out}"); + router.shutdown().await; + } + + /// A spoofed nonce-less PCP error that beats a lost real reply is not a refusal: the MAP may + /// have committed, so the delete still runs and the rule is gone. + #[tokio::test] + async fn unverified_pcp_error_still_deletes() { + let router = FakeRouter::start(RouterOptions::default()).await.unwrap(); + let mut spoof = vec![0u8; 24]; + spoof[0] = crate::pcp::VERSION; + spoof[1] = crate::pcp::R_BIT | crate::pcp::OP_MAP; + spoof[3] = 2; + // Only the MAP: the probe's ANNOUNCE goes first. + router.knobs_for(Some(RequestKind::PcpMap)); + router.spoof_before_next_reply(spoof); + router.commit_then_drop_next(1); + let opts = ProbeOptions { + protos: ProtoSet { + pmp: false, + pcp: true, + }, + ..ProbeOptions::default() + }; + let (code, out) = run_probe(&router, &opts, std::future::pending()).await; + assert_eq!(code, 1, "{out}"); + assert_eq!( + get(&kv(&out, "map", "proto=pcp"), "result"), + "not-authorized(2)" + ); + assert_eq!(get(&kv(&out, "delete", "proto=pcp"), "sent"), "ok"); + assert!(router.mappings().is_empty(), "{out}"); + router.shutdown().await; + } + + /// No responder: exit 1; the timed-out map is still followed by a delete attempt. + #[tokio::test] + async fn no_responder_exits_1() { + let router = FakeRouter::start(RouterOptions { + pmp: false, + pcp: false, + ..RouterOptions::default() + }) + .await + .unwrap(); + let (code, out) = + run_probe(&router, &ProbeOptions::default(), std::future::pending()).await; + assert_eq!(code, 1, "{out}"); + assert_eq!(get(&kv(&out, "probe", ""), "pmp"), "false"); + assert_eq!(get(&kv(&out, "map", "proto=pmp"), "result"), "timeout"); + assert_eq!(get(&kv(&out, "delete", "proto=pmp"), "reply"), "timeout"); + router.shutdown().await; + } + + /// Injected EHOSTUNREACH / EPERM are printed verbatim. + #[cfg(unix)] + #[tokio::test] + async fn os_errors_are_printed_verbatim() { + for (code, name) in [(libc::EHOSTUNREACH, "EHOSTUNREACH"), (libc::EPERM, "EPERM")] { + let router = FakeRouter::start(RouterOptions::default()).await.unwrap(); + let cap = TestCap::new(gateway(), &router, PolicyHandle::new(true)); + cap.inject_send_error(Some(code)); + let mut out = Vec::new(); + let exit = run_with( + &ProbeOptions::default(), + &*cap, + "discovered", + std::future::pending(), + &mut out, + ) + .await; + let out = String::from_utf8(out).unwrap(); + assert_eq!(exit, 1); + let err = get(&kv(&out, "probe", ""), "error").to_owned(); + assert!(err.starts_with(&format!("send: {name}: ")), "{out}"); + assert!(err.contains(&format!("os error {code}")), "{out}"); + assert_eq!(router.counters().num_total_recv, 0); + router.shutdown().await; + } + } + + /// The test port never equals a node port. + #[test] + fn reserved_ports_are_refused() { + for port in [41641, 41642, 50000] { + let opts = ProbeOptions { + test_port: port, + node_port: 50000, + ..ProbeOptions::default() + }; + assert!(opts.validate().is_err(), "{port}"); + } + assert!( + ProbeOptions { + test_port: 50001, + node_port: 50000, + ..ProbeOptions::default() + } + .validate() + .is_ok() + ); + for bad in [ + ProbeOptions { + lifetime: 0, + ..ProbeOptions::default() + }, + ProbeOptions { + lifetime: 7201, + ..ProbeOptions::default() + }, + ProbeOptions { + hold: Duration::from_secs(601), + lifetime: 7200, + ..ProbeOptions::default() + }, + ProbeOptions { + hold: Duration::from_secs(120), + ..ProbeOptions::default() + }, + ProbeOptions { + wait: Duration::from_secs(121), + ..ProbeOptions::default() + }, + ProbeOptions { + protos: ProtoSet { + pmp: false, + pcp: false, + }, + ..ProbeOptions::default() + }, + ] { + assert!(bad.validate().is_err(), "{bad:?}"); + } + } + + /// A reserved test port is refused before any socket or packet. + #[tokio::test] + async fn reserved_port_refused_before_io() { + let router = FakeRouter::start(RouterOptions::default()).await.unwrap(); + let opts = ProbeOptions { + test_port: 41641, + ..ProbeOptions::default() + }; + let (code, out) = run_probe(&router, &opts, std::future::pending()).await; + assert_eq!(code, 1); + assert!(out.contains("error msg="), "{out}"); + assert_eq!(router.counters().num_total_recv, 0); + router.shutdown().await; + } + + #[test] + fn values_are_quoted_when_needed() { + assert_eq!(v("ok"), "ok"); + assert_eq!(v(""), "-"); + assert_eq!(v("a b"), "\"a b\""); + assert_eq!(v("k=v"), "\"k=v\""); + assert!(env_line().starts_with("env os=")); + } +} diff --git a/ts_portmapper/src/testing/cap.rs b/ts_portmapper/src/testing/cap.rs new file mode 100644 index 00000000..f693b13e --- /dev/null +++ b/ts_portmapper/src/testing/cap.rs @@ -0,0 +1,203 @@ +//! `TestCap`: the loopback test transport. +//! +//! It runs the same exchange loop as [`crate::GatewayCap`] (same send gate, same source check, +//! same receive caps), but sends to a [`FakeRouter`] on loopback and presents the router's +//! replies as coming from the logical `gateway:5351`. Two routers can stand in for two gateways +//! that both listen on port 5351. + +use std::{ + io, + net::{Ipv4Addr, SocketAddr, SocketAddrV4}, + sync::{Arc, Mutex}, +}; + +use tokio::time::Instant; + +use super::{FakeRouter, RouterEndpoint}; +use crate::{ + Gateway, PXP_PORT, + cap::{ + BindMode, CapError, ExchangeStats, Flow, LanDest, PolicyHandle, ProbeListener, + PxpTransport, SendGate, exchange_on, sealed, + }, +}; + +/// The address a reply "from the wrong address" is presented as (TEST-NET-1). +pub const WRONG_IP: Ipv4Addr = Ipv4Addr::new(192, 0, 2, 66); + +/// A loopback transport for one logical gateway. See the module docs. +pub struct TestCap { + gateway: Gateway, + to: SocketAddr, + bind_ip: Ipv4Addr, + aliases: Vec<(SocketAddr, SocketAddrV4)>, + gate: SendGate, + inject_send_error: Mutex>, + panic_next: std::sync::atomic::AtomicBool, +} + +impl std::fmt::Debug for TestCap { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TestCap") + .field("gateway", &self.gateway) + .field("to", &self.to) + .finish_non_exhaustive() + } +} + +impl TestCap { + /// A transport for the logical `gateway` backed by `router`. The PCP client IP the client + /// sends is `gateway.self_ip`; the packets' real source is the router's loopback address, so + /// `self_ip = 127.0.0.1` is the source-consistent mode. + /// + /// # Panics + /// + /// If the router is not on loopback (it never is: [`FakeRouter::start`] refuses). + pub fn new(gateway: Gateway, router: &FakeRouter, policy: PolicyHandle) -> Arc { + Self::with_endpoint(gateway, &router.endpoint(), policy) + } + + /// [`TestCap::new`] from a router's detached [`RouterEndpoint`]. + /// + /// # Panics + /// + /// If the endpoint is not on loopback. + pub fn with_endpoint( + gateway: Gateway, + router: &RouterEndpoint, + policy: PolicyHandle, + ) -> Arc { + assert!( + router.addr.ip().is_loopback(), + "TestCap only talks to a loopback router" + ); + let gw = gateway.gw; + let bind_ip = match router.addr { + SocketAddr::V4(a) => *a.ip(), + SocketAddr::V6(_) => Ipv4Addr::LOCALHOST, + }; + Arc::new(Self { + aliases: vec![ + (router.addr, SocketAddrV4::new(gw, PXP_PORT)), + ( + router.wrong_port_addr, + SocketAddrV4::new(gw, PXP_PORT.wrapping_add(1)), + ), + (router.wrong_ip_addr, SocketAddrV4::new(WRONG_IP, PXP_PORT)), + ], + to: router.addr, + bind_ip, + gateway, + gate: SendGate::new(policy, None), + inject_send_error: Mutex::new(None), + panic_next: std::sync::atomic::AtomicBool::new(false), + }) + } + + /// Make the next exchange panic (to show that single-flight survives a panicking attempt). + pub fn panic_next_exchange(&self) { + self.panic_next + .store(true, std::sync::atomic::Ordering::SeqCst); + } + + /// Make every later send fail with this OS error (`None` to stop). + pub fn inject_send_error(&self, raw_os_error: Option) { + *self + .inject_send_error + .lock() + .unwrap_or_else(|p| p.into_inner()) = raw_os_error; + } + + /// Refuse new work, like [`crate::GatewayCap::set_closing`]. + pub fn set_closing(&self) { + self.gate.set_closing(); + } + + fn injected(&self) -> Result<(), CapError> { + let code = *self + .inject_send_error + .lock() + .unwrap_or_else(|p| p.into_inner()); + match code { + Some(c) => Err(CapError::io("send", &io::Error::from_raw_os_error(c))), + None => Ok(()), + } + } + + fn present(&self, src: SocketAddr) -> Option { + self.aliases + .iter() + .find(|(real, _)| *real == src) + .map(|(_, shown)| *shown) + } +} + +impl sealed::Sealed for TestCap {} + +impl PxpTransport for TestCap { + fn gateway(&self) -> &Gateway { + &self.gateway + } + + fn bind_mode(&self) -> BindMode { + BindMode::LoopbackTest + } + + async fn exchange( + &self, + dest: LanDest, + reqs: &[&[u8]], + deadline: Instant, + on_reply: &mut (dyn FnMut(&[u8]) -> Flow + Send), + ) -> Result { + let LanDest::Pxp = dest; + self.gate.check()?; + self.injected()?; + assert!( + !self + .panic_next + .swap(false, std::sync::atomic::Ordering::SeqCst), + "TestCap: injected panic" + ); + let sock = tokio::net::UdpSocket::bind((self.bind_ip, 0)) + .await + .map_err(|e| CapError::io("bind", &e))?; + let expected = SocketAddrV4::new(self.gateway.gw, PXP_PORT); + let present = |src| self.present(src); + exchange_on( + &sock, &self.gate, self.to, expected, &present, reqs, deadline, on_reply, + ) + .await + } + + fn send_nowait(&self, dest: LanDest, req: &[u8]) -> Result<(), CapError> { + let LanDest::Pxp = dest; + self.gate.check()?; + self.injected()?; + let s = + std::net::UdpSocket::bind((self.bind_ip, 0)).map_err(|e| CapError::io("bind", &e))?; + s.set_nonblocking(true) + .map_err(|e| CapError::io("bind", &e))?; + self.gate.send(|| s.send_to(req, self.to)).map(drop) + } + + fn bind_probe_listener(&self, port: u16) -> Result { + if tokio::runtime::Handle::try_current().is_err() { + return Err(CapError::NoRuntime); + } + let s = std::net::UdpSocket::bind((self.bind_ip, port)) + .map_err(|e| CapError::io("bind", &e))?; + s.set_nonblocking(true) + .map_err(|e| CapError::io("bind", &e))?; + let s = tokio::net::UdpSocket::from_std(s).map_err(|e| CapError::io("bind", &e))?; + ProbeListener::new(s) + } + + fn invalidate(&self) { + self.gate.invalidate(); + } + + fn is_valid(&self) -> bool { + self.gate.is_valid() + } +} diff --git a/ts_portmapper/src/testing/clock.rs b/ts_portmapper/src/testing/clock.rs new file mode 100644 index 00000000..6d7c8a89 --- /dev/null +++ b/ts_portmapper/src/testing/clock.rs @@ -0,0 +1,28 @@ +//! A manual lease clock in whole seconds, shared by the fake router (mapping expiry, epoch) and +//! the tests that drive it. + +use std::sync::{ + Arc, + atomic::{AtomicU64, Ordering}, +}; + +/// A manually advanced clock, in seconds. +#[derive(Debug, Clone, Default)] +pub struct LeaseClock(Arc); + +impl LeaseClock { + /// A clock reading `start` seconds. + pub fn new(start: u64) -> Self { + Self(Arc::new(AtomicU64::new(start))) + } + + /// The current reading. + pub fn now(&self) -> u64 { + self.0.load(Ordering::SeqCst) + } + + /// Move the clock forward by `secs`. + pub fn advance(&self, secs: u64) { + self.0.fetch_add(secs, Ordering::SeqCst); + } +} diff --git a/ts_portmapper/src/testing/mod.rs b/ts_portmapper/src/testing/mod.rs new file mode 100644 index 00000000..3384db0c --- /dev/null +++ b/ts_portmapper/src/testing/mod.rs @@ -0,0 +1,48 @@ +//! Test harness: a fake NAT-PMP/PCP router on loopback and `TestCap`, a loopback transport. +//! +//! Compiled only for this crate's tests and with the `test-harness` feature, which only +//! dev-dependencies may enable (the `portmapper_lan_only` check enforces it). Nothing here can +//! reach a real gateway: [`FakeRouter::start`] refuses a non-loopback bind address, `TestCap` +//! asserts its router is on loopback and sends only to it, and [`send_datagrams`] refuses a +//! non-loopback destination. +//! +//! The router decodes requests with its own decoder written from RFC 6886 and RFC 6887; it does +//! not reuse [`crate::pmp`] or [`crate::pcp`], so a codec bug cannot hide behind a matching +//! harness bug. + +mod cap; +mod clock; +mod router; +#[cfg(test)] +mod tests; + +use std::net::{Ipv4Addr, SocketAddr}; + +pub use cap::TestCap; +pub use clock::LeaseClock; +pub use router::{ + AssignPort, Barrier, Corruption, Counters, DecodedRequest, EpochMode, FakeRouter, MappingEntry, + RequestKind, ResultOverride, RouterEndpoint, RouterOptions, +}; + +/// Send `count` datagrams of `len` bytes to `to` from a loopback socket (for the probe's hold +/// window). Returns the sender's address. A non-loopback `to` is refused with +/// [`std::io::ErrorKind::InvalidInput`]. +pub async fn send_datagrams( + to: SocketAddr, + count: usize, + len: usize, +) -> std::io::Result { + if !to.ip().is_loopback() { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "send_datagrams sends to loopback only", + )); + } + let s = tokio::net::UdpSocket::bind((Ipv4Addr::LOCALHOST, 0)).await?; + let payload = vec![0x5a; len]; + for _ in 0..count { + s.send_to(&payload, to).await?; + } + s.local_addr() +} diff --git a/ts_portmapper/src/testing/router.rs b/ts_portmapper/src/testing/router.rs new file mode 100644 index 00000000..45e9e8eb --- /dev/null +++ b/ts_portmapper/src/testing/router.rs @@ -0,0 +1,1131 @@ +//! A fake NAT-PMP (RFC 6886) and PCP (RFC 6887) router on loopback. +//! +//! Its request decoder is written from the RFCs and shares no code with the crate's codecs. It +//! keeps a mapping table keyed by `(client ip, protocol, internal port)` with PCP nonce +//! ownership, expires entries on a [`LeaseClock`], and has runtime knobs for the failure modes the +//! client must survive: delays, drops, duplicates, reordering, commit-then-drop-reply, replies +//! from the wrong port or address, corrupted replies, junk bursts and trickles, result-code +//! overrides and request barriers. +//! +//! Requests are decoded and committed one at a time, in arrival order, under the state lock. Each +//! reply (barrier wait, delay, junk, send, duplicate) then runs in its own task, so a paused or +//! delayed reply holds back only itself: a later request is still decoded, committed and answered. +//! [`FakeRouter::shutdown`] aborts and joins every reply task. + +use std::{ + collections::{HashMap, VecDeque}, + net::{Ipv4Addr, Ipv6Addr, SocketAddr}, + sync::{Arc, Mutex, MutexGuard}, + time::Duration, +}; + +use tokio::{ + net::UdpSocket, + sync::{Notify, oneshot}, + task::{JoinHandle, JoinSet}, +}; + +use super::LeaseClock; + +/// How the router's epoch ("seconds since start of epoch") evolves. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EpochMode { + /// Always this value. + Fixed(u32), + /// `clock.now() - zero_at`: advances with the lease clock. + Clock { + /// Clock reading at which the epoch was 0. + zero_at: u64, + }, +} + +/// How the router assigns external ports to new mappings. A port already used by another +/// mapping is bumped to the next free one. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AssignPort { + /// The suggested port if nonzero, else `fallback`. + Suggested { + /// Port used when no port is suggested. + fallback: u16, + }, + /// Always this port. + Fixed(u16), + /// The internal port plus this offset (so external ≠ internal). + InternalPlus(u16), +} + +/// A forced result for one kind of request. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ResultOverride { + /// The result code to answer with (PMP codes are u16, PCP codes u8). + pub code: u16, + /// The lifetime in the reply (PCP: the error lifetime). + pub lifetime: u32, +} + +/// A one-shot corruption applied to the next reply. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Corruption { + /// Drop the last byte. + Truncate, + /// Change the version byte. + BadVersion, + /// Change the opcode. + WrongOpcode, + /// Clear the R bit (PCP) or reply bit (PMP). + NoRBit, + /// Change the PCP nonce. + WrongNonce, + /// Change the PCP protocol to TCP. + WrongProto, + /// Change the internal port. + WrongInternalPort, + /// Replace the reply with garbage of the same length. + Garbage, + /// Pad the reply to 1200 bytes (over the 1101-byte receive buffer). + Oversize, +} + +/// The kind of a decoded request. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum RequestKind { + /// NAT-PMP public-address request. + PmpPublicAddr, + /// NAT-PMP MAP (create, renew or delete). + PmpMap, + /// PCP ANNOUNCE. + PcpAnnounce, + /// PCP MAP (create, renew or delete). + PcpMap, + /// Anything else. + Other, +} + +/// Request counters. Names follow Go's `igdCounters` where they exist. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Counters { + /// Every datagram that reached the router's socket. + pub num_total_recv: u64, + /// NAT-PMP requests handled. + pub num_pmp_recv: u64, + /// NAT-PMP public-address requests. + pub num_pmp_public_addr_recv: u64, + /// NAT-PMP MAP requests. + pub num_pmp_map_recv: u64, + /// Malformed NAT-PMP requests. + pub num_pmp_bogus_recv: u64, + /// PCP requests handled. + pub num_pcp_recv: u64, + /// PCP ANNOUNCE requests. + pub num_pcp_disco_recv: u64, + /// PCP MAP requests. + pub num_pcp_map_recv: u64, + /// PCP requests with other opcodes or malformed headers. + pub num_pcp_other_recv: u64, + /// Malformed PCP MAP requests. + pub invalid_pcp_map_pkt: u64, + /// Delete requests (PMP or PCP lifetime 0). + pub num_deletes: u64, +} + +/// A request as the router decoded it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DecodedRequest { + /// Where it came from. + pub src: SocketAddr, + /// What it was. + pub kind: RequestKind, + /// Version byte. + pub version: u8, + /// Opcode (PCP: without the R bit). + pub opcode: u8, + /// MAP internal port. + pub internal: Option, + /// MAP suggested external port. + pub suggested_external: Option, + /// Requested lifetime. + pub lifetime: Option, + /// PCP MAP nonce. + pub nonce: Option<[u8; 12]>, + /// PCP client IP field, if IPv4-mapped. + pub pcp_client_ip: Option, +} + +/// One live mapping. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MappingEntry { + /// The owner's address (the request's source address). + pub client_ip: Ipv4Addr, + /// IANA protocol (17 for UDP). + pub proto: u8, + /// Internal port. + pub internal: u16, + /// Assigned external port. + pub external: u16, + /// PCP nonce that owns it (None if created by NAT-PMP). + pub nonce: Option<[u8; 12]>, + /// Lease-clock second at which it expires. + pub expires_at: u64, +} + +/// Options for [`FakeRouter::start`]. +#[derive(Debug, Clone)] +pub struct RouterOptions { + /// Answer NAT-PMP. + pub pmp: bool, + /// Answer PCP. + pub pcp: bool, + /// The public address it reports. + pub public_ip: Ipv4Addr, + /// Loopback address to bind (port 0 is always used). Anything else is refused by + /// [`FakeRouter::start`]. + pub bind_ip: Ipv4Addr, + /// Lease clock for expiry and epoch. + pub clock: LeaseClock, + /// Epoch behaviour. + pub epoch: EpochMode, + /// Answer PCP requests whose client IP field differs from the source address with + /// `ADDRESS_MISMATCH`. + pub enforce_pcp_client_ip: bool, + /// Send PCP errors as header-only (24-byte) responses. + pub short_pcp_errors: bool, +} + +impl Default for RouterOptions { + fn default() -> Self { + Self { + pmp: true, + pcp: true, + public_ip: Ipv4Addr::new(203, 0, 113, 7), + bind_ip: Ipv4Addr::LOCALHOST, + clock: LeaseClock::new(1_000_000), + epoch: EpochMode::Clock { + zero_at: 1_000_000 - 3600, + }, + enforce_pcp_client_ip: false, + short_pcp_errors: false, + } + } +} + +struct BarrierInner { + reached: Arc, + release: oneshot::Receiver<()>, +} + +/// Pauses the router before it replies to the next request of one kind. +#[derive(Debug)] +pub struct Barrier { + reached: Arc, + release: Option>, +} + +impl Barrier { + /// Wait until the router has received (and committed) the request and paused. + pub async fn reached(&self) { + self.reached.notified().await; + } + + /// Let the router send its reply. + pub fn release(mut self) { + if let Some(tx) = self.release.take() { + tx.send(()).ok(); + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Via { + Main, + WrongPort, + WrongIp, +} + +struct State { + opts: RouterOptions, + table: Vec, + overrides: HashMap, + lifetime_override: Option, + assign: AssignPort, + delay: Duration, + drop_next: u32, + drop_deletes: bool, + duplicate_next: u32, + reorder_next: bool, + commit_then_drop_next: u32, + wrong_port_next: u32, + wrong_ip_next: u32, + corrupt: VecDeque, + junk_before_next: usize, + trickle_next: Option<(usize, Duration)>, + spoof_next: Option>, + knobs_for: Option, + counters: Counters, + requests: Vec, + barriers: Vec<(RequestKind, BarrierInner)>, + held: Option<(Vec, SocketAddr)>, +} + +struct ReplyPlan { + reply: Vec, + /// The uncorrupted reply, sent after `reply` when duplicating. + original: Vec, + barrier: Option, + delay: Duration, + drop: bool, + via: Via, + duplicate: bool, + junk: usize, + /// Send this many junk datagrams this far apart, then drop the real reply. + trickle: Option<(usize, Duration)>, + /// Sent from the router's own address just before the real reply. + spoof: Option>, + /// The reply was parked in `State::held`; this task sends nothing. + hold: bool, +} + +impl ReplyPlan { + /// A reply with no one-shot knob applied. + fn plain(reply: Vec, barrier: Option, delay: Duration) -> Self { + Self { + original: reply.clone(), + reply, + barrier, + delay, + drop: false, + via: Via::Main, + duplicate: false, + junk: 0, + trickle: None, + spoof: None, + hold: false, + } + } +} + +/// A router's loopback addresses (see [`FakeRouter::endpoint`]). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RouterEndpoint { + /// Where requests go and replies come from. + pub addr: SocketAddr, + /// Where replies "from the wrong port" come from. + pub wrong_port_addr: SocketAddr, + /// Where replies "from the wrong address" come from. + pub wrong_ip_addr: SocketAddr, +} + +/// A fake NAT-PMP/PCP router. See the module docs. +pub struct FakeRouter { + addr: SocketAddr, + wrong_port_addr: SocketAddr, + wrong_ip_addr: SocketAddr, + state: Arc>, + shutdown: Arc, + task: Option>, +} + +impl std::fmt::Debug for FakeRouter { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("FakeRouter") + .field("addr", &self.addr) + .finish_non_exhaustive() + } +} + +fn be16(b: &[u8], at: usize) -> u16 { + b.get(at..at + 2) + .map_or(0, |s| u16::from_be_bytes([s[0], s[1]])) +} + +fn be32(b: &[u8], at: usize) -> u32 { + b.get(at..at + 4) + .map_or(0, |s| u32::from_be_bytes([s[0], s[1], s[2], s[3]])) +} + +fn v4_mapped(ip: Ipv4Addr) -> [u8; 16] { + ip.to_ipv6_mapped().octets() +} + +impl State { + fn epoch(&self) -> u32 { + match self.opts.epoch { + EpochMode::Fixed(e) => e, + EpochMode::Clock { zero_at } => { + u32::try_from(self.opts.clock.now().saturating_sub(zero_at)).unwrap_or(u32::MAX) + } + } + } + + fn find(&self, client: Ipv4Addr, proto: u8, internal: u16) -> Option { + self.table + .iter() + .position(|e| e.client_ip == client && e.proto == proto && e.internal == internal) + } + + fn assign(&self, internal: u16, suggested: u16) -> u16 { + let mut p = match self.assign { + AssignPort::Suggested { fallback } if suggested == 0 => fallback, + AssignPort::Suggested { .. } => suggested, + AssignPort::Fixed(p) => p, + AssignPort::InternalPlus(d) => internal.wrapping_add(d), + }; + for _ in 0..u16::MAX { + if p != 0 && !self.table.iter().any(|e| e.external == p) { + break; + } + p = p.wrapping_add(1); + } + p + } + + fn granted(&self, requested: u32) -> u32 { + self.lifetime_override.unwrap_or(requested) + } + + /// Create or renew `(client, proto, internal)`; returns the external port and lifetime. + fn upsert( + &mut self, + client: Ipv4Addr, + proto: u8, + internal: u16, + suggested: u16, + requested: u32, + nonce: Option<[u8; 12]>, + ) -> (u16, u32) { + let lifetime = self.granted(requested); + let expires_at = self.opts.clock.now().saturating_add(u64::from(lifetime)); + if let Some(i) = self.find(client, proto, internal) { + let e = &mut self.table[i]; + e.expires_at = expires_at; + if e.nonce.is_none() { + e.nonce = nonce; + } + return (e.external, lifetime); + } + let external = self.assign(internal, suggested); + self.table.push(MappingEntry { + client_ip: client, + proto, + internal, + external, + nonce, + expires_at, + }); + (external, lifetime) + } + + fn handle(&mut self, req: &[u8], src: SocketAddr) -> Option { + self.counters.num_total_recv += 1; + let now = self.opts.clock.now(); + self.table.retain(|e| e.expires_at > now); + if self.drop_next > 0 { + self.drop_next -= 1; + return None; + } + let client = match src { + SocketAddr::V4(a) => *a.ip(), + SocketAddr::V6(_) => return None, + }; + let (kind, reply) = match req.first() { + Some(0) => self.handle_pmp(req, src, client)?, + Some(2) => self.handle_pcp(req, src, client)?, + _ => { + // An unknown version: a NAT-PMP server answers "unsupported version". + if !self.opts.pmp || req.len() < 2 { + return None; + } + (RequestKind::Other, self.pmp_error_reply(req[1], 1)) + } + }; + let original = reply.clone(); + let mut reply = reply; + let barrier = self + .barriers + .iter() + .position(|(k, _)| *k == kind) + .map(|i| self.barriers.remove(i).1); + if self.knobs_for.is_some_and(|k| k != kind) { + // The one-shot knobs wait for a request of their kind. + return Some(ReplyPlan::plain(reply, barrier, self.delay)); + } + if let Some(c) = self.corrupt.pop_front() { + corrupt(&mut reply, c); + } + let take = |n: &mut u32| { + let hit = *n > 0; + *n = n.saturating_sub(1); + hit + }; + let dropped_delete = self.drop_deletes + && match req.first() { + Some(0) => req.len() == 12 && be32(req, 8) == 0, + Some(2) => req.len() >= 60 && req[1] == 1 && be32(req, 4) == 0, + _ => false, + }; + let drop = take(&mut self.commit_then_drop_next) || dropped_delete; + let hold = std::mem::take(&mut self.reorder_next); + if hold { + // Parked now, in arrival order, so the next request's reply task finds it. + self.held = Some((reply.clone(), src)); + } + let via = if take(&mut self.wrong_port_next) { + Via::WrongPort + } else if take(&mut self.wrong_ip_next) { + Via::WrongIp + } else { + Via::Main + }; + Some(ReplyPlan { + reply, + original, + barrier, + delay: self.delay, + drop, + via, + duplicate: take(&mut self.duplicate_next), + junk: std::mem::take(&mut self.junk_before_next), + trickle: self.trickle_next.take(), + spoof: self.spoof_next.take(), + hold, + }) + } + + /// An 8-byte NAT-PMP error reply. + fn pmp_error_reply(&self, op: u8, code: u16) -> Vec { + let mut r = vec![0, op | 0x80]; + r.extend_from_slice(&code.to_be_bytes()); + r.extend_from_slice(&self.epoch().to_be_bytes()); + r + } + + fn handle_pmp( + &mut self, + req: &[u8], + src: SocketAddr, + client: Ipv4Addr, + ) -> Option<(RequestKind, Vec)> { + if req.len() < 2 { + self.counters.num_pmp_bogus_recv += 1; + return None; + } + if !self.opts.pmp { + return None; + } + self.counters.num_pmp_recv += 1; + let op = req[1]; + let mut log = DecodedRequest { + src, + kind: RequestKind::Other, + version: 0, + opcode: op, + internal: None, + suggested_external: None, + lifetime: None, + nonce: None, + pcp_client_ip: None, + }; + let epoch = self.epoch(); + let out = match op { + 0 => { + self.counters.num_pmp_public_addr_recv += 1; + log.kind = RequestKind::PmpPublicAddr; + let code = self.overrides.get(&log.kind).map_or(0, |o| o.code); + let mut r = vec![0, 0x80]; + r.extend_from_slice(&code.to_be_bytes()); + r.extend_from_slice(&epoch.to_be_bytes()); + let ip = if code == 0 { + self.opts.public_ip + } else { + Ipv4Addr::UNSPECIFIED + }; + r.extend_from_slice(&ip.octets()); + r + } + 1 | 2 => { + if req.len() != 12 { + self.counters.num_pmp_bogus_recv += 1; + return None; + } + self.counters.num_pmp_map_recv += 1; + log.kind = RequestKind::PmpMap; + let proto = if op == 1 { 17 } else { 6 }; + let internal = be16(req, 4); + let suggested = be16(req, 6); + let lifetime = be32(req, 8); + log.internal = Some(internal); + log.suggested_external = Some(suggested); + log.lifetime = Some(lifetime); + let (code, ext, granted) = if lifetime == 0 && self.drop_deletes { + (0, 0, 0) // logged and barrier-observable, but never committed or replied + } else if let Some(o) = self.overrides.get(&log.kind) { + (o.code, 0, o.lifetime) + } else if lifetime == 0 { + self.counters.num_deletes += 1; + if internal == 0 { + self.table + .retain(|e| !(e.client_ip == client && e.proto == proto)); + } else if let Some(i) = self.find(client, proto, internal) { + self.table.remove(i); + } + (0, 0, 0) + } else { + let (ext, granted) = + self.upsert(client, proto, internal, suggested, lifetime, None); + (0, ext, granted) + }; + let mut r = vec![0, op | 0x80]; + r.extend_from_slice(&code.to_be_bytes()); + r.extend_from_slice(&epoch.to_be_bytes()); + r.extend_from_slice(&internal.to_be_bytes()); + r.extend_from_slice(&ext.to_be_bytes()); + r.extend_from_slice(&granted.to_be_bytes()); + r + } + _ => self.pmp_error_reply(op, 5), + }; + let kind = log.kind; + self.requests.push(log); + Some((kind, out)) + } + + /// A PCP response header. + fn pcp_header(&self, op: u8, code: u8, lifetime: u32) -> Vec { + let mut r = vec![2, op | 0x80, 0, code]; + r.extend_from_slice(&lifetime.to_be_bytes()); + r.extend_from_slice(&self.epoch().to_be_bytes()); + r.extend_from_slice(&[0; 12]); + r + } + + fn handle_pcp( + &mut self, + req: &[u8], + src: SocketAddr, + client: Ipv4Addr, + ) -> Option<(RequestKind, Vec)> { + if !self.opts.pcp { + // A NAT-PMP-only server answers a PCP request with "unsupported version". + return (self.opts.pmp && req.len() >= 2) + .then(|| (RequestKind::Other, self.pmp_error_reply(req[1], 1))); + } + if req.len() < 24 || req.len() > 1100 || !req.len().is_multiple_of(4) || req[1] & 0x80 != 0 + { + self.counters.num_pcp_other_recv += 1; + return None; + } + self.counters.num_pcp_recv += 1; + let op = req[1]; + let lifetime = be32(req, 4); + let mut client_field = [0u8; 16]; + client_field.copy_from_slice(&req[8..24]); + let client_ip = Ipv6Addr::from(client_field).to_ipv4_mapped(); + let mismatch = self.opts.enforce_pcp_client_ip && client_ip != Some(client); + let mut log = DecodedRequest { + src, + kind: RequestKind::Other, + version: 2, + opcode: op, + internal: None, + suggested_external: None, + lifetime: Some(lifetime), + nonce: None, + pcp_client_ip: client_ip, + }; + let out = match op { + 0 => { + self.counters.num_pcp_disco_recv += 1; + log.kind = RequestKind::PcpAnnounce; + let code = match self.overrides.get(&log.kind) { + Some(o) => u8::try_from(o.code).unwrap_or(u8::MAX), + None if mismatch => 12, + None => 0, + }; + self.pcp_header(0, code, 0) + } + 1 => { + self.counters.num_pcp_map_recv += 1; + log.kind = RequestKind::PcpMap; + if req.len() < 60 { + self.counters.invalid_pcp_map_pkt += 1; + self.requests.push(log); + return Some((RequestKind::PcpMap, self.pcp_header(1, 3, 30))); + } + let mut nonce = [0u8; 12]; + nonce.copy_from_slice(&req[24..36]); + let proto = req[36]; + let internal = be16(req, 40); + let suggested = be16(req, 42); + log.nonce = Some(nonce); + log.internal = Some(internal); + log.suggested_external = Some(suggested); + let existing = self.find(client, proto, internal); + let foreign_nonce = + existing.is_some_and(|i| self.table[i].nonce.is_some_and(|n| n != nonce)); + let result: Result<(u16, u32), (u8, u32)> = if lifetime == 0 && self.drop_deletes { + Ok((0, 0)) + } else if let Some(o) = self.overrides.get(&log.kind) { + Err((u8::try_from(o.code).unwrap_or(u8::MAX), o.lifetime)) + } else if mismatch { + Err((12, 30)) + } else if proto != 17 && proto != 6 { + Err((9, 30)) + } else if internal == 0 { + self.counters.invalid_pcp_map_pkt += 1; + Err((3, 30)) + } else if foreign_nonce { + Err((2, 30)) + } else if lifetime == 0 { + self.counters.num_deletes += 1; + if let Some(i) = existing { + self.table.remove(i); + } + Ok((0, 0)) + } else { + Ok(self.upsert(client, proto, internal, suggested, lifetime, Some(nonce))) + }; + match result { + Err((code, err_lifetime)) if self.opts.short_pcp_errors => { + self.pcp_header(1, code, err_lifetime) + } + Err((code, err_lifetime)) => { + let mut r = self.pcp_header(1, code, err_lifetime); + r.extend_from_slice(&nonce); + r.extend_from_slice(&[proto, 0, 0, 0]); + r.extend_from_slice(&internal.to_be_bytes()); + r.extend_from_slice(&[0, 0]); + r.extend_from_slice(&v4_mapped(Ipv4Addr::UNSPECIFIED)); + r + } + Ok((ext, granted)) => { + let mut r = self.pcp_header(1, 0, granted); + r.extend_from_slice(&nonce); + r.extend_from_slice(&[proto, 0, 0, 0]); + r.extend_from_slice(&internal.to_be_bytes()); + r.extend_from_slice(&ext.to_be_bytes()); + let ip = if granted == 0 { + Ipv4Addr::UNSPECIFIED + } else { + self.opts.public_ip + }; + r.extend_from_slice(&v4_mapped(ip)); + r + } + } + } + _ => { + self.counters.num_pcp_other_recv += 1; + self.pcp_header(op, 4, 30) + } + }; + let kind = log.kind; + self.requests.push(log); + Some((kind, out)) + } +} + +fn corrupt(r: &mut Vec, c: Corruption) { + let pcp = r.first() == Some(&2); + match c { + Corruption::Truncate => { + r.pop(); + } + Corruption::BadVersion => { + if let Some(v) = r.first_mut() { + *v ^= 1; + } + } + Corruption::WrongOpcode => { + if let Some(o) = r.get_mut(1) { + *o ^= 0x01; + } + } + Corruption::NoRBit => { + if let Some(o) = r.get_mut(1) { + *o &= 0x7f; + } + } + Corruption::WrongNonce => { + if let Some(n) = r.get_mut(24) { + *n ^= 0xff; + } + } + Corruption::WrongProto => { + if let Some(p) = r.get_mut(36) { + *p = 6; + } + } + Corruption::WrongInternalPort => { + let at = if pcp { 41 } else { 9 }; + if let Some(p) = r.get_mut(at) { + *p = p.wrapping_add(1); + } + } + Corruption::Garbage => r.fill(0xa5), + Corruption::Oversize => r.resize(1200, 0), + } +} + +impl FakeRouter { + /// Start a router on `opts.bind_ip`, ports chosen by the OS. A non-loopback `bind_ip` is + /// refused with [`std::io::ErrorKind::InvalidInput`]: the harness never listens on a real + /// network. + pub async fn start(opts: RouterOptions) -> std::io::Result { + if !opts.bind_ip.is_loopback() { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "the fake router binds loopback only", + )); + } + let main = UdpSocket::bind((opts.bind_ip, 0)).await?; + let wrong_port = UdpSocket::bind((opts.bind_ip, 0)).await?; + let wrong_ip = UdpSocket::bind((opts.bind_ip, 0)).await?; + let addr = main.local_addr()?; + let wrong_port_addr = wrong_port.local_addr()?; + let wrong_ip_addr = wrong_ip.local_addr()?; + let state = Arc::new(Mutex::new(State { + opts, + table: Vec::new(), + overrides: HashMap::new(), + lifetime_override: None, + assign: AssignPort::Suggested { fallback: 4242 }, + delay: Duration::ZERO, + drop_next: 0, + drop_deletes: false, + duplicate_next: 0, + reorder_next: false, + commit_then_drop_next: 0, + wrong_port_next: 0, + wrong_ip_next: 0, + corrupt: VecDeque::new(), + junk_before_next: 0, + trickle_next: None, + spoof_next: None, + knobs_for: None, + counters: Counters::default(), + requests: Vec::new(), + barriers: Vec::new(), + held: None, + })); + let shutdown = Arc::new(Notify::new()); + let task = tokio::spawn(run( + main, + wrong_port, + wrong_ip, + state.clone(), + shutdown.clone(), + )); + Ok(Self { + addr, + wrong_port_addr, + wrong_ip_addr, + state, + shutdown, + task: Some(task), + }) + } + + /// The addresses a [`super::TestCap`] needs to talk to this router, detached from the router + /// itself (so a cap factory can outlive a borrow of it). + pub fn endpoint(&self) -> RouterEndpoint { + RouterEndpoint { + addr: self.addr, + wrong_port_addr: self.wrong_port_addr, + wrong_ip_addr: self.wrong_ip_addr, + } + } + + /// The router's loopback address (presented to clients as `gateway:5351`). + pub fn addr(&self) -> SocketAddr { + self.addr + } + + /// Where replies routed "from the wrong port" come from. + pub fn wrong_port_addr(&self) -> SocketAddr { + self.wrong_port_addr + } + + /// Where replies routed "from the wrong address" come from. + pub fn wrong_ip_addr(&self) -> SocketAddr { + self.wrong_ip_addr + } + + fn st(&self) -> MutexGuard<'_, State> { + self.state.lock().unwrap_or_else(|p| p.into_inner()) + } + + /// The lease clock. + pub fn clock(&self) -> LeaseClock { + self.st().opts.clock.clone() + } + + /// The counters so far. + pub fn counters(&self) -> Counters { + self.st().counters.clone() + } + + /// Every decoded request so far. + pub fn requests(&self) -> Vec { + self.st().requests.clone() + } + + /// The live mapping table (expired entries removed). + pub fn mappings(&self) -> Vec { + let mut st = self.st(); + let now = st.opts.clock.now(); + st.table.retain(|e| e.expires_at > now); + st.table.clone() + } + + /// Drop delete requests before committing them (create/renew requests still work). + pub fn set_drop_deletes(&self, on: bool) { + self.st().drop_deletes = on; + } + + /// Answer NAT-PMP or not. + pub fn set_pmp(&self, on: bool) { + self.st().opts.pmp = on; + } + + /// Answer PCP or not. + pub fn set_pcp(&self, on: bool) { + self.st().opts.pcp = on; + } + + /// Change the reported public address. + pub fn set_public_ip(&self, ip: Ipv4Addr) { + self.st().opts.public_ip = ip; + } + + /// Fix the epoch at `epoch`. + pub fn set_epoch(&self, epoch: u32) { + self.st().opts.epoch = EpochMode::Fixed(epoch); + } + + /// Simulate a reboot: the table is cleared and the epoch restarts from 0 on the clock. + pub fn reboot(&self) { + let mut st = self.st(); + st.table.clear(); + let now = st.opts.clock.now(); + st.opts.epoch = EpochMode::Clock { zero_at: now }; + } + + /// Force (or with `None`, stop forcing) the result for `kind`. + pub fn set_result(&self, kind: RequestKind, result: Option) { + let mut st = self.st(); + match result { + Some(r) => st.overrides.insert(kind, r), + None => st.overrides.remove(&kind), + }; + } + + /// Grant this lifetime instead of the requested one (shorter or longer). + pub fn set_lifetime_override(&self, lifetime: Option) { + self.st().lifetime_override = lifetime; + } + + /// Change how external ports are assigned. + pub fn set_assign_port(&self, assign: AssignPort) { + self.st().assign = assign; + } + + /// Wait this long before every reply. Each reply waits on its own; later requests are still + /// handled meanwhile. + pub fn set_delay(&self, delay: Duration) { + self.st().delay = delay; + } + + /// Ignore the next `n` requests entirely (nothing is committed). + pub fn drop_next(&self, n: u32) { + self.st().drop_next = n; + } + + /// Send each of the next `n` replies twice. With a pending [`Corruption`], only the first + /// copy is corrupted, so a bad reply is followed by a good one. + pub fn duplicate_next(&self, n: u32) { + self.st().duplicate_next = n; + } + + /// Hold the next reply and send it after the one that follows. + pub fn reorder_next(&self) { + self.st().reorder_next = true; + } + + /// Commit the next `n` requests but drop their replies. + pub fn commit_then_drop_next(&self, n: u32) { + self.st().commit_then_drop_next = n; + } + + /// Send the next `n` replies from another port. + pub fn reply_from_wrong_port_next(&self, n: u32) { + self.st().wrong_port_next = n; + } + + /// Send the next `n` replies from a socket a `TestCap` presents as another address. + pub fn reply_from_wrong_ip_next(&self, n: u32) { + self.st().wrong_ip_next = n; + } + + /// Corrupt the next reply (queued; one corruption per reply). + pub fn corrupt_next(&self, c: Corruption) { + self.st().corrupt.push_back(c); + } + + /// Before the next reply, send `n` junk datagrams from the router's own address. + pub fn junk_before_next_reply(&self, n: usize) { + self.st().junk_before_next = n; + } + + /// For the next request, send `n` junk datagrams from the router's own address, `interval` + /// apart (the first after one `interval`), and then drop the real reply. With `n` below + /// [`crate::MAX_EXAMINED`] and `n * interval` well past the exchange timeout, this shows + /// whether junk can extend an exchange's deadline. + pub fn junk_trickle_next(&self, n: usize, interval: Duration) { + self.st().trickle_next = Some((n, interval)); + } + + /// Before the next reply, send `datagram` from the router's own address: an on-LAN spoofer + /// that wins the race against the real reply. + pub fn spoof_before_next_reply(&self, datagram: Vec) { + self.st().spoof_next = Some(datagram); + } + + /// Apply the one-shot reply knobs (corruption, commit-then-drop, wrong source, duplicate, + /// reorder, junk, trickle, spoof) only to requests of `kind`; `None` (the default) applies + /// them to whatever request comes next. Barriers, delays and [`FakeRouter::drop_next`] are + /// unaffected. + pub fn knobs_for(&self, kind: Option) { + self.st().knobs_for = kind; + } + + /// Enforce that the PCP client IP field equals the source address. + pub fn set_enforce_pcp_client_ip(&self, on: bool) { + self.st().opts.enforce_pcp_client_ip = on; + } + + /// Send PCP errors header-only. + pub fn set_short_pcp_errors(&self, on: bool) { + self.st().opts.short_pcp_errors = on; + } + + /// Pause before replying to the next request of `kind` (after committing it). Only that reply + /// waits; other requests are handled and answered meanwhile. + pub fn barrier(&self, kind: RequestKind) -> Barrier { + let reached = Arc::new(Notify::new()); + let (tx, rx) = oneshot::channel(); + self.st().barriers.push(( + kind, + BarrierInner { + reached: reached.clone(), + release: rx, + }, + )); + Barrier { + reached, + release: Some(tx), + } + } + + /// Stop the router and wait for its task and every reply task to finish. Nothing runs + /// afterwards. + pub async fn shutdown(mut self) { + self.shutdown.notify_one(); + if let Some(task) = self.task.take() { + task.await.ok(); + } + } +} + +impl Drop for FakeRouter { + fn drop(&mut self) { + // A backstop only; tests call `shutdown().await`. + if let Some(task) = &self.task { + task.abort(); + } + } +} + +/// The router's three sockets. +struct Socks { + main: UdpSocket, + wrong_port: UdpSocket, + wrong_ip: UdpSocket, +} + +async fn run( + main: UdpSocket, + wrong_port: UdpSocket, + wrong_ip: UdpSocket, + state: Arc>, + shutdown: Arc, +) { + let socks = Arc::new(Socks { + main, + wrong_port, + wrong_ip, + }); + let mut replies = JoinSet::new(); + let mut buf = vec![0u8; 2048]; + loop { + let (n, src) = tokio::select! { + () = shutdown.notified() => break, + // Reap finished reply tasks so the set stays small. + Some(_) = replies.join_next(), if !replies.is_empty() => continue, + r = socks.main.recv_from(&mut buf) => match r { + Ok(v) => v, + Err(_) => continue, + }, + }; + // Decode and commit synchronously, in arrival order. + let plan = { + let mut st = state.lock().unwrap_or_else(|p| p.into_inner()); + st.handle(&buf[..n], src) + }; + if let Some(plan) = plan { + replies.spawn(reply(plan, src, socks.clone(), state.clone())); + } + } + // Abort and join every reply still paused, delayed or trickling. + replies.shutdown().await; +} + +/// Send one reply per its plan. Runs in its own task. +async fn reply(plan: ReplyPlan, src: SocketAddr, socks: Arc, state: Arc>) { + if let Some(b) = plan.barrier { + b.reached.notify_one(); + // A dropped `Barrier` releases too. + b.release.await.ok(); + } + if !plan.delay.is_zero() { + tokio::time::sleep(plan.delay).await; + } + if let Some((n, interval)) = plan.trickle { + for _ in 0..n { + tokio::time::sleep(interval).await; + socks.main.send_to(&[0xa5; 16], src).await.ok(); + } + return; + } + // Junk and a spoof are not the reply: they go out even when the reply is dropped or held. + for _ in 0..plan.junk { + socks.main.send_to(&[0xa5; 16], src).await.ok(); + } + if let Some(d) = &plan.spoof { + socks.main.send_to(d, src).await.ok(); + } + if plan.drop || plan.hold { + return; + } + let sock = match plan.via { + Via::Main => &socks.main, + Via::WrongPort => &socks.wrong_port, + Via::WrongIp => &socks.wrong_ip, + }; + sock.send_to(&plan.reply, src).await.ok(); + if plan.duplicate { + sock.send_to(&plan.original, src).await.ok(); + } + let held = state.lock().unwrap_or_else(|p| p.into_inner()).held.take(); + if let Some((r, to)) = held { + socks.main.send_to(&r, to).await.ok(); + } +} diff --git a/ts_portmapper/src/testing/tests.rs b/ts_portmapper/src/testing/tests.rs new file mode 100644 index 00000000..2416a671 --- /dev/null +++ b/ts_portmapper/src/testing/tests.rs @@ -0,0 +1,637 @@ +//! Tests of the fake router, `TestCap`, and the shared exchange loop's acceptance rules. + +use std::{ + net::Ipv4Addr, + num::{NonZeroU16, NonZeroU32}, + sync::Arc, + time::Duration, +}; + +use tokio::time::Instant; + +use super::*; +use crate::{ + CapError, DenyReason, ExchangeStats, Flow, Gateway, LanDest, MAX_EXAMINED, PolicyHandle, + PxpTransport, pcp, pmp, +}; + +const P1234: NonZeroU16 = NonZeroU16::new(1234).unwrap(); +const LIFE: NonZeroU32 = NonZeroU32::new(7200).unwrap(); + +fn gateway(self_ip: Ipv4Addr) -> Gateway { + Gateway { + gw: Ipv4Addr::new(192, 168, 1, 1), + self_ip, + prefix_len: 24, + if_index: 4, + if_name: "en0".to_owned(), + } +} + +async fn setup(opts: RouterOptions) -> (FakeRouter, Arc, PolicyHandle) { + let router = FakeRouter::start(opts).await.unwrap(); + let policy = PolicyHandle::new(true); + let cap = TestCap::new(gateway(Ipv4Addr::LOCALHOST), &router, policy.clone()); + (router, cap, policy) +} + +/// Send `reqs` and collect accepted replies until `want` arrived or `wait` passed. +async fn collect( + cap: &TestCap, + reqs: &[&[u8]], + want: usize, + wait: Duration, +) -> (Vec>, ExchangeStats) { + let mut got = Vec::new(); + let mut on = |b: &[u8]| { + got.push(b.to_vec()); + if got.len() >= want { + Flow::Done + } else { + Flow::Continue + } + }; + let stats = cap + .exchange(LanDest::Pxp, reqs, Instant::now() + wait, &mut on) + .await + .unwrap(); + (got, stats) +} + +const WAIT: Duration = Duration::from_secs(2); + +#[tokio::test] +async fn pmp_address_map_renew_delete() { + let (router, cap, _) = setup(RouterOptions::default()).await; + let (r, _) = collect(&cap, &[&pmp::public_address_request()], 1, WAIT).await; + let a = pmp::parse_public_addr_reply(&r[0]).unwrap(); + assert_eq!(a.public_ip, Ipv4Addr::new(203, 0, 113, 7)); + assert_eq!(a.epoch, 3600); + + let req = pmp::map_request(P1234, 0, LIFE); + let (r, _) = collect(&cap, &[&req], 1, WAIT).await; + let m = pmp::parse_map_reply(&r[0], P1234, pmp::MapKind::Map).unwrap(); + assert_eq!((m.external, m.lifetime), (4242, 7200)); + // A renewal keeps the rule and its external port. + let req = pmp::map_request(P1234, m.external, LIFE); + let (r, _) = collect(&cap, &[&req], 1, WAIT).await; + let m2 = pmp::parse_map_reply(&r[0], P1234, pmp::MapKind::Map).unwrap(); + assert_eq!(m2.external, 4242); + assert_eq!(router.mappings().len(), 1); + assert_eq!(router.mappings()[0].client_ip, Ipv4Addr::LOCALHOST); + + let (r, _) = collect(&cap, &[&pmp::delete_request(P1234)], 1, WAIT).await; + pmp::parse_map_reply(&r[0], P1234, pmp::MapKind::Delete).unwrap(); + assert!(router.mappings().is_empty()); + let c = router.counters(); + assert_eq!( + ( + c.num_pmp_recv, + c.num_pmp_public_addr_recv, + c.num_pmp_map_recv, + c.num_deletes + ), + (4, 1, 3, 1) + ); + let reqs = router.requests(); + assert_eq!(reqs[3].suggested_external, Some(0)); + assert_eq!(reqs[3].lifetime, Some(0)); + router.shutdown().await; +} + +#[tokio::test] +async fn pcp_announce_map_delete_with_nonce_ownership() { + let (router, cap, _) = setup(RouterOptions::default()).await; + let me = Ipv4Addr::LOCALHOST; + let (r, _) = collect(&cap, &[&pcp::announce_request(me)], 1, WAIT).await; + assert_eq!( + pcp::parse_announce_reply(&r[0]).unwrap().result, + pcp::ResultCode::Success + ); + + let nonce = pcp::Nonce::random().unwrap(); + let req = pcp::map_request(me, P1234, 0, LIFE, &nonce); + let (r, _) = collect(&cap, &[&req], 1, WAIT).await; + let m = pcp::parse_map_reply(&r[0], &nonce, P1234, pcp::MapKind::Map).unwrap(); + assert_eq!(m.external, "203.0.113.7:4242".parse().unwrap()); + assert_eq!(router.mappings()[0].nonce, Some(*nonce.as_bytes())); + + // Another nonce can neither renew nor delete the rule. + let thief = pcp::Nonce::random().unwrap(); + let (r, _) = collect(&cap, &[&pcp::delete_request(me, P1234, &thief)], 1, WAIT).await; + assert!(matches!( + pcp::parse_map_reply(&r[0], &thief, P1234, pcp::MapKind::Delete), + Err(pcp::ReplyError::Router { + code: pcp::ResultCode::NotAuthorized, + .. + }) + )); + assert_eq!(router.mappings().len(), 1); + + let (r, _) = collect(&cap, &[&pcp::delete_request(me, P1234, &nonce)], 1, WAIT).await; + pcp::parse_map_reply(&r[0], &nonce, P1234, pcp::MapKind::Delete).unwrap(); + assert!(router.mappings().is_empty()); + let c = router.counters(); + assert_eq!( + (c.num_pcp_recv, c.num_pcp_disco_recv, c.num_pcp_map_recv), + (4, 1, 3) + ); + router.shutdown().await; +} + +/// The router enforces the PCP client IP; the source-consistent loopback mode passes. +#[tokio::test] +async fn pcp_client_ip_enforcement() { + let opts = RouterOptions { + enforce_pcp_client_ip: true, + ..RouterOptions::default() + }; + let (router, ok_cap, policy) = setup(opts).await; + let (r, _) = collect( + &ok_cap, + &[&pcp::announce_request(Ipv4Addr::LOCALHOST)], + 1, + WAIT, + ) + .await; + assert_eq!( + pcp::parse_announce_reply(&r[0]).unwrap().result, + pcp::ResultCode::Success + ); + // The "inconsistent" mode: the client IP field says 198.51.100.4, the source is loopback. + let other = Ipv4Addr::new(198, 51, 100, 4); + let bad_cap = TestCap::new(gateway(other), &router, policy); + let (r, _) = collect(&bad_cap, &[&pcp::announce_request(other)], 1, WAIT).await; + assert_eq!( + pcp::parse_announce_reply(&r[0]).unwrap().result, + pcp::ResultCode::AddressMismatch + ); + router.shutdown().await; +} + +/// A NAT-PMP-only router answers a PCP request with a NAT-PMP "unsupported version". +#[tokio::test] +async fn pmp_only_router_answers_pcp_with_unsupported_version() { + let opts = RouterOptions { + pcp: false, + ..RouterOptions::default() + }; + let (router, cap, _) = setup(opts).await; + let (r, _) = collect( + &cap, + &[&pcp::announce_request(Ipv4Addr::LOCALHOST)], + 1, + WAIT, + ) + .await; + assert_eq!(r[0][0], 0); + assert!(matches!( + pmp::parse_public_addr_reply(&r[0]), + Err(pmp::ReplyError::Router { + code: pmp::ResultCode::UnsupportedVersion, + .. + }) + )); + router.shutdown().await; +} + +#[tokio::test] +async fn expiry_reboot_and_epoch() { + let (router, cap, _) = setup(RouterOptions::default()).await; + let clock = router.clock(); + router.set_lifetime_override(Some(60)); + let (r, _) = collect(&cap, &[&pmp::map_request(P1234, 0, LIFE)], 1, WAIT).await; + assert_eq!( + pmp::parse_map_reply(&r[0], P1234, pmp::MapKind::Map) + .unwrap() + .lifetime, + 60 + ); + clock.advance(59); + assert_eq!(router.mappings().len(), 1); + clock.advance(1); + assert!(router.mappings().is_empty()); + + router.set_lifetime_override(None); + collect(&cap, &[&pmp::map_request(P1234, 0, LIFE)], 1, WAIT).await; + router.reboot(); + assert!(router.mappings().is_empty()); + clock.advance(5); + let (r, _) = collect(&cap, &[&pmp::public_address_request()], 1, WAIT).await; + assert_eq!(pmp::parse_public_addr_reply(&r[0]).unwrap().epoch, 5); + router.set_epoch(9); + let (r, _) = collect(&cap, &[&pmp::public_address_request()], 1, WAIT).await; + assert_eq!(pmp::parse_public_addr_reply(&r[0]).unwrap().epoch, 9); + router.shutdown().await; +} + +/// At the harness level, a delete is scoped to its internal port, so a co-resident client's +/// rule (here internal 41644, like a Go tailscaled) survives. +#[tokio::test] +async fn delete_is_scoped_to_internal_port() { + let (router, cap, _) = setup(RouterOptions::default()).await; + let go = NonZeroU16::new(41644).unwrap(); + collect(&cap, &[&pmp::map_request(go, 0, LIFE)], 1, WAIT).await; + collect(&cap, &[&pmp::map_request(P1234, 0, LIFE)], 1, WAIT).await; + assert_eq!(router.mappings().len(), 2); + collect(&cap, &[&pmp::delete_request(P1234)], 1, WAIT).await; + let left = router.mappings(); + assert_eq!(left.len(), 1); + assert_eq!(left[0].internal, 41644); + router.shutdown().await; +} + +#[tokio::test] +async fn drop_commit_duplicate_and_reorder_knobs() { + let (router, cap, _) = setup(RouterOptions::default()).await; + let short = Duration::from_millis(150); + + router.drop_next(1); + let (r, _) = collect(&cap, &[&pmp::map_request(P1234, 0, LIFE)], 1, short).await; + assert!(r.is_empty()); + assert!( + router.mappings().is_empty(), + "a dropped request commits nothing" + ); + + router.commit_then_drop_next(1); + let (r, _) = collect(&cap, &[&pmp::map_request(P1234, 0, LIFE)], 1, short).await; + assert!(r.is_empty()); + assert_eq!( + router.mappings().len(), + 1, + "commit-then-drop keeps the rule" + ); + + router.duplicate_next(1); + let (r, _) = collect(&cap, &[&pmp::public_address_request()], 3, short).await; + assert_eq!(r.len(), 2); + + // Reorder: the first reply is held until after the second. + router.reorder_next(); + let (r, _) = collect( + &cap, + &[ + &pmp::public_address_request(), + &pmp::map_request(P1234, 0, LIFE), + ], + 2, + WAIT, + ) + .await; + assert_eq!(r.len(), 2); + assert_eq!(r[0][1], pmp::OP_MAP_UDP | pmp::OP_REPLY); + assert_eq!(r[1][1], pmp::OP_PUBLIC_ADDR | pmp::OP_REPLY); + router.shutdown().await; +} + +/// Replies from the wrong port or address never reach the handler. +#[tokio::test] +async fn wrong_source_replies_are_dropped() { + let (router, cap, _) = setup(RouterOptions::default()).await; + let short = Duration::from_millis(150); + router.reply_from_wrong_port_next(1); + let (r, s) = collect(&cap, &[&pmp::public_address_request()], 1, short).await; + assert!(r.is_empty()); + assert_eq!(s.wrong_source, 1); + router.reply_from_wrong_ip_next(1); + let (r, s) = collect(&cap, &[&pmp::public_address_request()], 1, short).await; + assert!(r.is_empty()); + assert_eq!(s.wrong_source, 1); + router.shutdown().await; +} + +/// An oversize datagram fills the 1101-byte buffer and is dropped; other corruptions +/// arrive and are rejected by the codecs. +#[tokio::test] +async fn corrupted_replies() { + let (router, cap, _) = setup(RouterOptions::default()).await; + let short = Duration::from_millis(150); + router.corrupt_next(Corruption::Oversize); + let (r, s) = collect(&cap, &[&pmp::public_address_request()], 1, short).await; + assert!(r.is_empty()); + assert_eq!(s.oversize, 1); + + let nonce = pcp::Nonce::random().unwrap(); + let me = Ipv4Addr::LOCALHOST; + for c in [ + Corruption::Truncate, + Corruption::BadVersion, + Corruption::WrongOpcode, + Corruption::NoRBit, + Corruption::WrongNonce, + Corruption::WrongProto, + Corruption::WrongInternalPort, + Corruption::Garbage, + ] { + router.corrupt_next(c); + let req = pcp::map_request(me, P1234, 0, LIFE, &nonce); + let (r, _) = collect(&cap, &[&req], 1, WAIT).await; + assert!( + matches!( + pcp::parse_map_reply(&r[0], &nonce, P1234, pcp::MapKind::Map), + Err(pcp::ReplyError::Malformed(_)) + ), + "{c:?}" + ); + } + router.shutdown().await; +} + +/// A burst of 1000 junk datagrams from the right source is bounded: the exchange stops +/// after examining `MAX_EXAMINED` of them. +#[tokio::test] +async fn junk_burst_is_bounded() { + let (router, cap, _) = setup(RouterOptions::default()).await; + router.junk_before_next_reply(1000); + let start = std::time::Instant::now(); + let deadline = Instant::now() + crate::EXCHANGE_TIMEOUT; + let mut accepted = 0; + let mut on = |_: &[u8]| { + accepted += 1; + Flow::Continue + }; + let s = cap + .exchange( + LanDest::Pxp, + &[&pmp::public_address_request()], + deadline, + &mut on, + ) + .await + .unwrap(); + assert!(s.examined <= MAX_EXAMINED); + assert!(s.hit_cap || Instant::now() >= deadline); + assert!(start.elapsed() < Duration::from_secs(2)); + assert!(accepted <= MAX_EXAMINED); + router.shutdown().await; +} + +/// Junk that trickles in from the right source (fewer datagrams than `MAX_EXAMINED`, spread +/// over six times the exchange timeout) cannot extend the absolute deadline. A per-receive +/// timeout would keep waiting for 1.5 s; the absolute deadline ends the exchange at 250 ms. +#[tokio::test] +async fn junk_trickle_cannot_extend_the_deadline() { + let (router, cap, _) = setup(RouterOptions::default()).await; + let n = 30; + let interval = Duration::from_millis(50); + assert!(n < MAX_EXAMINED); + assert!(interval * u32::try_from(n).unwrap() > crate::EXCHANGE_TIMEOUT * 4); + router.junk_trickle_next(n, interval); + let start = std::time::Instant::now(); + let mut accepted = 0; + let mut on = |_: &[u8]| { + accepted += 1; + Flow::Continue + }; + let s = cap + .exchange( + LanDest::Pxp, + &[&pmp::public_address_request()], + Instant::now() + crate::EXCHANGE_TIMEOUT, + &mut on, + ) + .await + .unwrap(); + let elapsed = start.elapsed(); + assert!(!s.hit_cap && s.examined < MAX_EXAMINED, "{s:?}"); + assert!( + accepted >= 1, + "some junk arrived before the deadline: {s:?}" + ); + assert!( + elapsed < Duration::from_millis(600), + "junk extended the exchange to {elapsed:?}" + ); + router.shutdown().await; +} + +/// Staging: while one reply is paused at a barrier, a later request is decoded, committed +/// and answered. +#[tokio::test] +async fn barrier_holds_only_its_own_reply() { + let (router, cap, _) = setup(RouterOptions::default()).await; + let other = NonZeroU16::new(4321).unwrap(); + collect(&cap, &[&pmp::map_request(other, 0, LIFE)], 1, WAIT).await; + let b = router.barrier(RequestKind::PmpMap); + let cap2 = cap.clone(); + let pending = tokio::spawn(async move { + collect(&cap2, &[&pmp::map_request(P1234, 0, LIFE)], 1, WAIT) + .await + .0 + }); + b.reached().await; + let before = router.requests().len(); + // A delete of the other rule commits and is answered while the first reply is held. + let (r, _) = collect(&cap, &[&pmp::delete_request(other)], 1, WAIT).await; + assert_eq!(r.len(), 1, "the second request was answered"); + pmp::parse_map_reply(&r[0], other, pmp::MapKind::Delete).unwrap(); + assert_eq!(router.requests().len(), before + 1); + let left = router.mappings(); + assert_eq!(left.len(), 1); + assert_eq!(left[0].internal, 1234); + assert!(!pending.is_finished(), "the first reply is still held"); + b.release(); + assert_eq!(pending.await.unwrap().len(), 1); + router.shutdown().await; +} + +/// The harness refuses to listen on, or send to, anything but loopback. +#[tokio::test] +async fn harness_is_loopback_only() { + for ip in [Ipv4Addr::UNSPECIFIED, Ipv4Addr::new(192, 168, 1, 1)] { + let r = FakeRouter::start(RouterOptions { + bind_ip: ip, + ..RouterOptions::default() + }) + .await; + assert_eq!( + r.map(drop).unwrap_err().kind(), + std::io::ErrorKind::InvalidInput, + "{ip}" + ); + } + let to = std::net::SocketAddr::from((Ipv4Addr::new(192, 168, 1, 1), 5351)); + assert_eq!( + send_datagrams(to, 1, 1).await.unwrap_err().kind(), + std::io::ErrorKind::InvalidInput + ); +} + +#[tokio::test] +async fn barrier_pauses_after_commit() { + let (router, cap, _) = setup(RouterOptions::default()).await; + let b = router.barrier(RequestKind::PmpMap); + let cap2 = cap.clone(); + let pending = tokio::spawn(async move { + collect(&cap2, &[&pmp::map_request(P1234, 0, LIFE)], 1, WAIT) + .await + .0 + }); + b.reached().await; + assert_eq!(router.mappings().len(), 1, "committed before the pause"); + b.release(); + assert_eq!(pending.await.unwrap().len(), 1); + router.shutdown().await; +} + +#[tokio::test] +async fn result_overrides_and_short_errors() { + let (router, cap, _) = setup(RouterOptions::default()).await; + router.set_result( + RequestKind::PmpMap, + Some(ResultOverride { + code: 2, + lifetime: 0, + }), + ); + let (r, _) = collect(&cap, &[&pmp::map_request(P1234, 0, LIFE)], 1, WAIT).await; + assert!(matches!( + pmp::parse_map_reply(&r[0], P1234, pmp::MapKind::Map), + Err(pmp::ReplyError::Router { + code: pmp::ResultCode::NotAuthorized, + .. + }) + )); + router.set_short_pcp_errors(true); + router.set_result( + RequestKind::PcpMap, + Some(ResultOverride { + code: 8, + lifetime: 120, + }), + ); + let nonce = pcp::Nonce::random().unwrap(); + let req = pcp::map_request(Ipv4Addr::LOCALHOST, P1234, 0, LIFE, &nonce); + let (r, _) = collect(&cap, &[&req], 1, WAIT).await; + assert_eq!(r[0].len(), 24); + assert_eq!( + pcp::parse_map_reply(&r[0], &nonce, P1234, pcp::MapKind::Map), + Err(pcp::ReplyError::Unmatched { + code: pcp::ResultCode::NoResources, + lifetime: 120, + epoch: 3600 + }) + ); + router.shutdown().await; +} + +/// With the test transport, a refused send writes nothing (router counter 0), and injected +/// OS errors surface verbatim. +#[tokio::test] +async fn test_cap_gate_and_injected_errors() { + let (router, cap, policy) = setup(RouterOptions::default()).await; + let deadline = Instant::now() + Duration::from_millis(100); + let mut ignore = |_: &[u8]| Flow::Continue; + let req = pmp::public_address_request(); + + policy.set_lan_sends_allowed(false); + let r = cap + .exchange(LanDest::Pxp, &[&req], deadline, &mut ignore) + .await; + assert!(matches!(r, Err(CapError::Denied(DenyReason::Policy)))); + assert!(cap.send_nowait(LanDest::Pxp, &req).is_err()); + policy.set_lan_sends_allowed(true); + + cap.inject_send_error(Some(65)); + let r = cap + .exchange(LanDest::Pxp, &[&req], deadline, &mut ignore) + .await; + let msg = r.unwrap_err().to_string(); + assert!(msg.contains("os error 65"), "{msg}"); + cap.inject_send_error(None); + + cap.invalidate(); + let r = cap + .exchange(LanDest::Pxp, &[&req], deadline, &mut ignore) + .await; + assert!(matches!(r, Err(CapError::Denied(DenyReason::Invalidated)))); + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!(router.counters().num_total_recv, 0); + router.shutdown().await; +} + +/// Two routers stand in for two gateways that both use port 5351. +#[tokio::test] +async fn two_routers() { + let a = FakeRouter::start(RouterOptions::default()).await.unwrap(); + let b = FakeRouter::start(RouterOptions { + public_ip: Ipv4Addr::new(198, 51, 100, 9), + ..RouterOptions::default() + }) + .await + .unwrap(); + let p = PolicyHandle::new(true); + let mut gb = gateway(Ipv4Addr::LOCALHOST); + gb.gw = Ipv4Addr::new(10, 0, 0, 1); + let ca = TestCap::new(gateway(Ipv4Addr::LOCALHOST), &a, p.clone()); + let cb = TestCap::new(gb, &b, p); + let (r, _) = collect(&cb, &[&pmp::public_address_request()], 1, WAIT).await; + assert_eq!( + pmp::parse_public_addr_reply(&r[0]).unwrap().public_ip, + Ipv4Addr::new(198, 51, 100, 9) + ); + collect(&ca, &[&pmp::public_address_request()], 1, WAIT).await; + assert_eq!(a.counters().num_pmp_recv, 1); + assert_eq!(b.counters().num_pmp_recv, 1); + a.shutdown().await; + b.shutdown().await; +} + +/// Shutdown also ends replies that are still paused at a barrier or delayed. +#[tokio::test] +async fn shutdown_aborts_paused_replies() { + let (router, cap, _) = setup(RouterOptions::default()).await; + let b = router.barrier(RequestKind::PmpPublicAddr); + router.set_delay(Duration::from_secs(30)); + let cap2 = cap.clone(); + let pending = tokio::spawn(async move { + collect( + &cap2, + &[&pmp::public_address_request()], + 1, + Duration::from_millis(500), + ) + .await + .0 + }); + b.reached().await; + let start = std::time::Instant::now(); + router.shutdown().await; + assert!(start.elapsed() < Duration::from_secs(2)); + // The barrier is never released and the reply never sent. + assert!(pending.await.unwrap().is_empty()); + drop(b); +} + +/// After `shutdown().await` the router is gone. +#[tokio::test] +async fn shutdown_stops_the_router() { + let (router, cap, _) = setup(RouterOptions::default()).await; + let counters = router.counters(); + router.shutdown().await; + let (r, _) = collect( + &cap, + &[&pmp::public_address_request()], + 1, + Duration::from_millis(100), + ) + .await; + assert!(r.is_empty()); + assert_eq!(counters.num_total_recv, 0); +} + +/// Linux has the whole 127/8 on loopback, so a router can bind a second loopback address for +/// real. +#[cfg(target_os = "linux")] +#[tokio::test] +async fn router_on_another_loopback_address() { + let opts = RouterOptions { + bind_ip: Ipv4Addr::new(127, 0, 0, 2), + ..RouterOptions::default() + }; + let (router, cap, _) = setup(opts).await; + let (r, _) = collect(&cap, &[&pmp::public_address_request()], 1, WAIT).await; + assert!(pmp::parse_public_addr_reply(&r[0]).is_ok()); + router.shutdown().await; +} From 39eb7d3b534ca6aa440597493d0e9030550fa647 Mon Sep 17 00:00:00 2001 From: Kasper Nielsen Date: Sat, 17 Oct 2026 23:06:46 +0200 Subject: [PATCH 02/12] feat(portmapper): add the lifecycle client Add the port mapper's lifecycle client: create, renew, expire and release a mapping, with generation-guarded transitions and Go's retry, renewal and probe-epoch rules. A bounded ledger of four rules tracks every rule the router may hold for us, including unacknowledged deletes and rules on networks that went away. A slot is freed only by an acknowledged delete or proven router expiry. One private owner settles every transition (see the table in docs/PORTMAPPER.md). Shutdown and Drop release with cached capabilities, without a runtime and without fresh OS discovery. Tests run against the fake router. An independent ownership model checks conservation in the shared mapping and settling helpers, and router-inventory checks compare the ledger with the fake router's rule table in the cancellation, cleanup and reset scenarios, including every MAP and cleanup cancellation point. Signed-off-by: Kasper Nielsen Co-Authored-By: GPT-6.1 (codex) Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 + docs/PORTMAPPER.md | 134 + ts_portmapper/src/blocking.rs | 78 +- ts_portmapper/src/cap.rs | 160 +- ts_portmapper/src/client.rs | 1297 +++++++ ts_portmapper/src/client_tests.rs | 4839 ++++++++++++++++++++++++ ts_portmapper/src/clock.rs | 52 + ts_portmapper/src/lib.rs | 19 +- ts_portmapper/src/lifecycle.rs | 944 +++++ ts_portmapper/src/oneshot.rs | 32 +- ts_portmapper/src/owned.rs | 728 ++++ ts_portmapper/src/ownership_model.rs | 215 ++ ts_portmapper/src/schedule.rs | 354 ++ ts_portmapper/src/test_time.rs | 46 + ts_portmapper/src/testing/cap.rs | 42 +- ts_portmapper/src/testing/lifecycle.rs | 201 + ts_portmapper/src/testing/mod.rs | 11 + 17 files changed, 9113 insertions(+), 43 deletions(-) create mode 100644 docs/PORTMAPPER.md create mode 100644 ts_portmapper/src/client.rs create mode 100644 ts_portmapper/src/client_tests.rs create mode 100644 ts_portmapper/src/clock.rs create mode 100644 ts_portmapper/src/lifecycle.rs create mode 100644 ts_portmapper/src/owned.rs create mode 100644 ts_portmapper/src/ownership_model.rs create mode 100644 ts_portmapper/src/schedule.rs create mode 100644 ts_portmapper/src/test_time.rs create mode 100644 ts_portmapper/src/testing/lifecycle.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 6422ed97..039abaa2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ Record breaking or significant changes here. All dates are UTC. ## Unreleased +### Features + +* **portmapper:** add `ts_portmapper`, a NAT-PMP and PCP client modelled on Go's `net/portmapper`, which only ever sends to the LAN gateway's port 5351. + ### Performance * **exit node:** forwarded TCP flows run without Nagle, the underlay UDP sockets ask for Go's 7 MiB buffers, netstack egress goes out in batches of up to 64 packets, and the netstack tracks 32 receive holes for uploads. Kernel receive batching, splice coalescing, targeted replay and Apple connected UDP sockets are opt-in experiments, off unless their variable is `1`. diff --git a/docs/PORTMAPPER.md b/docs/PORTMAPPER.md new file mode 100644 index 00000000..99a2429e --- /dev/null +++ b/docs/PORTMAPPER.md @@ -0,0 +1,134 @@ +# NAT port mapper (NAT-PMP and PCP) + +[`ts_portmapper`](../ts_portmapper/src/lib.rs) asks the LAN gateway for an inbound UDP mapping to +the node's underlay port, as Go Tailscale's `net/portmapper` does, so that a peer behind a hard +NAT can reach the node directly instead of over DERP. NAT-PMP (RFC 6886) and PCP (RFC 6887) are +implemented. UPnP-IGD is not (bead `tsr-x4yg`, phase 2). + +The crate holds the lifecycle client, the codecs, the gateway lookup and `GatewayCap`, the only +code that sends packets. (The gateway lookup's kernel source-address check connects a UDP socket +but never sends on it.) Nothing in the node runs it yet. + +## Deviations from Go + +- **Advertising (`advertise_nonpublic_external`).** The default is Go's: advertise every external + address. With the option off, a private or CGNAT external address (a double NAT) is kept and + renewed, but not advertised. +- **Renewal while idle (`renew_while_idle`).** The default is Go's: renewal rides on activity (the + periodic ReSTUN and other kicks). With the option on, a due mapping is renewed even while idle, + as long as there are peers. With no peers a mapping is never renewed and expires. +- **Notifications for every protocol.** Success, loss, expiry and disable publish for NAT-PMP and + PCP alike. In Go only PCP successes publish. +- **Strict replies.** Replies are accepted only from exactly `gateway:5351`, and every field is + validated: version, opcode, R bit, length, internal port, protocol and nonce. Go's probe checks + only the source port. +- **NAT-PMP delete.** The suggested external port is 0, as RFC 6886 §3.4 requires. Go sends the + current one. +- **Renewal protocol.** A usable mapping keeps its owning protocol, even if another service + becomes available. PMP and PCP may share one router rule, so switching and then deleting the + old rule could delete its replacement. +- **Gateway selection.** Only the most preferred default route counts, and it must be private and + on-link. There is no fall-through to a less preferred private route (Go can pick a VPN gateway). + Sockets are bound to the host's own address and, on macOS and on Linux with permission for + `SO_BINDTODEVICE`, to its interface. Otherwise (source-address mode) the route is checked again + before each send. +- **Lease trust.** Local trust is `min(granted, 2 h)`. A PCP grant longer than requested is + recorded as the router's expiry but not trusted beyond 2 h. +- **Renewal timing.** Renewal starts at half the trusted lifetime, clamped to between 10 s and 1 h + and never later than expiry. Go renews at half the granted lifetime. +- **IPv4 only.** Mappings are for the IPv4 underlay socket. A PCP reply with a native IPv6 + external address is rejected; Go accepts it. +- **Epoch.** Go's rule applies to probe replies only. An epoch that goes backwards drops the + mapping without a release. + +## Router state, stated honestly + +- **Normal state.** One UDP rule, `external:assigned → self_ip:underlay port`. It is open to any + source host. WireGuard and disco authenticate the traffic, but the listener receives unsolicited + internet UDP, the same exposure Go accepts. +- **Clean shutdown.** Deletes use cached validated capabilities within the shutdown timeout + (at most 1 s of it), with no fresh OS discovery. The `Drop` path also sends them, best-effort + and unacknowledged. Source-IP-only Linux mode needs fresh route validation before a send, so + timed/Drop release fails closed there and leaves the rule to expiry. A lost delete also leaves + the rule until its granted lifetime ends. +- **Crash, SIGKILL or power loss.** Rules survive until the router's **granted** lifetime ends. We + ask for 7200 s. PCP may legally grant more, so the bound is "granted lifetime", not "2 h". +- **Outstanding rules.** The ledger holds at most four, including rules whose deletes failed or + whose networks disappeared. Such tombstones keep their slots until acknowledgement or possible + router expiry. After three unacknowledged deletes, fresh validation and retries continue at + most once per rule per ten minutes, without freeing its slot. Every MAP reserves its identity + before I/O. Possible ownership expands only when its nonblocking send syscall is attempted, + after socket setup, route validation and the final send permit. A fresh never-sent identity + retires without DELETE; an unsent adoption/renewal retains its previous grant and nonce/protocol. + NAT-PMP's public-address request alone cannot create a mapping. A matched success records the + actual grant. After a MAP attempt without a success reply, the grant could exceed the request, + so the tombstone conservatively uses the full wire maximum (`u32::MAX` seconds, about 136 years) + from the latest syscall attempt. + Four unresolved rules can therefore stop further creates for this process. Wall-clock jumps + withdraw advertisement trust but do not shorten possible ownership. +- **Network down.** Nothing is deleted, since a delete could reach the wrong router. Such rules are + left to expire. A create for the same rule on the same network adopts them, because PCP refuses a + second nonce for a live rule. +- **`only-tcp-443`.** No deletes either: the knob means "no UDP". Leftovers are deleted first if + the knob is lifted on the same network. +- **DHCP reassignment.** A later holder of our LAN address could receive traffic for a stale rule + until it expires. +- **Never.** No wildcard deletes (internal port 0) and no overwriting of other clients' rules. A + delete names our own internal port, so a co-resident Go `tailscaled` (41644) keeps its mapping. + +## OS discovery and rebind + +Routing-table reads, interface enumeration, capability validation and source-mode route checks +run on a bounded worker: at most four detached OS threads, with a one-second wait budget (the +exchange deadline can be shorter). Cancellation retains the worker slot until its OS call ends; +a stuck call cannot block a Tokio worker or runtime teardown. Results are generation checked. +A same-port rebind withdraws the endpoint synchronously and restores it after fresh asynchronous +network validation. A changed port invalidates ownership for cleanup immediately. + +## Lifecycle transitions + +The private [`Lifecycle`](../ts_portmapper/src/lifecycle.rs) owner serializes settlement under +the client state mutex. Production callers have only an immutable state view and named event +methods; mutable access for fault injection exists only in tests. I/O and discovery run outside +that mutex. The ledger conserves **possible router ownership**, separately from the shorter +advertisement lease. Its four slots include fenced and terminal orphans. An unacknowledged +final send never retires a slot. A UDP wire identity (gateway, self address, interface, internal +port) has at most one MAP or delete reservation, across both protocols. + +An entry is pending, active (backed by a held lease), or orphaned. A MAP reservation retains that +disposition while its operation lives; a delete reservation always owns an orphan. Fenced means +that deletion is forbidden until validated adoption. Final duty is an obligation handed to the +current reservation, or claimed immediately if idle. Retired entries leave the ledger only with +the evidence below. Close progresses Running → Closing → Closed; direct Drop-path release +progresses Running → Closed (or completes Closing → Closed). + +| Event | Required evidence / precondition | Settlement and next owner | +|---|---|---| +| Create / adopt / renew | Validated owning network, enabled policy, spacing, no existing reservation for the wire identity; free admission capacity only for a new identity | Reserve MAP before I/O; expand the possible grant at its syscall boundary, not admission; operation owns entry; renewal/adoption reuses its slot, protocol and nonce even when the ledger is full | +| MAP success | Matched identity/nonce and grant | Current generation: active entry plus held lease; stale generation: orphan with observed grant; MAP reservation survives until finalizer | +| MAP never sent | Transport future ended or was dropped; no MAP syscall was entered (including setup, route, deadline or permit failure); a PMP address request alone is insufficient | Retire a fresh reservation without DELETE and free its slot; adoption/renewal retains prior possible expiry and protocol/nonce, with no added grant | +| MAP refusal / loss | Matched refusal or proven pre-MAP local rejection retires only a new create; an existing rule may still live | Keep held renewal; otherwise retain an orphan for uncertain ownership (including a partial PMP success); matched new-create refusal retires | +| MAP completion / cancellation | Operation finalizer, including cancellation before first poll | Return reservation; a never-sent fresh reservation retires; otherwise unsettled pending becomes orphan, retaining prior ownership or the attempted MAP's possible grant; held active remains held; orphan goes to cleanup or deferred final duty | +| Lease expiry / clock aging | Trusted lease expired; clock aging alone is not router-expiry evidence | Remove held owner, orphan even during renewal, keep reservation/nonce/grant; retire only if possible router expiry is proven and no MAP owns it | +| Cleanup claim | Idle, unfenced orphan; retry due, or closing | Delete operation owns reservation; reserve recovery cooldown before validation/I/O | +| Delete acknowledgement | Matched per-rule acknowledgement | Retire slot; operation finalizer becomes a no-op for that entry | +| Delete refusal / loss | No acknowledgement | Return orphan, count attempt; after three attempts use ten-minute recovery spacing with fresh owning-network validation | +| Ordinary cleanup validation unavailable | Worker timeout/failure while Running | Retain orphan, return delete reservation and defer fresh validation/recovery for ten minutes | +| Close-time validation unavailable / source-IP-only capability | Closing client; no valid cached release capability, or cached capability requires fresh source-route validation | Terminally fence orphan and return delete reservation; retain admission slot, schedule no recovery and send nothing; no fresh discovery during close | +| Cleanup cancellation | Operation finalizer | Retain orphan and return reservation without manufacturing acknowledgement; pending final duty can pass to final release | +| Accepted router reset | Fresh nonzero probe epoch rollback, held lease committed before probe; same gateway/protocol | Retire idle affected entries; retain reserved entries whose requests may reach restarted router; orphan held lease | +| Port / network change | Generation change; old network must freshly validate before cleanup | Withdraw held lease, cancel MAP handle but preserve its reservation until finalizer; orphan entries; retain old release capabilities | +| Network down / rejected network | Synchronous notification or failed validation | Orphan and fence entries, invalidate normal and release send gates; no delete; validated same-identity adoption can retain nonce and reopen entry | +| Same-port rebind | Socket rebound, awaiting asynchronous validation | Withdraw advertisement, cancel MAP handle; privately retain held owner until network validation or expiry; reservation stays with operation | +| Failed rebind | Rebind left no underlay socket | Clear local port, invalidate/withdraw held mapping, cancel MAP handle and orphan entries; schedule cleanup; reservations stay with operations until finalization | +| Policy disable / enable | Shared send permit fences disable before it returns | Disable withdraws held lease/cancels attempt and retains orphans; no LAN sends. Enable permits cleanup on validated network | +| Close begin | Running client; one absolute shutdown deadline | Closing, stop new work, cancel MAP/tasks/public cleanup; wait for reservations, then cleanup with cached capabilities | +| Close completion / cancellation | Closing client; terminal transition is idempotent | Closing → Closed; arm final duty once on unresolved entries; live operations retain reservation and inherit duty | +| Drop-path release | Running or Closing client; no runtime required; Closed is a no-op | Running → Closed or Closing → Closed; invalidate/withdraw held mapping, cancel work and arm final duty once; live operations retain reservation and inherit duty | +| Final release | Unfenced idle orphan, unexpired possible grant, enabled policy, cached valid interface-bound capability | Consume duty, reserve delete through nonblocking send outside state lock; return counted terminal orphan without acknowledgement; no route discovery | +| Proven router expiry | Monotonic possible-grant deadline reached, no operation reservation | Retire slot; wall-clock correction alone cannot justify retirement | + +The send permit is separate from the state mutex: readiness and route validation precede it; +the final validity/policy check and one nonblocking syscall hold it. Invalidation and disable +share it. No permit holder acquires the state mutex or awaits. Runtime-less final release uses +the same permit and retains its reservation until the syscall returns. diff --git a/ts_portmapper/src/blocking.rs b/ts_portmapper/src/blocking.rs index 95575176..d14416c0 100644 --- a/ts_portmapper/src/blocking.rs +++ b/ts_portmapper/src/blocking.rs @@ -36,48 +36,63 @@ async fn run_on( #[cfg(test)] mod tests { + use crate::test_time::{FrozenTime, finish_at_deadline}; + use std::{ + sync::Arc, + sync::atomic::{AtomicUsize, Ordering}, + time::Duration, + }; + #[tokio::test] async fn os_wait_is_bounded_when_worker_keeps_running() { - let result = super::run(|| { - std::thread::sleep(std::time::Duration::from_millis(1500)); + // A private pool and an explicit gate avoid a sleeping detached worker consuming a + // production-pool slot after this test has returned. + static SLOTS: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(1); + let _frozen = FrozenTime::new(); + let (entered, received) = tokio::sync::oneshot::channel(); + let (release, wait) = std::sync::mpsc::channel::<()>(); + let job = tokio::spawn(super::run_on(&SLOTS, move || { + let _ = entered.send(()); + let _ = wait.recv(); 42 - }) - .await; + })); + received.await.unwrap(); + assert_eq!(SLOTS.available_permits(), 0); + tokio::time::advance(Duration::from_millis(1001)).await; assert_eq!( - result, None, + job.await.unwrap(), + None, "a slow OS call must not hold its async waiter indefinitely" ); + assert_eq!( + SLOTS.available_permits(), + 0, + "timed-out OS work keeps its slot" + ); + drop(release); // also unblocks the worker if an assertion unwinds + while SLOTS.available_permits() != 1 { + tokio::task::yield_now().await; + } } -} - -#[cfg(test)] -mod worker_tests { - static TEST_SLOTS: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(4); #[tokio::test] async fn cancelled_workers_keep_slots_and_recover() { - use std::sync::atomic::{AtomicUsize, Ordering}; - use std::sync::{Arc, Condvar, Mutex}; - let gate = Arc::new((Mutex::new(false), Condvar::new())); + static SLOTS: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(4); + let _frozen = FrozenTime::new(); let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel(); - let finished = Arc::new(AtomicUsize::new(0)); + let mut releases = Vec::new(); let mut jobs = Vec::new(); for _ in 0..4 { - let gate = gate.clone(); let tx = tx.clone(); - let finished = finished.clone(); - jobs.push(tokio::spawn(super::run_on(&TEST_SLOTS, move || { + let (release, wait) = std::sync::mpsc::channel::<()>(); + releases.push(release); + jobs.push(tokio::spawn(super::run_on(&SLOTS, move || { tx.send(()).unwrap(); - let ready = gate.0.lock().unwrap(); - drop(gate.1.wait_while(ready, |open| !*open).unwrap()); - finished.fetch_add(1, Ordering::SeqCst); + let _ = wait.recv(); }))); } for _ in 0..4 { - tokio::time::timeout(std::time::Duration::from_secs(2), rx.recv()) - .await - .unwrap() - .unwrap(); + rx.recv().await.unwrap(); } for job in &jobs { job.abort(); @@ -87,21 +102,24 @@ mod worker_tests { } let fifth_started = Arc::new(AtomicUsize::new(0)); let count = fifth_started.clone(); - let blocked = - super::run_on(&TEST_SLOTS, move || count.fetch_add(1, Ordering::SeqCst)).await; - *gate.0.lock().unwrap() = true; - gate.1.notify_all(); + let blocked = finish_at_deadline( + super::run_on(&SLOTS, move || count.fetch_add(1, Ordering::SeqCst)), + Duration::from_secs(1), + ) + .await; assert!(blocked.is_none()); assert_eq!( fifth_started.load(Ordering::SeqCst), 0, "aborted waiters must not release running OS slots" ); - while finished.load(Ordering::SeqCst) != 4 { + drop(releases); + // Observe permit return itself, rather than a flag set just before its destructor. + while SLOTS.available_permits() != 4 { tokio::task::yield_now().await; } assert_eq!( - super::run_on(&TEST_SLOTS, || 42).await, + super::run_on(&SLOTS, || 42).await, Some(42), "completion must return slots without a lost wakeup" ); diff --git a/ts_portmapper/src/cap.rs b/ts_portmapper/src/cap.rs index 4d4f262c..20ddfe07 100644 --- a/ts_portmapper/src/cap.rs +++ b/ts_portmapper/src/cap.rs @@ -30,7 +30,7 @@ use std::{ net::{SocketAddr, SocketAddrV4}, sync::{ Arc, Mutex, MutexGuard, - atomic::{AtomicBool, Ordering}, + atomic::{AtomicBool, AtomicU64, Ordering}, }, time::Duration, }; @@ -263,6 +263,56 @@ pub struct ExchangeStats { pub recv_error: Option, } +/// Per-MAP evidence shared with the lifecycle reservation, including after cancellation. +/// Constructed internally; only the sealed transports can record a syscall attempt. The +/// NAT-PMP public-address request does not count as a MAP. Socket/route/permit failures do not +/// record evidence. Once recorded, evidence is never cleared, even when the syscall fails. +#[derive(Debug)] +pub struct SendEvidence { + map_index: usize, + logical_origin: Instant, + runtime_origin: Instant, + attempted_ns: AtomicU64, +} + +impl SendEvidence { + pub(crate) fn new(map_index: usize, now: Instant) -> Self { + Self { + map_index, + logical_origin: now, + runtime_origin: Instant::now(), + attempted_ns: AtomicU64::new(0), + } + } + + pub(crate) fn enter(&self, index: usize) { + if index == self.map_index { + let elapsed = Instant::now().saturating_duration_since(self.runtime_origin); + // Zero means never attempted. Retrying a WouldBlock must advance the possible + // grant from the latest attempt, even when an earlier attempt already marked it. + let ns = elapsed.as_nanos().min(u128::from(u64::MAX - 1)) as u64; + self.attempted_ns.fetch_max(ns + 1, Ordering::SeqCst); + } + } + + pub(crate) fn possible_until(&self) -> Option { + let ns = self.attempted_ns.load(Ordering::SeqCst).checked_sub(1)?; + let elapsed = Duration::from_nanos(ns); + let logical = self + .logical_origin + .checked_add(elapsed) + .unwrap_or(self.logical_origin); + let runtime = self + .runtime_origin + .checked_add(elapsed) + .unwrap_or(self.runtime_origin); + Some(crate::oneshot::lease_deadline( + logical.max(runtime), + u32::MAX, + )) + } +} + /// The checks every send passes first. pub(crate) struct SendGate { policy: PolicyHandle, @@ -355,8 +405,9 @@ pub(crate) async fn exchange_on( reqs: &[&[u8]], deadline: Instant, on_reply: &mut (dyn FnMut(&[u8]) -> Flow + Send), + evidence: Option<&SendEvidence>, ) -> Result { - for req in reqs { + for (index, req) in reqs.iter().enumerate() { loop { gate.check()?; tokio::time::timeout_at(deadline, sock.writable()) @@ -369,6 +420,9 @@ pub(crate) async fn exchange_on( if Instant::now() >= deadline { return Err(io::ErrorKind::TimedOut.into()); } + if let Some(evidence) = evidence { + evidence.enter(index); + } sock.try_send_to(req, to) }); match sent { @@ -479,6 +533,19 @@ pub trait PxpTransport: sealed::Sealed + Send + Sync { reqs: &[&[u8]], deadline: Instant, on_reply: &mut (dyn FnMut(&[u8]) -> Flow + Send), + ) -> impl Future> + Send { + self.exchange_with_evidence(dest, reqs, deadline, on_reply, None) + } + + /// The exchange implementation, optionally recording the MAP syscall boundary. Evidence + /// is operation-scoped and must be recorded after all gates, immediately before the syscall. + fn exchange_with_evidence( + &self, + dest: LanDest, + reqs: &[&[u8]], + deadline: Instant, + on_reply: &mut (dyn FnMut(&[u8]) -> Flow + Send), + evidence: Option<&SendEvidence>, ) -> impl Future> + Send; /// One unacknowledged, non-blocking send (best-effort release from `Drop`; no runtime @@ -786,12 +853,13 @@ impl PxpTransport for GatewayCap { self.bind_mode } - async fn exchange( + async fn exchange_with_evidence( &self, dest: LanDest, reqs: &[&[u8]], deadline: Instant, on_reply: &mut (dyn FnMut(&[u8]) -> Flow + Send), + evidence: Option<&SendEvidence>, ) -> Result { // Refuse before creating a socket at all. self.gate.check()?; @@ -806,7 +874,7 @@ impl PxpTransport for GatewayCap { .map(|ip| SocketAddrV4::new(ip, a.port())), }; exchange_on( - &sock, &self.gate, to, expected, &present, reqs, deadline, on_reply, + &sock, &self.gate, to, expected, &present, reqs, deadline, on_reply, evidence, ) .await } @@ -1229,6 +1297,7 @@ pub(crate) mod send_tests { let entered = Arc::new(tokio::sync::Notify::new()); let open = Arc::new((Mutex::new(false), std::sync::Condvar::new())); let (hit, barrier) = (entered.clone(), open.clone()); + let evidence = SendEvidence::new(0, Instant::now()); let policy = PolicyHandle::new(true); let gate = SendGate::new( policy.clone(), @@ -1261,6 +1330,7 @@ pub(crate) mod send_tests { &reqs, Instant::now() + Duration::from_secs(2), &mut on_reply, + Some(&evidence), )); tokio::select! { _ = entered.notified() => {}, @@ -1274,6 +1344,10 @@ pub(crate) mod send_tests { *open.0.lock().unwrap() = true; open.1.notify_one(); assert!(matches!(exchange.await, Err(CapError::Denied(_)))); + assert!( + evidence.possible_until().is_none(), + "route denial is not send evidence" + ); assert_eq!( recv.recv(&mut [0u8; 32]).unwrap_err().kind(), io::ErrorKind::WouldBlock @@ -1281,6 +1355,68 @@ pub(crate) mod send_tests { } } + #[tokio::test] + async fn map_syscall_error_still_records_possible_ownership() { + let _frozen = crate::test_time::FrozenTime::new(); + let sock = tokio::net::UdpSocket::bind((Ipv4Addr::LOCALHOST, 0)) + .await + .unwrap(); + let gate = SendGate::new(PolicyHandle::new(true), None); + let evidence = SendEvidence::new(0, Instant::now()); + // A native IPv6 destination on an IPv4 socket fails in the syscall; no LAN traffic. + let to = SocketAddr::V6(std::net::SocketAddrV6::new( + std::net::Ipv6Addr::LOCALHOST, + 9, + 0, + 0, + )); + let expected = SocketAddrV4::new(Ipv4Addr::LOCALHOST, 9); + let present = |a| match a { + SocketAddr::V4(a) => Some(a), + _ => None, + }; + let mut ignore = |_: &[u8]| Flow::Continue; + let result = exchange_on( + &sock, + &gate, + to, + expected, + &present, + &[b"map"], + Instant::now() + Duration::from_secs(1), + &mut ignore, + Some(&evidence), + ) + .await; + assert!( + matches!(result, Err(CapError::Io { op: "send", .. })), + "{result:?}" + ); + assert!( + evidence.possible_until().is_some(), + "syscall errors cannot prove absence" + ); + let first = evidence.possible_until().unwrap(); + tokio::time::advance(Duration::from_secs(2)).await; + let retry = exchange_on( + &sock, + &gate, + to, + expected, + &present, + &[b"map"], + Instant::now() + Duration::from_secs(1), + &mut ignore, + Some(&evidence), + ) + .await; + assert!(matches!(retry, Err(CapError::Io { op: "send", .. }))); + assert!( + evidence.possible_until().unwrap() >= first + Duration::from_secs(2), + "latest attempt must advance the possible grant" + ); + } + type Hook = (std::sync::mpsc::Sender<()>, std::sync::mpsc::Receiver<()>); std::thread_local! { static NOWAIT_HOOK: std::cell::RefCell> = const { std::cell::RefCell::new(None) }; @@ -1337,6 +1473,7 @@ pub(crate) mod send_tests { let send = tokio::net::UdpSocket::bind((Ipv4Addr::LOCALHOST, 0)) .await .unwrap(); + let evidence = SendEvidence::new(0, Instant::now()); let policy = PolicyHandle::new(true); let gate = SendGate::new(policy.clone(), None); let expected = match dest { @@ -1359,6 +1496,7 @@ pub(crate) mod send_tests { &requests, Instant::now() + Duration::from_millis(20), &mut on_reply, + Some(&evidence), )); poll_fn(|cx| { assert!(exchange.as_mut().poll(cx).is_pending()); @@ -1377,6 +1515,10 @@ pub(crate) mod send_tests { gate.invalidate(); } drop(exchange.await); + assert!( + evidence.possible_until().is_none(), + "no MAP syscall crossed the fence" + ); assert_eq!( recv.recv(&mut buf).unwrap_err().kind(), io::ErrorKind::WouldBlock, @@ -1412,6 +1554,10 @@ pub(crate) mod send_tests { }); } pub(crate) struct ClearHook; + pub(crate) fn set_send_hook(hook: impl FnMut() + 'static) -> ClearHook { + BEFORE_SEND.with(|h| *h.borrow_mut() = Some(Box::new(hook))); + ClearHook + } impl Drop for ClearHook { fn drop(&mut self) { BEFORE_SEND.with(|hook| drop(hook.borrow_mut().take())); @@ -1427,6 +1573,7 @@ pub(crate) mod send_tests { .await .unwrap(); send.writable().await.unwrap(); + let evidence = SendEvidence::new(usize::from(between), Instant::now()); let policy = PolicyHandle::new(true); let gate = Arc::new(SendGate::new(policy.clone(), None)); let entered = Arc::new(std::sync::atomic::AtomicBool::new(false)); @@ -1479,6 +1626,7 @@ pub(crate) mod send_tests { reqs, Instant::now() + Duration::from_millis(20), &mut on_reply, + Some(&evidence), ) .await; assert!( @@ -1492,6 +1640,10 @@ pub(crate) mod send_tests { "must receive zero datagrams after the fence returns" ); assert!(matches!(result, Err(CapError::Denied(_)))); + assert!( + evidence.possible_until().is_none(), + "a PMP address request is not MAP evidence" + ); } #[tokio::test] diff --git a/ts_portmapper/src/client.rs b/ts_portmapper/src/client.rs new file mode 100644 index 00000000..ac1a837b --- /dev/null +++ b/ts_portmapper/src/client.rs @@ -0,0 +1,1297 @@ +//! The lifecycle client: one UDP mapping for the node's underlay port, created, renewed, expired +//! and released with generation-guarded transitions. +//! +//! # Lifecycle and generations +//! +//! All state lives in one `std::sync::Mutex` that is never held across an `.await`. Every +//! invalidation (local port change, network change, network down, epoch rollback, disable, close) +//! bumps `Lifecycle::generation`, aborts the in-flight attempt and takes the mapping out. An attempt snapshots +//! the generation when it starts; when its I/O finishes it commits only if the generation is +//! unchanged and the client still runs. A stale success is never installed: its rule goes to the +//! owned-rule ledger as orphaned and is deleted like any other leftover. +//! +//! # Router state +//! +//! A rule is recorded in the ledger (`owned`) **before** its MAP is sent, so a rule whose +//! reply was lost, or that committed after an invalidation, is still deletable. Deletes go only to +//! a network that still validates, only while the policy allows LAN sends, and only before the +//! router's grant ends. The ledger holds at most four rules (`owned::MAX_OWNED`). +//! +//! # Scheduling +//! +//! [`crate::schedule`] keeps attempt spacing, renewal and expiry apart: a failed renewal keeps the +//! mapping until its local expiry and retries on backoff, and expiry happens whatever the activity. + +use std::{ + collections::{HashMap, HashSet}, + fmt, + future::Future, + net::{Ipv4Addr, SocketAddrV4}, + num::{NonZeroU16, NonZeroU32}, + pin::Pin, + sync::{Arc, Mutex, MutexGuard}, + time::{Duration, SystemTime}, +}; + +use tokio::{ + sync::{Notify, watch}, + task::{AbortHandle, JoinSet}, + time::Instant, +}; + +use crate::{ + Gateway, GatewayCap, GatewayLookup, LookupError, PolicyHandle, PxpTransport, + cap::{CapError, LanDest, NetGen}, + clock::{Clock, SystemClock}, + gateway::{GatewayOverride, SystemLookup}, + is_advertisable, is_plausible_external, + oneshot::{self, MapError, Proto, ProtoSet, Rule, delete_request_bytes, lease_deadline}, + owned::{LeftReason, OwnedRule, OwnedRules, RuleId}, + pcp, pmp, + schedule::{Failure, Lease, REQUESTED_LIFETIME, Schedule}, +}; + +#[path = "lifecycle.rs"] +mod lifecycle; +use lifecycle::*; + +/// How long a positive probe result is trusted (Go `trustServiceStillAvailableDuration`). +pub const PROBE_CACHE: Duration = Duration::from_secs(10 * 60); +/// After a probe this recent found nothing, an attempt is not even tried (Go: 5 s). +pub const RECENT_PROBE: Duration = Duration::from_secs(5); +/// A wall-clock vs monotonic disagreement larger than this is taken as a resume or a clock jump. +pub const CLOCK_SKEW: Duration = Duration::from_secs(30); +/// The most distinct warnings remembered per network generation (for warn-once), in each of the +/// two budgets (see `Lifecycle::warned_router`). +const MAX_WARNED: usize = 32; + +/// Builds the capability for a gateway. Building is also how an old network is revalidated before +/// a delete is sent to it: a gateway whose interface no longer holds its address fails to build. +pub trait CapFactory: Send + Sync + 'static { + /// Validate `gateway` and build its capability for `net_gen`, sharing `policy`. + fn build( + &self, + gateway: &Gateway, + net_gen: NetGen, + policy: &PolicyHandle, + ) -> Result, CapError>; +} + +/// The production factory: [`GatewayCap::validate`]. +pub struct SystemCaps { + lookup: Arc, +} + +impl SystemCaps { + /// A factory whose source-IP-mode caps revalidate against `lookup`. + pub fn new(lookup: Arc) -> Self { + Self { lookup } + } +} + +impl CapFactory for SystemCaps { + fn build( + &self, + gateway: &Gateway, + net_gen: NetGen, + policy: &PolicyHandle, + ) -> Result, CapError> { + GatewayCap::validate( + gateway.clone(), + net_gen, + policy.clone(), + Some(self.lookup.clone()), + ) + } +} + +/// Client configuration. +pub struct Config { + /// Where the gateway comes from. + pub lookup: Arc, + /// How capabilities are built. + pub caps: Arc>, + /// The time source. + pub clock: Arc, + /// The protocols that may be used (both by default). + pub protos: ProtoSet, + /// Advertise an external address that is private, shared (CGNAT) or otherwise not publicly + /// reachable. `true` (the default) is Go's behaviour; the mapping is kept and renewed either + /// way. + pub advertise_nonpublic_external: bool, + /// The lifetime requested for every mapping. + pub lifetime: NonZeroU32, +} + +impl Config { + /// The production configuration: the OS gateway lookup (or a validated override), real caps + /// and clocks, both protocols, Go's advertising default. + pub fn system(gateway_override: Option) -> Self { + let lookup: Arc = Arc::new(SystemLookup::new(gateway_override)); + Self { + caps: Arc::new(SystemCaps::new(lookup.clone())), + lookup, + clock: Arc::new(SystemClock), + protos: ProtoSet::BOTH, + advertise_nonpublic_external: true, + lifetime: requested_lifetime(), + } + } +} + +/// [`REQUESTED_LIFETIME`] as a `NonZeroU32`. +pub const fn requested_lifetime() -> NonZeroU32 { + match NonZeroU32::new(REQUESTED_LIFETIME) { + Some(l) => l, + None => panic!("nonzero"), + } +} + +/// Why mapping is disabled at runtime. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DisableReason { + /// Control's `only-tcp-443` attribute: no UDP at all, deletes included. + OnlyTcp443, +} + +/// What the client knows about the LAN's port-mapping services. `None` means "not checked". +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct ProbeFacets { + /// UPnP-IGD (never probed in this phase, so always `None`). + pub upnp: Option, + /// NAT-PMP. + pub pmp: Option, + /// PCP. + pub pcp: Option, +} + +/// A mapping as published. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PortmapEndpoint { + /// The router's external address and port. + pub external: SocketAddrV4, + /// The protocol that holds it. + pub proto: Proto, + /// When local trust ends, on the client's clock. + pub expires_at: Instant, +} + +/// The published state. A `watch` carries the latest value and may coalesce intermediate ones; +/// consumers re-read it, and nothing promises exactly-once delivery. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct PortmapState { + /// The client's generation when this was published. + pub generation: u64, + /// The current mapping, if any. + pub mapping: Option, + /// Whether the mapping may be offered to peers (after the advertising policy). Never true + /// without a mapping. + pub advertise: bool, + /// The probe facets. + pub facets: ProbeFacets, +} + +impl PortmapState { + /// The endpoint to advertise, if any: the mapping when [`advertise`](Self::advertise) holds. + pub fn advertised(&self) -> Option { + self.mapping.filter(|_| self.advertise) + } +} + +/// A client error. +#[derive(Debug, thiserror::Error)] +pub enum Error { + /// Mapping is disabled (policy or `only-tcp-443`). + #[error("port mapping is disabled")] + Disabled, + /// The client is closing or closed. + #[error("port mapper closed")] + Closed, + /// No port-mapping service is available. + #[error("no port-mapping service")] + NoServices, + /// The local port is 0: never mapped (a request with internal port 0 means "all ports"). + #[error("local port 0 is never mapped")] + LocalPortZero, + /// No local port was set yet. + #[error("no local port")] + NoLocalPort, + /// No usable network (no default route, or the gateway did not validate). + #[error("no usable gateway")] + NoNetwork, + /// The gateway lookup failed. + #[error("{0}")] + Gateway(LookupError), + /// The capability refused or the OS failed. + #[error("{0}")] + Cap(CapError), + /// The exchange failed or the router refused. + #[error("{0}")] + Map(MapError), + /// The result arrived after an invalidation and was not installed. + #[error("stale result discarded")] + Stale, + /// An attempt is already in flight. + #[error("attempt in flight")] + Busy, + /// The next attempt is not due yet. + #[error("backing off")] + Backoff, + /// The owned-rule ledger is full. + #[error("too many outstanding router rules")] + LedgerFull, + /// The OS random source failed. + #[error("{0}")] + Nonce(pcp::NonceError), +} + +/// What [`Client::check_network`] found. +#[derive(Debug, PartialEq, Eq)] +pub enum NetCheck { + /// Same gateway, self address and interface. + Unchanged, + /// A new network; the old one's rules are released if it still validates. + Changed, + /// No default route, a lookup error or a gateway that does not validate: like network-down. + Down, + /// The client is not running. + Inactive, +} + +/// When the driver should next wake the client. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct ScheduleSnapshot { + /// The current mapping's local expiry. + pub expires_at: Option, + /// When to renew the current mapping: `max(renew_after, earliest attempt start)`. + pub renewal_due: Option, + /// The earliest next attempt start. + pub next_attempt: Option, + /// Set when a wanted attempt was refused by spacing or failed: when the next may start. + pub retry_at: Option, + /// Whether the refusal was a failure backoff (rather than only the 2 s floor). + pub backing_off: bool, + /// Cleanup/recovery deadline, independent of peer activity. Absent while disabled/closed. + pub cleanup_due: Option, +} + +struct Inner { + cfg: Config, + policy: PolicyHandle, + state: Mutex>, + tx: watch::Sender, + tasks: Mutex>, + /// All cleanup waiters (including public/direct awaits) observe the same shutdown fence. + stop_cleanup: watch::Sender, + operations_changed: Notify, +} + +/// The lifecycle client. Cheap to clone: clones share one state. +pub struct Client { + inner: Arc>, +} + +impl Clone for Client { + fn clone(&self) -> Self { + Self { + inner: self.inner.clone(), + } + } +} + +impl fmt::Debug for Client { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Client").finish_non_exhaustive() + } +} + +fn lock(m: &Mutex) -> MutexGuard<'_, S> { + m.lock().unwrap_or_else(std::sync::PoisonError::into_inner) +} + +/// A jitter factor in `[0.8, 1.2]` from the OS random source (1.0 if it fails). +fn jitter() -> f64 { + let mut b = [0u8; 4]; + if getrandom::fill(&mut b).is_err() { + return 1.0; + } + 0.8 + 0.4 * f64::from(u32::from_le_bytes(b)) / f64::from(u32::MAX) +} + +/// How an attempt's failure spaces the next attempt. +fn failure_kind(e: &MapError) -> Failure { + match e { + MapError::Pmp { + code: pmp::ResultCode::NotAuthorized, + .. + } + | MapError::Pcp { + code: pcp::ResultCode::NotAuthorized, + .. + } => Failure::NotAuthorized, + MapError::Pcp { lifetime, .. } | MapError::PcpUnverified { lifetime, .. } => { + Failure::Refused { + hold: Some(Duration::from_secs(u64::from(*lifetime))), + } + } + MapError::Pmp { .. } => Failure::Refused { hold: None }, + MapError::PmpUnverified { .. } + | MapError::PmpAddr { .. } + | MapError::NoServices { .. } + | MapError::Cap(_) + | MapError::Nonce(_) => Failure::Transient, + } +} + +/// A result code's name for a warn-once key: unknown codes share one key, so a router cannot +/// mint new keys by varying the code. +fn code_key(name: String, unknown: bool) -> String { + if unknown { "unknown".to_owned() } else { name } +} + +fn pmp_code_key(c: pmp::ResultCode) -> String { + code_key(c.to_string(), matches!(c, pmp::ResultCode::Unknown(_))) +} + +fn pcp_code_key(c: pcp::ResultCode) -> String { + code_key(c.to_string(), matches!(c, pcp::ResultCode::Unknown(_))) +} + +/// A capability error for a warn-once key: its kind, never free text. +fn cap_key(e: &CapError) -> String { + match e { + CapError::Denied(r) => format!("denied {r}"), + CapError::InvalidGateway(why) => format!("invalid-gateway {why}"), + CapError::ReservedPort(p) => format!("reserved-port {p}"), + CapError::NoRuntime => "no-runtime".to_owned(), + CapError::Io { + op, raw_os_error, .. + } => format!("io {op} {raw_os_error:?}"), + } +} + +/// What makes two failures "the same" for warn-once: the kind and result code, never the epoch, +/// lifetime, counts or reply text that change from reply to reply (the warning logs the error +/// itself). The flag says whether the key goes to the forgeable-router-reply budget. +fn warn_key(e: &MapError) -> (String, bool) { + match e { + MapError::Pmp { code, .. } => (format!("pmp {}", pmp_code_key(*code)), true), + MapError::PmpUnverified { code, .. } => { + (format!("pmp-unverified {}", pmp_code_key(*code)), true) + } + MapError::Pcp { code, .. } => (format!("pcp {}", pcp_code_key(*code)), false), + MapError::PcpUnverified { code, .. } => { + (format!("pcp-unverified {}", pcp_code_key(*code)), true) + } + MapError::PmpAddr { code, .. } => (format!("pmp-addr {}", pmp_code_key(*code)), true), + MapError::NoServices { malformed } => ( + if malformed.is_some() { + "no-services malformed".to_owned() + } else { + "no-services".to_owned() + }, + true, + ), + MapError::Cap(c) => (format!("cap {}", cap_key(c)), false), + MapError::Nonce(_) => ("nonce".to_owned(), false), + } +} + +/// Whether a failed create may still have left a rule on the router. Only a refusal tied to our +/// own request proves it did not. +fn may_exist(e: &MapError) -> bool { + !matches!(e, MapError::Pmp { .. } | MapError::Pcp { .. }) +} + +impl Client { + /// A client and the receiver its state is published on. No I/O happens until the driver (or + /// a caller) asks for it. + pub fn new(cfg: Config) -> (Self, watch::Receiver) { + let (tx, rx) = watch::channel(PortmapState::default()); + let client = Self { + inner: Arc::new(Inner { + cfg, + policy: PolicyHandle::new(true), + state: Mutex::new(Lifecycle::new()), + tx, + tasks: Mutex::new(JoinSet::new()), + stop_cleanup: watch::channel(false).0, + operations_changed: Notify::new(), + }), + }; + (client, rx) + } + + /// A new receiver of the published state. + pub fn subscribe(&self) -> watch::Receiver { + self.inner.tx.subscribe() + } + + /// The policy shared by every capability this client builds. + pub fn policy(&self) -> &PolicyHandle { + &self.inner.policy + } + + fn now(&self) -> Instant { + self.inner.cfg.clock.now() + } + + fn state(&self) -> MutexGuard<'_, Lifecycle> { + lock(&self.inner.state) + } + + // --------------------------------------------------------------------------------------- + // Publishing + + fn snapshot(&self, st: &mut Lifecycle) -> PortmapState { + let mapping = st + .mapping + .as_ref() + .filter(|_| !st.rebind_pending) + .map(|h| PortmapEndpoint { + external: h.m.external, + proto: h.m.proto, + expires_at: h.lease.expires_at, + }); + let advertise = mapping.is_some_and(|m| { + let ip = *m.external.ip(); + let valid = m.external.port() != 0 && is_plausible_external(ip); + let allowed = self.inner.cfg.advertise_nonpublic_external || is_advertisable(ip); + if valid && !allowed && st.note_once(format!("suppress {}", m.external)) { + tracing::info!( + external = %m.external, + "portmapper: not advertising a non-public external address; the mapping is \ + kept and renewed" + ); + } + valid && allowed + }); + PortmapState { + generation: st.generation, + mapping, + advertise, + facets: st.facets, + } + } + + fn publish(&self, st: &mut Lifecycle) { + let s = self.snapshot(st); + self.inner.tx.send_replace(s); + } + + // --------------------------------------------------------------------------------------- + // Invalidation + + /// Spawn a cleanup of orphaned rules, if a runtime is available (otherwise the next tick or + /// close does it). + fn spawn_cleanup(&self) { + if tokio::runtime::Handle::try_current().is_err() { + return; + } + let me = self.clone(); + let mut tasks = lock(&self.inner.tasks); + while tasks.try_join_next().is_some() {} + tasks.spawn(async move { + me.cleanup().await; + }); + } + + // --------------------------------------------------------------------------------------- + // Inputs from the driver + + /// Set the local (underlay) port to map. A change invalidates the mapping and releases its + /// rule. Port 0 is never mapped: it returns [`Error::LocalPortZero`] and invalidates. + pub fn set_local_port(&self, port: u16) -> Result<(), Error> { + let new = NonZeroU16::new(port); + let changed = { + let mut st = self.state(); + let changed = st.port_changed(new); + if changed { + self.publish(&mut st); + } + changed + }; + if changed { + self.spawn_cleanup(); + } + new.map(drop).ok_or(Error::LocalPortZero) + } + + /// The underlay socket was rebound: `Some(port)` on success, `None` when the rebind left no + /// socket. Withdraws synchronously without OS calls. A same-port mapping is kept privately + /// until asynchronous network validation restores it; a changed port or failed rebind + /// invalidates ownership for cleanup. + pub fn note_rebind(&self, port: Option) { + let Some(port) = port else { + { + let mut st = self.state(); + st.rebind_failed(); + self.publish(&mut st); + } + self.spawn_cleanup(); + return; + }; + let same = { + let st = self.state(); + st.local_port == NonZeroU16::new(port) + }; + if same { + let mut st = self.state(); + st.rebind_started(true); + self.publish(&mut st); + } else { + let _ignore = self.set_local_port(port); + let mut st = self.state(); + st.rebind_started(false); + self.publish(&mut st); + } + } + + /// The network went down: invalidate without releasing anything (a delete could reach the + /// wrong router). + pub fn note_network_down(&self) { + let mut st = self.state(); + // A closed/closing client may still have deferred release duty in an operation + // finalizer. Network-down must fence those capabilities too. + st.network_down("network down"); + self.publish(&mut st); + } + + /// Look the gateway up and compare it with the current network (run by the driver's + /// periodic poll, independent of activity). A new identity invalidates and releases the old + /// rules to the old network only if it still validates; no gateway, a lookup error or a + /// gateway that fails validation behaves like network-down. + pub async fn check_network(&self) -> NetCheck { + let (generation, candidate_gen, known) = { + let mut st = self.state(); + if st.phase != Phase::Running { + return NetCheck::Inactive; + } + let g = st.network_lookup_started(); + ( + st.generation, + g, + st.net + .as_ref() + .map(|n| (n.gateway.clone(), n.net_gen, n.cap.is_valid())), + ) + }; + let lookup = self.inner.cfg.lookup.clone(); + let caps = self.inner.cfg.caps.clone(); + let policy = self.inner.policy.clone(); + let result = crate::blocking::run(move || { + let gw = match lookup.lookup() { + Ok(Some(gw)) => gw, + Ok(None) => return Discovery::Down(None), + Err(e) => return Discovery::Down(Some(e)), + }; + let net_gen = match known { + Some((old, n, valid)) if old.same_network(&gw) => { + if valid { + return Discovery::Unchanged; + } + n + } + _ => candidate_gen, + }; + match caps.build(&gw, net_gen, &policy).and_then(|cap| { + caps.build(&gw, net_gen, &policy) + .map(|release| (cap, release)) + }) { + Ok((cap, release_cap)) => Discovery::Ready(gw, net_gen, cap, release_cap), + Err(e) => Discovery::Rejected(gw, e), + } + }) + .await; + let mut st = self.state(); + if st.phase != Phase::Running || st.generation != generation { + return NetCheck::Inactive; + } + let (gw, net_gen, cap, release_cap) = match result { + Some(Discovery::Ready(gw, n, cap, release)) => (gw, n, cap, release), + Some(Discovery::Unchanged) => return NetCheck::Unchanged, + Some(Discovery::Rejected(gw, e)) => { + if st.warn_once(format!("gateway {gw} {}", cap_key(&e))) { + tracing::warn!(gateway = %gw, error = %e, "portmapper: gateway rejected"); + } + if st.net.is_some() { + st.network_down("gateway rejected"); + self.publish(&mut st); + } + return NetCheck::Down; + } + Some(Discovery::Down(err)) => { + if let Some(e) = err + && st.warn_once(format!("lookup {e}")) + { + tracing::warn!(error = %e, "portmapper: gateway lookup failed"); + } + if st.net.is_some() { + st.network_down("no usable default route"); + self.publish(&mut st); + } + return NetCheck::Down; + } + None => return NetCheck::Inactive, + }; + let same = st.network_validated(gw, net_gen, cap, release_cap); + self.publish(&mut st); + drop(st); + if !same { + self.spawn_cleanup(); + } + NetCheck::Changed + } + + /// Compare the wall clock's progress with the monotonic clock's since the last call. A + /// disagreement over [`CLOCK_SKEW`] (a resume from sleep, or a wall-clock jump) forces a fresh + /// probe and makes the mapping due for renewal now (still subject to attempt spacing). + /// + /// [`Instant`] stops while the host sleeps, but the router's clock does not: when the wall + /// clock moved further than the monotonic one, the trusted advertisement lease is aged by + /// the difference. Possible ownership is never shortened by an unverified wall-clock step. A mapping whose local trust ended during the sleep + /// is withdrawn at once; one that is still trusted keeps only what is left of its lease, so a + /// failed renewal cannot keep advertising it past the router's grant. Either way the new + /// state is published. Returns whether a jump was seen. A false positive (a forward wall-clock + /// step) costs at most one renewal, or one re-create when the step exceeds the lease. + pub fn check_clock(&self) -> bool { + let now = self.now(); + let wall = self.inner.cfg.clock.wall(); + let mut st = self.state(); + let Some(slept) = st.clock_observed(now, wall) else { + return false; + }; + tracing::info!( + slept_secs = slept.as_secs(), + "portmapper: resume or clock jump; re-probing and renewing" + ); + let changed = st.clock_aged(now, slept); + if changed { + self.publish(&mut st); + } + true + } + + /// Turn mapping off at runtime (control's `only-tcp-443`): the policy that every send checks + /// is cleared first, in-flight work is aborted, the mapping is withdrawn, and **no** further LAN + /// packet is sent, deletes included. Owned rules are left until re-enabled on the same network + /// (then deleted first) or until the router's grant ends. + pub fn disable(&self, why: DisableReason) { + self.inner.policy.set_lan_sends_allowed(false); + let mut st = self.state(); + if !st.policy_disabled(why) { + return; + } + self.publish(&mut st); + } + + /// Undo [`disable`](Self::disable). Leftover rules are deleted by the next + /// [`tick`](Self::tick) before anything else. + pub fn enable(&self) { + let mut st = self.state(); + if !st.policy_enabled(&self.inner.policy) { + return; + } + self.publish(&mut st); + } + + /// Whether a usable network is known. + pub fn has_network(&self) -> bool { + self.state().net.as_ref().is_some_and(|n| n.cap.is_valid()) + } + + /// Whether a mapping is held and not expired. + pub fn have_mapping(&self) -> bool { + let now = self.now(); + self.state() + .mapping + .as_ref() + .is_some_and(|h| !h.lease.expired(now)) + } + + /// The current mapping's local expiry, renewal and the next attempt start. + pub fn schedule(&self) -> ScheduleSnapshot { + let st = self.state(); + ScheduleSnapshot { + expires_at: st.mapping.as_ref().map(|h| h.lease.expires_at), + renewal_due: st.mapping.as_ref().map(|h| st.sched.renewal_due(&h.lease)), + next_attempt: st.sched.earliest_start(), + // Only while an attempt could actually start: a leftover request with mapping + // disabled or no local port would otherwise re-arm the driver every second. + retry_at: (st.retry_wanted + && st.phase == Phase::Running + && st.disabled.is_none() + && st.local_port.is_some()) + .then(|| st.sched.earliest_start()) + .flatten(), + backing_off: st.sched.failures > 0, + cleanup_due: (st.phase == Phase::Running + && st.disabled.is_none() + && self.inner.policy.lan_sends_allowed()) + .then(|| st.owned.next_cleanup(self.now())) + .flatten(), + } + } + + /// The real monotonic instant for `t` on the client's clock (for arming timers). + pub fn to_real(&self, t: Instant) -> Instant { + self.inner.cfg.clock.to_real(t) + } + + /// Withdraw the mapping if its local trust has ended, whatever the activity. Returns whether + /// it did. + pub fn expire_if_due(&self) -> bool { + let now = self.now(); + let mut st = self.state(); + let expired = st.mapping.as_ref().is_some_and(|h| h.lease.expired(now)); + if !expired { + return false; + } + st.lease_expired(now); + self.publish(&mut st); + true + } + + // --------------------------------------------------------------------------------------- + // Probe + + /// Probe for NAT-PMP and PCP, respecting the positive cache (a service seen within + /// [`PROBE_CACHE`] is not probed again). Returns the facets. + pub async fn probe(&self) -> Result { + let now = self.now(); + // A mapping committed after this instant is newer than any reply this probe gets. + let started = now; + let (generation, cap, want) = { + let mut st = self.state(); + if st.phase != Phase::Running { + return Err(Error::Closed); + } + if st.disabled.is_some() || !self.inner.policy.lan_sends_allowed() { + return Err(Error::Disabled); + } + let Some(net) = &st.net else { + return Err(Error::NoNetwork); + }; + let cap = net.cap.clone(); + let protos = self.inner.cfg.protos; + let want = ProtoSet { + pmp: protos.pmp && !st.probe.saw_pmp(now), + pcp: protos.pcp && !st.probe.saw_pcp(now), + }; + if !want.pmp && !want.pcp { + st.probe_cached(now); + let facets = st.facets; + return Ok(facets); + } + if st.probe.found_nothing && !st.sched.may_start(now) { + // Nothing answered last time and the backoff still holds: report that again. + return Ok(st.facets); + } + (st.generation, cap, want) + }; + let r = match oneshot::probe(&*cap, want).await { + Ok(r) => r, + Err(e) => { + let mut st = self.state(); + if st.warn_once(format!("probe {}", cap_key(&e))) { + tracing::warn!(error = %e, "portmapper: probe failed"); + } + return Err(Error::Cap(e)); + } + }; + let now = self.now(); + let mut st = self.state(); + if generation != st.generation || st.phase != Phase::Running || st.disabled.is_some() { + return Err(Error::Stale); + } + if !r.errors.is_empty() && st.warn_once_router("probe-malformed".to_owned()) { + tracing::warn!(errors = ?r.errors, "portmapper: malformed replies to the probe"); + } + let rolled_back = + st.probe_observed(&r, want, self.inner.cfg.protos, started, now, cap.gateway()); + self.publish(&mut st); + let facets = st.facets; + drop(st); + if rolled_back { + // An attempt that was in flight may have left a rule on the restarted router. + self.spawn_cleanup(); + } + Ok(facets) + } + + // --------------------------------------------------------------------------------------- + // Attempts + + /// Return the cached mapping if it is valid, starting a renewal in the background when it is + /// due; with no valid mapping, start creating one. Never blocks; every start respects attempt + /// spacing. + pub fn get_cached_mapping_or_start_creating_one(&self) -> Option { + let now = self.now(); + let mut st = self.state(); + if st.mapping.as_ref().is_some_and(|h| h.lease.expired(now)) { + st.lease_expired(now); + self.publish(&mut st); + } + if st.rebind_pending { + return None; + } + let out = st.mapping.as_ref().map(|h| (h.m.external, h.lease)); + match out { + Some((external, lease)) => { + if lease.renew_due(now) { + let _ignore = self.maybe_start_locked(&mut st, now); + } + Some(external) + } + None => { + let _ignore = self.maybe_start_locked(&mut st, now); + None + } + } + } + + /// Begin an attempt under the lock: check every precondition, choose the protocol and + /// rule, record the rule in the ledger **before** sending, and spawn the I/O. + fn maybe_start_locked(&self, st: &mut Lifecycle, now: Instant) -> Result<(), Error> { + let (id, ctx) = st.map_started(&self.inner.cfg, &self.inner.policy, now)?; + let rule_id = ctx.rule_id; + let guard = SettlementGuard { + client: self.clone(), + event: FinalizerEvent::Map { id, rule_id }, + }; + let me = self.clone(); + let abort = { + let mut tasks = lock(&self.inner.tasks); + while tasks.try_join_next().is_some() {} + tasks.spawn(async move { + let need_cleanup = me.run_attempt(ctx).await; + // Single-flight ends with the attempt, not with the cleanup that may follow it. + drop(guard); + if need_cleanup { + me.cleanup().await; + } + }) + }; + st.map_task_attached(id, abort); + Ok(()) + } + + /// Run one attempt's I/O and commit (or reject) its result. Returns whether orphaned rules + /// are waiting for a cleanup. + async fn run_attempt(&self, ctx: AttemptCtx) -> bool { + let res = oneshot::map_with_evidence( + &*ctx.cap, + &ctx.rule, + self.inner.cfg.lifetime, + ctx.prev.as_ref(), + &ctx.evidence, + ) + .await; + let now = self.now(); + let mut st = self.state(); + match res { + Ok(m) => { + let (publish, cleanup) = st.map_success(&ctx, m, now); + if publish { + self.publish(&mut st); + } + cleanup + } + Err(e) => st.map_refused(&ctx, e, now), + } + } + + // --------------------------------------------------------------------------------------- + // Cleanup, tick, close + + /// Delete orphaned rules: only while the policy allows LAN sends, only to a network that still + /// validates, and only before the router's possible grant ends. Unacknowledged rules keep + /// admission slots, with infrequent recovery attempts after the initial bounded retries. + /// Returns how many deletes were acknowledged. + pub async fn cleanup(&self) -> usize { + let mut stop = self.inner.stop_cleanup.subscribe(); + if *stop.borrow() { + return 0; + } + tokio::select! { + biased; + _ = stop.changed() => 0, + acked = self.cleanup_inner(false) => acked, + } + } + + async fn cleanup_inner(&self, closing: bool) -> usize { + let now = self.now(); + let claimed = { + let mut st = self.state(); + st.cleanup_started(closing, &self.inner.policy, now) + }; + let mut guard = SettlementGuard { + client: self.clone(), + event: FinalizerEvent::Delete(claimed.iter().map(|e| e.id).collect()), + }; + let mut acked = 0; + // An entry leaves the guard only once it is settled, so a cleanup cancelled mid-delete + // (close aborting it, or its budget running out) hands the entry back for another try. + for e in claimed { + let cap = if closing { + self.state() + .release_caps + .get(&e.id) + .filter(|cap| cap.is_valid()) + .cloned() + } else { + let caps = self.inner.cfg.caps.clone(); + let gateway = e.gateway.clone(); + let policy = self.inner.policy.clone(); + crate::blocking::run(move || caps.build(&gateway, e.net_gen, &policy)) + .await + .and_then(Result::ok) + }; + let Some(cap) = cap else { + self.state() + .delete_validation_failed(e.id, closing, self.now()); + tracing::debug!( + internal = e.rule.internal().get(), + "portmapper: release validation unavailable; ownership retained" + ); + guard.settled(e.id); + continue; + }; + if !self.state().delete_validated(e.id, closing, cap.clone()) { + continue; + } + if closing && cap.bind_mode() == crate::BindMode::SourceIp { + // Source-only sends require a fresh OS route check. Leave these rules to expiry + // on a timed/no-runtime release rather than initiating discovery during shutdown. + self.state().delete_fenced(e.id); + continue; + } + let r = oneshot::delete(&*cap, &e.rule).await; + let ok = r.is_ok(); + let mut st = self.state(); + match r { + Ok(()) => tracing::info!( + proto = %e.rule.proto(), + internal = e.rule.internal().get(), + gateway = %e.gateway.gw, + "portmapper: router rule released" + ), + Err(err) => tracing::debug!(error = %err, "portmapper: delete not acknowledged"), + } + if ok { + acked += 1; + st.delete_acknowledged(e.id, self.now()); + } else { + st.delete_unacknowledged(e.id, self.now()); + } + guard.settled(e.id); + } + acked + } + + /// One activity-driven round: delete leftovers first, find the network if none is known, + /// probe (cache-respecting), then return or start the mapping. + pub async fn tick(&self) -> Option { + if self.state().phase != Phase::Running { + return None; + } + self.cleanup().await; + if !self.has_network() { + self.check_network().await; + } + let _ignore = self.probe().await; + self.get_cached_mapping_or_start_creating_one() + } + + /// Close: stop new work first, abort and await in-flight work, then send the deletes allowed + /// by the ledger rules, all within `budget`. Idempotent. + pub async fn close(&self, budget: Duration) { + let deadline = tokio::time::Instant::now() + budget; + { + let mut st = self.state(); + if !st.close_started() { + return; + } + self.inner.stop_cleanup.send_replace(true); + self.publish(&mut st); + } + // If this close future is cancelled or panics at any await, its finalizer owns the + // terminal release transition, including operations whose guards have not run yet. + let _finish = SettlementGuard { + client: self.clone(), + event: FinalizerEvent::Close, + }; + let mut tasks = std::mem::take(&mut *lock(&self.inner.tasks)); + tasks.abort_all(); + let _ = tokio::time::timeout_at(deadline, async { + while tasks.join_next().await.is_some() {} + }) + .await; + // Public cleanup futures live in their callers, not our JoinSet. Cancel their inner + // futures and wait for claim guards to hand back reservations before taking them. + let _ = tokio::time::timeout_at(deadline, self.wait_for_operations()).await; + let _ = tokio::time::timeout_at(deadline, self.cleanup_inner(true)).await; + let mut st = self.state(); + st.close_finished(); + drop(st); + self.release_ready_final(); + } + + async fn wait_for_operations(&self) { + loop { + let changed = self.inner.operations_changed.notified(); + tokio::pin!(changed); + changed.as_mut().enable(); + if !self.state().owned.has_claims() { + return; + } + changed.await; + } + } + + /// The `Drop`-path release: no runtime needed, no reply awaited. Marks the client closed and + /// sends one unacknowledged delete per deletable rule. Does nothing after close, while the + /// policy forbids LAN sends, or for a network that does not validate. + pub fn release_blocking_best_effort(&self) { + { + let mut st = self.state(); + if !st.drop_requested() { + return; + } + self.inner.stop_cleanup.send_replace(true); + self.publish(&mut st); + } + lock(&self.inner.tasks).abort_all(); + self.release_ready_final(); + } + + /// Called by release and operation finalizers. Claim under the state lock, send outside it; + /// retain ownership even when the final send succeeds (no acknowledgement was awaited). + fn release_ready_final(&self) { + let claimed = { + let mut st = self.state(); + st.final_release_started(&self.inner.policy, self.now()) + }; + if claimed.is_empty() { + return; + } + let _guard = SettlementGuard { + client: self.clone(), + event: FinalizerEvent::FinalRelease(claimed.iter().map(|e| e.id).collect()), + }; + for e in claimed { + let cap = self + .state() + .release_caps + .get(&e.id) + .filter(|cap| cap.is_valid()) + .cloned(); + let Some(cap) = cap else { + log_left(&e, LeftReason::NetworkGone); + continue; + }; + let req = delete_request_bytes(&e.rule, e.gateway.self_ip); + match cap.send_nowait(LanDest::Pxp, &req) { + Ok(()) => { + tracing::info!(proto = %e.rule.proto(), internal = e.rule.internal().get(), + "portmapper: router rule release sent") + } + Err(err) => tracing::info!(error = %err, "portmapper: release not sent"), + } + } + } + + fn settle_finalizer(&self, event: &FinalizerEvent) { + match event { + FinalizerEvent::Close => { + self.release_blocking_best_effort(); + return; + } + FinalizerEvent::Map { id, rule_id } => self.state().map_finished(*id, *rule_id), + FinalizerEvent::Delete(ids) => self.state().cleanup_cancelled(ids), + FinalizerEvent::FinalRelease(ids) => self.state().final_release_finished(ids), + } + self.inner.operations_changed.notify_waiters(); + self.release_ready_final(); + } + + /// The number of rules in the owned-rule ledger (tests and diagnostics). + pub fn owned_rules(&self) -> usize { + self.state().owned.len() + } + + /// Snapshot for the independent ownership oracle. + #[cfg(test)] + pub(crate) fn owned_snapshot(&self) -> Vec { + self.state().owned.entries().to_vec() + } + + /// Cleanup diagnostics for other crates' fake-router tests. + #[cfg(feature = "test-harness")] + pub fn cleanup_snapshot(&self) -> Vec { + self.state() + .owned + .entries() + .iter() + .map(|e| crate::testing::CleanupSnapshot { + delete_tries: e.delete_tries, + deleting: e.deleting(), + }) + .collect() + } + + /// Whether an attempt is in flight. + pub fn attempt_in_flight(&self) -> bool { + self.state().inflight.is_some() + } + + /// Reap finished background tasks; returns how many are still running. + pub fn reap(&self) -> usize { + let mut tasks = lock(&self.inner.tasks); + while tasks.try_join_next().is_some() {} + tasks.len() + } +} + +enum Discovery { + Unchanged, + Down(Option), + Rejected(Gateway, CapError), + Ready(Gateway, NetGen, Arc, Arc), +} + +struct AttemptCtx { + generation: u64, + cap: Arc, + net_gen: NetGen, + rule: Rule, + rule_id: RuleId, + existed: bool, + evidence: Arc, + prev: Option, +} + +/// One finalizer for MAP, cleanup and close. It only delivers its settlement event. +struct SettlementGuard { + client: Client, + event: FinalizerEvent, +} + +impl SettlementGuard { + fn settled(&mut self, id: RuleId) { + if let FinalizerEvent::Delete(ids) = &mut self.event { + ids.retain(|i| *i != id); + } + } +} + +impl Drop for SettlementGuard { + fn drop(&mut self) { + self.client.settle_finalizer(&self.event); + } +} + +fn log_left(e: &OwnedRule, why: LeftReason) { + let secs = e + .router_expiry() + .saturating_duration_since(Instant::now()) + .as_secs(); + tracing::info!( + proto = %e.rule.proto(), + internal = e.rule.internal().get(), + reason = ?why, + "portmapper: router rule left to expire (at most {secs} s)" + ); +} + +/// The client as the runtime drives it: object-safe, so the runtime needs no transport type. +pub trait PortMapper: Send + Sync + 'static { + /// A receiver of the published state. + fn subscribe(&self) -> watch::Receiver; + /// See [`Client::set_local_port`]. + fn set_local_port(&self, port: u16) -> Result<(), Error>; + /// See [`Client::note_rebind`]. + fn note_rebind(&self, port: Option); + /// See [`Client::note_network_down`]. + fn note_network_down(&self); + /// See [`Client::check_network`]. + fn check_network(&self) -> Pin + Send + '_>>; + /// See [`Client::check_clock`]. + fn check_clock(&self) -> bool; + /// See [`Client::disable`]. + fn disable(&self, why: DisableReason); + /// See [`Client::enable`]. + fn enable(&self); + /// See [`Client::expire_if_due`]. + fn expire_if_due(&self) -> bool; + /// See [`Client::schedule`]. + fn schedule(&self) -> ScheduleSnapshot; + /// See [`Client::to_real`]. + fn to_real(&self, t: Instant) -> Instant; + /// See [`Client::tick`]. + fn tick(&self) -> Pin> + Send + '_>>; + /// See [`Client::cleanup`]. + fn cleanup(&self) -> Pin + Send + '_>>; + /// See [`Client::close`]. + fn close(&self, budget: Duration) -> Pin + Send + '_>>; + /// See [`Client::release_blocking_best_effort`]. + fn release_blocking_best_effort(&self); + /// See [`Client::reap`]. + fn reap(&self) -> usize; +} + +impl PortMapper for Client { + fn subscribe(&self) -> watch::Receiver { + Client::subscribe(self) + } + fn set_local_port(&self, port: u16) -> Result<(), Error> { + Client::set_local_port(self, port) + } + fn note_rebind(&self, port: Option) { + Client::note_rebind(self, port); + } + fn note_network_down(&self) { + Client::note_network_down(self); + } + fn check_network(&self) -> Pin + Send + '_>> { + Box::pin(Client::check_network(self)) + } + fn check_clock(&self) -> bool { + Client::check_clock(self) + } + fn disable(&self, why: DisableReason) { + Client::disable(self, why); + } + fn enable(&self) { + Client::enable(self); + } + fn expire_if_due(&self) -> bool { + Client::expire_if_due(self) + } + fn schedule(&self) -> ScheduleSnapshot { + Client::schedule(self) + } + fn to_real(&self, t: Instant) -> Instant { + Client::to_real(self, t) + } + fn tick(&self) -> Pin> + Send + '_>> { + Box::pin(Client::tick(self)) + } + fn cleanup(&self) -> Pin + Send + '_>> { + Box::pin(Client::cleanup(self)) + } + fn close(&self, budget: Duration) -> Pin + Send + '_>> { + Box::pin(Client::close(self, budget)) + } + fn release_blocking_best_effort(&self) { + Client::release_blocking_best_effort(self); + } + fn reap(&self) -> usize { + Client::reap(self) + } +} + +#[cfg(test)] +#[path = "client_tests.rs"] +mod tests; diff --git a/ts_portmapper/src/client_tests.rs b/ts_portmapper/src/client_tests.rs new file mode 100644 index 00000000..ca14342a --- /dev/null +++ b/ts_portmapper/src/client_tests.rs @@ -0,0 +1,4839 @@ +//! Lifecycle client tests: every case runs against the fake router on loopback through +//! [`TestCap`], with a manual clock for the lease arithmetic. Nothing here reaches a real gateway. + +use std::{ + net::{Ipv4Addr, SocketAddrV4}, + sync::Arc, + time::Duration, +}; + +use super::*; +use crate::{ + oneshot::ProtoSet, + test_time::{FrozenTime, finish_at_deadline}, + testing::{ + AssignPort, Corruption, FakeRouter, LeaseClock, ManualClock, RequestKind, ResultOverride, + RouterOptions, ScriptedLookup, TestCap, TestCaps, + }, +}; + +/// The OS error a bind to a vanished local address returns. +#[cfg(unix)] +const ADDR_NOT_AVAILABLE: i32 = libc::EADDRNOTAVAIL; +#[cfg(windows)] +const ADDR_NOT_AVAILABLE: i32 = 10049; // WSAEADDRNOTAVAIL + +const GW1: Ipv4Addr = Ipv4Addr::new(192, 168, 1, 1); +const GW2: Ipv4Addr = Ipv4Addr::new(10, 0, 0, 1); +const PORT: u16 = 41641; + +fn gateway(gw: Ipv4Addr) -> Gateway { + Gateway { + gw, + // Source-consistent loopback mode: the PCP client IP is the real source address. + self_ip: Ipv4Addr::LOCALHOST, + prefix_len: 24, + if_index: 4, + if_name: "en0".to_owned(), + } +} + +struct Rig { + router: FakeRouter, + caps: Arc, + lookup: Arc, + clock: Arc, + lease_clock: LeaseClock, + client: Client, + rx: watch::Receiver, +} + +struct Opts { + router: RouterOptions, + protos: ProtoSet, + advertise_nonpublic: bool, +} + +impl Default for Opts { + fn default() -> Self { + Self { + router: RouterOptions { + enforce_pcp_client_ip: true, + ..RouterOptions::default() + }, + protos: ProtoSet::BOTH, + advertise_nonpublic: true, + } + } +} + +async fn rig_with(o: Opts) -> Rig { + let lease_clock = o.router.clock.clone(); + let router = FakeRouter::start(o.router).await.unwrap(); + let caps = TestCaps::new(); + caps.add(GW1, router.endpoint()); + let lookup = ScriptedLookup::new(gateway(GW1)); + let clock = ManualClock::new(); + let (client, rx) = Client::new(Config { + lookup: lookup.clone(), + caps: caps.clone(), + clock: clock.clone(), + protos: o.protos, + advertise_nonpublic_external: o.advertise_nonpublic, + lifetime: requested_lifetime(), + }); + Rig { + router, + caps, + lookup, + clock, + lease_clock, + client, + rx, + } +} + +/// Observe the lifecycle transition instead of guessing when a spawned close was polled. +async fn wait_for_close(client: &Client) { + tokio::time::timeout(secs(2), async { + while client.state().phase == Phase::Running { + tokio::task::yield_now().await; + } + }) + .await + .expect("close did not start"); +} + +async fn rig() -> Rig { + rig_with(Opts::default()).await +} + +impl Rig { + /// Advance the client's clock and the router's lease clock together. + fn advance(&self, d: Duration) { + self.clock.advance(d); + self.lease_clock.advance(d.as_secs()); + } + + /// Wait until no attempt or cleanup is in flight. + async fn settle(&self) { + let deadline = std::time::Instant::now() + Duration::from_secs(5); + loop { + assert_owned_invariant(&self.client); + if !self.client.attempt_in_flight() && self.client.reap() == 0 { + return; + } + assert!( + std::time::Instant::now() < deadline, + "client did not settle" + ); + tokio::time::sleep(Duration::from_millis(5)).await; + } + } + + /// Set the port, run one tick and wait for the attempt. + async fn map(&self) -> Option { + self.client.set_local_port(PORT).ok(); + assert_owned_invariant(&self.client); + self.client.tick().await; + assert_owned_invariant(&self.client); + self.settle().await; + self.client.subscribe().borrow().mapping.map(|m| m.external) + } + + fn state(&self) -> PortmapState { + self.client.subscribe().borrow().clone() + } + + fn maps(&self) -> u64 { + let c = self.router.counters(); + c.num_pmp_map_recv + c.num_pcp_map_recv + } + + fn creates_and_renewals(&self) -> Vec<(Option, Option, Option)> { + self.router + .requests() + .into_iter() + .filter(|r| { + matches!(r.kind, RequestKind::PmpMap | RequestKind::PcpMap) + && r.lifetime.is_some_and(|n| n > 0) + }) + .map(|r| (r.internal, r.suggested_external, r.lifetime)) + .collect() + } + + async fn shutdown(self) { + self.client.close(Duration::from_millis(500)).await; + self.router.shutdown().await; + } +} + +const fn secs(n: u64) -> Duration { + Duration::from_secs(n) +} + +// --------------------------------------------------------------------------------------------- +// Protocol selection, caching, renewal, expiry + +/// Protocol selection parity (`preferPCP`) over which services were seen and which +/// protocols are allowed; nothing seen → `NoServices` and backoff, with no MAP sent. +#[tokio::test] +async fn protocol_selection_matrix() { + for (pmp_on, pcp_on, protos, want) in [ + (true, true, ProtoSet::BOTH, Some(Proto::Pmp)), + (true, false, ProtoSet::BOTH, Some(Proto::Pmp)), + (false, true, ProtoSet::BOTH, Some(Proto::Pcp)), + ( + true, + true, + ProtoSet { + pmp: false, + pcp: true, + }, + Some(Proto::Pcp), + ), + ( + true, + true, + ProtoSet { + pmp: true, + pcp: false, + }, + Some(Proto::Pmp), + ), + (false, false, ProtoSet::BOTH, None), + ] { + let r = rig_with(Opts { + router: RouterOptions { + pmp: pmp_on, + pcp: pcp_on, + ..RouterOptions::default() + }, + protos, + ..Opts::default() + }) + .await; + let got = r.map().await; + let st = r.state(); + assert_eq!( + st.mapping.map(|m| m.proto), + want, + "{pmp_on} {pcp_on} {protos:?}" + ); + if want.is_none() { + assert!(got.is_none()); + assert_eq!(r.maps(), 0, "no MAP without a service"); + let s = r.client.schedule(); + assert!(s.next_attempt.unwrap() > r.clock.now() + secs(5), "backoff"); + } + r.shutdown().await; + } +} + +/// A call before `renew_after` sends nothing; no probe traffic within 10 min +/// of "seen"; probe traffic again after the cache expires and after an invalidation. +#[tokio::test] +async fn caches_respect_renewal_and_probe_windows() { + let r = rig().await; + assert!(r.map().await.is_some()); + let c0 = r.router.counters(); + assert_eq!((c0.num_pmp_public_addr_recv, c0.num_pcp_disco_recv), (2, 1)); + for _ in 0..20 { + r.advance(secs(10)); + r.client.tick().await; + r.settle().await; + } + let c = r.router.counters(); + assert_eq!( + c.num_pmp_map_recv, c0.num_pmp_map_recv, + "no renewal before renew_after" + ); + assert_eq!(c.num_pcp_disco_recv, 1, "probe cached"); + assert_eq!(c.num_pmp_public_addr_recv, c0.num_pmp_public_addr_recv); + // Past the 10-minute cache: probed again. + r.advance(secs(11 * 60)); + r.client.tick().await; + r.settle().await; + assert_eq!(r.router.counters().num_pcp_disco_recv, 2); + // An invalidation clears the cache. + r.client.note_network_down(); + r.client.check_network().await; + r.client.tick().await; + r.settle().await; + assert_eq!(r.router.counters().num_pcp_disco_recv, 3); + r.shutdown().await; +} + +/// Renewal past `trusted / 2` suggests the previous external port; a router-assigned new +/// port updates the published state; external ≠ internal flows through to the delete. +#[tokio::test] +async fn renewal_suggests_previous_port_and_publishes_changes() { + let r = rig().await; + r.router.set_assign_port(AssignPort::InternalPlus(1000)); + let ext = r.map().await.unwrap(); + assert_eq!( + ext, + SocketAddrV4::new(Ipv4Addr::new(203, 0, 113, 7), PORT + 1000) + ); + let st = r.state(); + assert!(st.advertise); + assert_eq!(st.advertised().unwrap().external, ext); + + r.advance(secs(3601)); + let mut rx = r.client.subscribe(); + rx.borrow_and_update(); + assert_eq!( + r.client.get_cached_mapping_or_start_creating_one(), + Some(ext) + ); + r.settle().await; + let last = *r.creates_and_renewals().last().unwrap(); + assert_eq!(last, (Some(PORT), Some(PORT + 1000), Some(7200))); + assert_eq!( + r.state().mapping.unwrap().external, + ext, + "unchanged external" + ); + + // The router moves the mapping on the next renewal: the change is published. + r.router.set_public_ip(Ipv4Addr::new(198, 51, 100, 9)); + r.advance(secs(3601)); + r.client.get_cached_mapping_or_start_creating_one(); + r.settle().await; + assert!(rx.has_changed().unwrap()); + assert_eq!( + r.state().mapping.unwrap().external, + SocketAddrV4::new(Ipv4Addr::new(198, 51, 100, 9), PORT + 1000) + ); + + // The delete carries the internal port, not the external one, and suggested port 0. + r.client.close(secs(1)).await; + let del = r.router.requests().last().cloned().unwrap(); + assert_eq!( + (del.internal, del.suggested_external, del.lifetime), + (Some(PORT), Some(0), Some(0)) + ); + assert!(r.router.mappings().is_empty()); + r.router.shutdown().await; +} + +/// PCP renewals and the delete reuse the creating nonce. +#[tokio::test] +async fn pcp_renewal_and_delete_reuse_the_nonce() { + let r = rig_with(Opts { + router: RouterOptions { + pmp: false, + enforce_pcp_client_ip: true, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + r.map().await.unwrap(); + let nonce = r.router.requests().last().unwrap().nonce.unwrap(); + r.advance(secs(3601)); + r.client.get_cached_mapping_or_start_creating_one(); + r.settle().await; + let renew = r.router.requests().last().cloned().unwrap(); + assert_eq!(renew.nonce, Some(nonce)); + assert_eq!(renew.suggested_external, Some(4242)); + r.client.close(secs(1)).await; + let del = r.router.requests().last().cloned().unwrap(); + assert_eq!((del.nonce, del.lifetime), (Some(nonce), Some(0))); + // Every PCP packet carried self_ip as the client IP, from self_ip. + for q in r.router.requests() { + if q.version == 2 { + assert_eq!(q.pcp_client_ip, Some(Ipv4Addr::LOCALHOST)); + assert_eq!(q.src.ip(), std::net::IpAddr::from(Ipv4Addr::LOCALHOST)); + } + } + assert!(r.router.mappings().is_empty()); + r.router.shutdown().await; +} + +/// Past `expires_at` the mapping is withdrawn with no tick and no peer activity; +/// the next call starts a create. +#[tokio::test] +async fn expiry_is_independent_of_activity() { + let r = rig().await; + r.router.set_lifetime_override(Some(60)); + r.map().await.unwrap(); + let mut rx = r.client.subscribe(); + rx.borrow_and_update(); + let s = r.client.schedule(); + assert_eq!(s.expires_at, Some(r.state().mapping.unwrap().expires_at)); + r.advance(secs(59)); + assert!(!r.client.expire_if_due()); + assert!(r.client.have_mapping()); + r.advance(secs(2)); + assert!(!r.client.have_mapping()); + assert!(r.client.expire_if_due()); + assert!(rx.has_changed().unwrap()); + assert!(r.state().mapping.is_none()); + // Nothing to release: the router's grant ended too. + assert_eq!(r.client.owned_rules(), 0); + let before = r.maps(); + assert!( + r.client + .get_cached_mapping_or_start_creating_one() + .is_none() + ); + r.settle().await; + assert_eq!(r.maps(), before + 1, "a new create"); + r.shutdown().await; +} + +/// The granted lifetime may differ from the request; the schedule trusts at most 2 h and the +/// ledger records the router's real bound. +#[tokio::test] +async fn granted_lifetimes_shape_the_schedule() { + for (granted, trusted) in [(600u32, 600u64), (86_400, 7200)] { + let r = rig().await; + r.router.set_lifetime_override(Some(granted)); + r.map().await.unwrap(); + let now = r.clock.now(); + let s = r.client.schedule(); + let exp = s.expires_at.unwrap(); + assert!(exp > now + secs(trusted - 2) && exp <= now + secs(trusted)); + let renew = s.renewal_due.unwrap(); + assert!(renew <= now + secs(trusted / 2)); + let owned = r.client.owned_snapshot(); + let router_expiry = owned[0].router_expiry(); + assert!(router_expiry > now + secs(u64::from(granted) - 2)); + r.shutdown().await; + } +} + +// --------------------------------------------------------------------------------------------- +// Single-flight, races, stale commits + +/// 50 concurrent calls start exactly one MAP; single-flight resets after success, error, +/// a panic in the attempt and an abort; the task set is reaped. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn single_flight() { + let r = rig().await; + r.client.set_local_port(PORT).unwrap(); + r.client.check_network().await; + r.client.probe().await.unwrap(); + let barrier = r.router.barrier(RequestKind::PmpMap); + let mut joins = Vec::new(); + for _ in 0..50 { + let c = r.client.clone(); + joins.push(tokio::spawn(async move { + c.get_cached_mapping_or_start_creating_one() + })); + } + for j in joins { + j.await.unwrap(); + } + barrier.reached().await; + assert!(r.client.attempt_in_flight()); + barrier.release(); + r.settle().await; + assert_eq!(r.router.counters().num_pmp_map_recv, 1); + assert!(r.client.have_mapping()); + + // Error: the router refuses the renewal; single-flight resets. + r.router.set_result( + RequestKind::PmpMap, + Some(ResultOverride { + code: 3, + lifetime: 0, + }), + ); + r.advance(secs(3601)); + r.client.get_cached_mapping_or_start_creating_one(); + r.settle().await; + assert!(!r.client.attempt_in_flight()); + r.router.set_result(RequestKind::PmpMap, None); + + // Panic inside the attempt future. + r.advance(secs(601)); + for cap in r.caps.built() { + cap.panic_next_exchange(); + } + r.client.get_cached_mapping_or_start_creating_one(); + r.settle().await; + assert!(!r.client.attempt_in_flight()); + + // Abort: an invalidation while the attempt waits on the router. + r.advance(secs(601)); + let barrier = r.router.barrier(RequestKind::PmpMap); + r.client.get_cached_mapping_or_start_creating_one(); + barrier.reached().await; + r.client.note_network_down(); + assert!(!r.client.attempt_in_flight()); + barrier.release(); + r.settle().await; + assert_eq!(r.client.reap(), 0); + r.shutdown().await; +} + +/// A success that lands after a local-port change, a network change, a disable or a close is +/// rejected: nothing stale is installed, the rule is orphaned and deleted, the newer state wins. +#[tokio::test] +async fn stale_commits_are_rejected_and_cleaned_up() { + for case in ["port", "network", "disable", "close"] { + let r = rig().await; + let caps2_router = FakeRouter::start(RouterOptions::default()).await.unwrap(); + r.caps.add(GW2, caps2_router.endpoint()); + r.client.set_local_port(PORT).unwrap(); + r.client.check_network().await; + r.client.probe().await.unwrap(); + let barrier = r.router.barrier(RequestKind::PmpMap); + r.client.get_cached_mapping_or_start_creating_one(); + barrier.reached().await; + assert_eq!( + r.router.mappings().len(), + 1, + "{case}: committed at the router" + ); + match case { + "port" => r.client.set_local_port(PORT + 1).unwrap(), + "network" => { + r.lookup.set(gateway(GW2)); + assert_eq!(r.client.check_network().await, NetCheck::Changed); + } + "disable" => r.client.disable(DisableReason::OnlyTcp443), + _ => {} + } + if case == "close" { + let c = r.client.clone(); + let close = tokio::spawn(async move { c.close(secs(2)).await }); + wait_for_close(&r.client).await; + barrier.release(); + close.await.unwrap(); + } else { + barrier.release(); + } + r.settle().await; + let st = r.state(); + assert!(st.mapping.is_none(), "{case}: stale mapping installed"); + match case { + "disable" => { + // No delete while disabled: the rule waits in the ledger. + assert_eq!(r.router.mappings().len(), 1); + assert_eq!(r.client.owned_rules(), 1); + r.client.enable(); + r.client.cleanup().await; + assert!(r.router.mappings().is_empty(), "deleted once re-enabled"); + } + _ => assert!( + r.router.mappings().iter().all(|m| m.internal != PORT), + "{case}: the stale rule was deleted" + ), + } + if case == "port" { + r.advance(secs(2)); + r.client.tick().await; + r.settle().await; + assert_eq!(r.state().mapping.unwrap().proto, Proto::Pmp); + assert_eq!( + r.router.mappings()[0].internal, + PORT + 1, + "newer state wins" + ); + } + caps2_router.shutdown().await; + r.shutdown().await; + } +} + +/// A router that commits but whose reply is lost, and an attempt aborted after the router +/// committed: the ledger holds the rule and cleanup or close deletes it. +#[tokio::test] +async fn rules_committed_without_a_reply_are_deleted() { + for proto_pcp in [false, true] { + let r = rig_with(Opts { + router: RouterOptions { + pmp: !proto_pcp, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + let kind = if proto_pcp { + RequestKind::PcpMap + } else { + RequestKind::PmpMap + }; + r.router.knobs_for(Some(kind)); + r.router.commit_then_drop_next(1); + assert!(r.map().await.is_none()); + assert_eq!( + r.router.mappings().len(), + 0, + "cleanup ran after the lost reply" + ); + assert_eq!(r.client.owned_rules(), 0); + + // Abort after send: committed, then invalidated before the reply. + r.advance(secs(60)); + let barrier = r.router.barrier(kind); + r.client.get_cached_mapping_or_start_creating_one(); + barrier.reached().await; + r.client.set_local_port(PORT + 7).unwrap(); + barrier.release(); + r.settle().await; + assert!(r.router.mappings().iter().all(|m| m.internal != PORT)); + r.shutdown().await; + } +} + +// --------------------------------------------------------------------------------------------- +// Network changes, local port, down, rebind + +/// A new gateway releases the old rule to the old router only and maps through +/// the new one; when the old network no longer validates, no delete is sent anywhere. +#[tokio::test] +async fn network_change_releases_only_to_a_validating_old_network() { + for old_validates in [true, false] { + let r = rig().await; + let router2 = FakeRouter::start(RouterOptions::default()).await.unwrap(); + r.caps.add(GW2, router2.endpoint()); + r.map().await.unwrap(); + assert_eq!(r.router.mappings().len(), 1); + r.caps.set_present(GW1, old_validates); + r.lookup.set(gateway(GW2)); + assert_eq!(r.client.check_network().await, NetCheck::Changed); + assert!(r.state().mapping.is_none()); + r.settle().await; + r.advance(secs(2)); + r.client.tick().await; + r.settle().await; + assert_eq!( + r.router.counters().num_deletes, + u64::from(old_validates), + "old_validates={old_validates}" + ); + assert_eq!(r.router.mappings().len(), usize::from(!old_validates)); + assert_eq!( + router2.counters().num_deletes, + 0, + "nothing for the old rule" + ); + assert_eq!( + router2.mappings().len(), + 1, + "new mapping via the new router" + ); + assert!(r.state().mapping.is_some()); + assert_eq!(r.client.owned_rules(), if old_validates { 1 } else { 2 }); + router2.shutdown().await; + r.shutdown().await; + } +} + +/// No default route, a lookup error, or network-down invalidates without +/// any release. +#[tokio::test] +async fn down_invalidates_without_release() { + for how in ["no-route", "error", "down"] { + let r = rig().await; + r.map().await.unwrap(); + match how { + "no-route" => { + r.lookup.set_no_route(); + assert_eq!(r.client.check_network().await, NetCheck::Down); + } + "error" => { + r.lookup.set_error(); + assert_eq!(r.client.check_network().await, NetCheck::Down); + } + _ => r.client.note_network_down(), + } + r.settle().await; + r.client.cleanup().await; + assert!(r.state().mapping.is_none(), "{how}"); + assert_eq!(r.router.counters().num_deletes, 0, "{how}: no release"); + // Kept (never deleted) only so a create on the same network can adopt it. + assert!( + r.client.owned_snapshot().iter().all(|e| !e.releasable()), + "{how}" + ); + assert!(!r.client.has_network()); + r.client.close(secs(1)).await; + assert_eq!( + r.router.counters().num_deletes, + 0, + "{how}: not even on close" + ); + r.router.shutdown().await; + } +} + +/// The same port is a no-op; a new port releases the old rule and maps the new one; port +/// 0 is refused, invalidates, and no MAP ever carries internal port 0. +#[tokio::test] +async fn local_port_changes() { + let r = rig().await; + r.map().await.unwrap(); + let before = r.router.requests().len(); + r.client.set_local_port(PORT).unwrap(); + r.settle().await; + assert_eq!(r.router.requests().len(), before, "same port: no I/O"); + assert!(r.client.have_mapping()); + + r.client.set_local_port(PORT + 1).unwrap(); + assert!(r.state().mapping.is_none(), "invalidated synchronously"); + r.settle().await; + // The 2 s floor between attempt starts holds across invalidations. + assert!(r.client.tick().await.is_none()); + let retry = r.client.schedule().retry_at.unwrap(); + assert!(!r.client.schedule().backing_off); + assert!(retry <= r.clock.now() + secs(2)); + r.advance(secs(2)); + r.client.tick().await; + r.settle().await; + let table: Vec = r.router.mappings().iter().map(|m| m.internal).collect(); + assert_eq!(table, [PORT + 1]); + + assert!(matches!( + r.client.set_local_port(0), + Err(Error::LocalPortZero) + )); + assert!(r.state().mapping.is_none()); + r.settle().await; + for _ in 0..3 { + r.advance(secs(700)); + r.client.tick().await; + r.settle().await; + } + assert!(r.state().mapping.is_none()); + for (internal, _, _) in r.creates_and_renewals() { + assert_ne!(internal, Some(0), "a MAP with internal port 0"); + } + assert!(r.router.mappings().is_empty(), "the old rule was released"); + r.shutdown().await; +} + +/// Client half of the rebind behaviour: a rebind on the same port and network restores the mapping with no I/O; +/// a rebind to a new port invalidates, releases and maps again; a failed rebind invalidates. +#[tokio::test] +async fn rebind_keeps_or_replaces_the_mapping() { + let r = rig().await; + let ext = r.map().await.unwrap(); + let mut rx = r.client.subscribe(); + rx.borrow_and_update(); + let before = r.router.requests().len(); + r.client.note_rebind(Some(PORT)); + assert!( + r.state().mapping.is_none(), + "withdrawn before OS validation" + ); + r.client.check_network().await; + assert!(rx.has_changed().unwrap(), "re-published for the magicsock"); + assert_eq!(r.state().mapping.unwrap().external, ext); + r.settle().await; + assert_eq!(r.router.requests().len(), before, "no I/O"); + + r.client.note_rebind(Some(PORT + 2)); + assert!(r.state().mapping.is_none()); + r.settle().await; + r.advance(secs(2)); + let before = r.creates_and_renewals().len(); + r.client.get_cached_mapping_or_start_creating_one(); + r.settle().await; + assert_eq!( + r.creates_and_renewals().len(), + before, + "changed-port rebind requires fresh network validation" + ); + r.client.tick().await; + r.settle().await; + let table: Vec = r.router.mappings().iter().map(|m| m.internal).collect(); + assert_eq!(table, [PORT + 2]); + + r.client.note_rebind(None); + assert!(r.state().mapping.is_none()); + r.settle().await; + assert!(r.router.mappings().is_empty(), "released: the port is gone"); + r.shutdown().await; +} + +// --------------------------------------------------------------------------------------------- +// Close and the Drop path + +/// Close deletes (PMP: internal port, external 0; PCP: lifetime 0 and the nonce), is +/// idempotent, stops new work before awaiting, and finishes within its budget with the router +/// gone. +#[tokio::test] +async fn close_releases_within_budget() { + let r = rig().await; + r.map().await.unwrap(); + r.client.close(secs(1)).await; + assert!(r.router.mappings().is_empty()); + let deletes = r.router.counters().num_deletes; + assert_eq!(deletes, 1); + r.client.close(secs(1)).await; + r.client.release_blocking_best_effort(); + assert_eq!(r.router.counters().num_deletes, deletes, "idempotent"); + assert!(r.client.tick().await.is_none()); + assert!( + r.client + .get_cached_mapping_or_start_creating_one() + .is_none() + ); + assert_eq!(r.router.counters().num_deletes, deletes); + r.router.shutdown().await; + + // Router gone: close still ends within its budget. + let r = rig().await; + r.map().await.unwrap(); + r.router.drop_next(1000); + let t = std::time::Instant::now(); + r.client.close(Duration::from_millis(300)).await; + assert!( + t.elapsed() < Duration::from_millis(600), + "{:?}", + t.elapsed() + ); + r.router.shutdown().await; + + // Close while an attempt waits on the router: no new work starts after it began. + let r = rig().await; + r.client.set_local_port(PORT).unwrap(); + r.client.check_network().await; + r.client.probe().await.unwrap(); + let barrier = r.router.barrier(RequestKind::PmpMap); + r.client.get_cached_mapping_or_start_creating_one(); + barrier.reached().await; + let c = r.client.clone(); + let close = tokio::spawn(async move { c.close(secs(2)).await }); + wait_for_close(&r.client).await; + let maps = r.creates_and_renewals().len(); + assert!( + r.client + .get_cached_mapping_or_start_creating_one() + .is_none() + ); + barrier.release(); + close.await.unwrap(); + assert_eq!( + r.creates_and_renewals().len(), + maps, + "nothing started after close began" + ); + assert!( + r.router.mappings().is_empty(), + "the in-flight rule was deleted" + ); + r.router.shutdown().await; +} + +/// The Drop-path release sends the deletes without a tokio runtime and without waiting +/// for replies; it does nothing after close. +#[tokio::test] +async fn drop_path_release_needs_no_runtime() { + for pcp in [false, true] { + let r = rig_with(Opts { + router: RouterOptions { + pmp: !pcp, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + r.map().await.unwrap(); + let c = r.client.clone(); + std::thread::spawn(move || { + assert!(tokio::runtime::Handle::try_current().is_err()); + c.release_blocking_best_effort(); + }) + .join() + .unwrap(); + let deadline = std::time::Instant::now() + secs(2); + while !r.router.mappings().is_empty() { + assert!(std::time::Instant::now() < deadline, "delete not received"); + tokio::time::sleep(Duration::from_millis(5)).await; + } + let del = r.router.requests().last().cloned().unwrap(); + assert_eq!(del.lifetime, Some(0)); + r.client.close(secs(1)).await; + assert_eq!( + r.router.counters().num_deletes, + 1, + "nothing after the Drop path" + ); + r.router.shutdown().await; + } +} + +// --------------------------------------------------------------------------------------------- +// Router errors, hostile replies, backoff + +/// Router refusals map to no mapping and backoff; NOT_AUTHORIZED holds ≥ 30 min; a PCP +/// error lifetime is honoured; ANNOUNCE refusals report `pcp = false`. +#[tokio::test] +async fn router_errors_back_off() { + for code in 1..=5u16 { + let r = rig().await; + r.router.set_result( + RequestKind::PmpMap, + Some(ResultOverride { code, lifetime: 0 }), + ); + assert!(r.map().await.is_none()); + assert_eq!(r.client.owned_rules(), 0, "a refused create left nothing"); + let wait = r.client.schedule().next_attempt.unwrap() - r.clock.now(); + if code == 2 { + assert!(wait >= secs(30 * 60) - secs(1), "{code}: {wait:?}"); + } else { + assert!(wait >= secs(7) && wait <= secs(13), "{code}: {wait:?}"); + } + // No hot loop: many ticks inside the backoff send nothing. + for _ in 0..50 { + r.client.tick().await; + } + r.settle().await; + assert_eq!(r.router.counters().num_pmp_map_recv, 1); + r.shutdown().await; + } + + // PCP MAP refusal with an error lifetime: the next attempt honours it. + let r = rig_with(Opts { + router: RouterOptions { + pmp: false, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + r.router.set_result( + RequestKind::PcpMap, + Some(ResultOverride { + code: 8, + lifetime: 600, + }), + ); + assert!(r.map().await.is_none()); + let wait = r.client.schedule().next_attempt.unwrap() - r.clock.now(); + assert!(wait >= secs(598), "{wait:?}"); + r.router.set_result( + RequestKind::PcpMap, + Some(ResultOverride { + code: 2, + lifetime: 5, + }), + ); + r.advance(secs(601)); + r.client.tick().await; + r.settle().await; + let wait = r.client.schedule().next_attempt.unwrap() - r.clock.now(); + assert!(wait >= secs(30 * 60) - secs(1), "NOT_AUTHORIZED: {wait:?}"); + r.shutdown().await; + + // ANNOUNCE refused: PCP present but unusable. + for code in [2u16, 12] { + let r = rig().await; + r.router.set_result( + RequestKind::PcpAnnounce, + Some(ResultOverride { code, lifetime: 0 }), + ); + r.client.check_network().await; + let f = r.client.probe().await.unwrap(); + assert_eq!((f.pmp, f.pcp, f.upnp), (Some(true), Some(false), None)); + r.shutdown().await; + } +} + +/// Hostile replies in create, renew and delete never install or roll back +/// state; a failed renewal keeps the mapping until it expires. +#[tokio::test] +async fn hostile_replies_through_the_lifecycle() { + let corruptions = [ + Corruption::Truncate, + Corruption::BadVersion, + Corruption::WrongOpcode, + Corruption::NoRBit, + Corruption::WrongNonce, + Corruption::WrongProto, + Corruption::WrongInternalPort, + Corruption::Garbage, + Corruption::Oversize, + ]; + for pcp in [false, true] { + let kind = if pcp { + RequestKind::PcpMap + } else { + RequestKind::PmpMap + }; + for c in corruptions { + if !pcp && matches!(c, Corruption::WrongNonce | Corruption::WrongProto) { + // NAT-PMP replies have no nonce or protocol field. + continue; + } + let r = rig_with(Opts { + router: RouterOptions { + pmp: !pcp, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + // Create. + r.router.knobs_for(Some(kind)); + r.router.corrupt_next(c); + assert!(r.map().await.is_none(), "{pcp} {c:?}: create"); + // The rule may exist: the ledger deleted it. + assert!(r.router.mappings().is_empty(), "{pcp} {c:?}"); + // A good create, then a corrupted renewal: the mapping stays. + r.advance(secs(700)); + let ext = r.map().await.unwrap(); + r.advance(secs(3601)); + r.router.corrupt_next(c); + r.client.get_cached_mapping_or_start_creating_one(); + r.settle().await; + assert_eq!( + r.state().mapping.unwrap().external, + ext, + "{pcp} {c:?}: renew" + ); + assert!(r.client.have_mapping()); + // A corrupted delete acknowledgement: retried by close. + r.router.corrupt_next(c); + r.client.close(secs(2)).await; + assert!(r.router.mappings().is_empty(), "{pcp} {c:?}: delete"); + r.router.shutdown().await; + } + } + + // Short (header-only) PCP errors and wrong-port replies: no mapping. + let r = rig_with(Opts { + router: RouterOptions { + pmp: false, + short_pcp_errors: true, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + r.router.set_result( + RequestKind::PcpMap, + Some(ResultOverride { + code: 8, + lifetime: 60, + }), + ); + assert!(r.map().await.is_none()); + r.router.set_result(RequestKind::PcpMap, None); + r.router.knobs_for(Some(RequestKind::PcpMap)); + // Every reply (the leftover's delete too) comes from the wrong port. + r.router.reply_from_wrong_port_next(10); + r.advance(secs(1800)); + r.client.tick().await; + r.settle().await; + assert!(r.state().mapping.is_none()); + r.shutdown().await; +} + +/// A duplicate of an earlier success arriving after a renewal does not roll state back. +#[tokio::test] +async fn duplicate_success_after_renewal_does_not_roll_back() { + let r = rig().await; + r.router.set_assign_port(AssignPort::Fixed(5000)); + r.router.knobs_for(Some(RequestKind::PmpMap)); + r.router.duplicate_next(1); + let first = r.map().await.unwrap(); + assert_eq!(first.port(), 5000); + r.advance(secs(3601)); + r.router.set_public_ip(Ipv4Addr::new(198, 51, 100, 9)); + r.client.get_cached_mapping_or_start_creating_one(); + r.settle().await; + let renewed = r.state().mapping.unwrap(); + assert_eq!(*renewed.external.ip(), Ipv4Addr::new(198, 51, 100, 9)); + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!(r.state().mapping.unwrap(), renewed); + r.shutdown().await; +} + +/// A failed renewal with a past `renew_after` is spaced by backoff: count MAPs over ten +/// simulated minutes of repeated kicks. +#[tokio::test] +async fn failed_renewal_never_storms() { + let r = rig().await; + r.router.set_lifetime_override(Some(7200)); + let ext = r.map().await.unwrap(); + r.advance(secs(3601)); + r.router.set_pmp(false); + r.router.set_pcp(false); + let maps = r.maps(); + let requests = r.router.counters().num_total_recv; + for _ in 0..600 { + r.advance(secs(1)); + r.client.get_cached_mapping_or_start_creating_one(); + r.client.get_cached_mapping_or_start_creating_one(); + r.settle().await; + assert!(r.client.schedule().renewal_due.unwrap() >= r.clock.now() - secs(1)); + } + let sent = r.router.counters().num_total_recv - requests; + assert_eq!(r.maps(), maps, "the router is silent"); + // 10 + 30 + 60 + 120 + 300 s (±20 %) fit in 600 s: at most 6 attempts, 2 datagrams each. + assert!(sent <= 12, "{sent} datagrams in 10 min"); + assert_eq!( + r.state().mapping.unwrap().external, + ext, + "kept until expiry" + ); + r.shutdown().await; +} + +/// With the router never answering a MAP, outstanding rules never exceed the ledger cap and +/// new attempts are refused while it is full. +#[tokio::test] +async fn ledger_bounds_outstanding_rules() { + let r = rig().await; + r.router.knobs_for(Some(RequestKind::PmpMap)); + r.router.commit_then_drop_next(10_000); + r.client.set_local_port(PORT).unwrap(); + let mut max = 0; + for i in 0..40u16 { + // A different port each round, so each create is a new rule. + r.client.set_local_port(PORT + i).unwrap(); + r.advance(secs(700)); + r.client.tick().await; + r.settle().await; + max = max.max(r.client.owned_rules()); + assert!(r.client.owned_rules() <= crate::owned::MAX_OWNED); + } + assert!(max >= 1); + assert!(r.router.mappings().len() <= 40); + r.shutdown().await; +} + +/// A delete is scoped to our internal port: a second client on the same host with another +/// internal port (Go's tailscaled on 41644) keeps its mapping. +#[tokio::test] +async fn delete_is_scoped_to_our_internal_port() { + let r = rig().await; + r.map().await.unwrap(); + let lookup = ScriptedLookup::new(gateway(GW1)); + let (other, _rx) = Client::new(Config { + lookup, + caps: r.caps.clone(), + clock: r.clock.clone(), + protos: ProtoSet::BOTH, + advertise_nonpublic_external: true, + lifetime: requested_lifetime(), + }); + other.set_local_port(41644).unwrap(); + other.tick().await; + while other.attempt_in_flight() { + tokio::time::sleep(Duration::from_millis(5)).await; + } + assert_eq!(r.router.mappings().len(), 2); + r.client.close(secs(1)).await; + let left: Vec = r.router.mappings().iter().map(|m| m.internal).collect(); + assert_eq!(left, [41644]); + other.close(secs(1)).await; + assert!(r.router.mappings().is_empty()); + r.router.shutdown().await; +} + +// --------------------------------------------------------------------------------------------- +// Epochs, clocks + +/// The probe-only epoch rule, for both protocols. +#[tokio::test] +async fn epoch_rollback_drops_without_release() { + for pcp in [false, true] { + let r = rig_with(Opts { + router: RouterOptions { + pmp: !pcp, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + r.router.set_epoch(5000); + r.map().await.unwrap(); + // 0 is ignored; equal and higher are fine (the cache must expire for a probe to be sent). + for epoch in [0, 5000, 9000] { + r.router.set_epoch(epoch); + r.advance(secs(601)); + r.client.probe().await.unwrap(); + assert!(r.client.have_mapping(), "{pcp} epoch {epoch}"); + } + // Reboot: the table is cleared, then the epoch restarts small and nonzero. + r.router.reboot(); + r.advance(secs(601)); + let mut rx = r.client.subscribe(); + rx.borrow_and_update(); + r.client.probe().await.unwrap(); + assert!(!r.client.have_mapping(), "{pcp}: rollback detected"); + assert!(rx.has_changed().unwrap()); + assert_eq!(r.router.counters().num_deletes, 0, "no release"); + assert_eq!(r.client.owned_rules(), 0); + // The cache was cleared: the next tick probes and maps again. + r.client.tick().await; + r.settle().await; + assert!(r.client.have_mapping()); + r.shutdown().await; + } +} + +/// A stale probe's epoch cannot drop a newer mapping. +#[tokio::test] +async fn stale_probe_cannot_drop_a_newer_mapping() { + let r = rig().await; + r.client.set_local_port(PORT).unwrap(); + r.client.check_network().await; + let frozen = FrozenTime::new(); + let barrier = r.router.barrier(RequestKind::PmpPublicAddr); + r.router.set_epoch(10); + let c = r.client.clone(); + let probe = tokio::spawn(async move { c.probe().await }); + barrier.reached().await; + // Meanwhile the network is re-found (an invalidation) and a mapping made at epoch 9000. + r.client.note_network_down(); + r.client.check_network().await; + r.router.set_epoch(9000); + barrier.release(); + assert!(matches!(probe.await.unwrap(), Err(Error::Stale))); + drop(frozen); + r.client.tick().await; + r.settle().await; + assert!(r.client.have_mapping()); + r.shutdown().await; +} + +/// A wall-clock jump or a resume re-probes and renews; a false positive costs exactly one +/// renewal. +#[tokio::test] +async fn clock_jump_forces_one_renewal() { + let r = rig().await; + r.map().await.unwrap(); + assert!(!r.client.check_clock()); + r.advance(secs(10)); + assert!(!r.client.check_clock(), "both clocks moved together"); + let maps = r.creates_and_renewals().len(); + let probes = r.router.counters().num_pcp_disco_recv; + r.clock.suspend(secs(3 * 3600)); + assert!(r.client.check_clock()); + r.advance(secs(3)); + r.client.tick().await; + r.settle().await; + assert_eq!(r.creates_and_renewals().len(), maps + 1, "renewed once"); + assert_eq!( + r.router.counters().num_pcp_disco_recv, + probes + 1, + "re-probed" + ); + for _ in 0..10 { + r.advance(secs(5)); + assert!(!r.client.check_clock()); + r.client.tick().await; + r.settle().await; + } + assert_eq!(r.creates_and_renewals().len(), maps + 1, "exactly one"); + r.shutdown().await; +} + +// --------------------------------------------------------------------------------------------- +// Policy, disable, notifications + +/// The advertising policy, both settings: a private or shared external address is +/// advertised by default (Go); with suppression it is kept, renewed (not recreated) and released +/// on close, but never advertised. +#[tokio::test] +async fn advertising_policy() { + for ip in [Ipv4Addr::new(10, 1, 2, 3), Ipv4Addr::new(100, 64, 1, 1)] { + for nonpublic in [true, false] { + let r = rig_with(Opts { + router: RouterOptions { + public_ip: ip, + ..Opts::default().router + }, + advertise_nonpublic: nonpublic, + ..Opts::default() + }) + .await; + r.map().await.unwrap(); + let st = r.state(); + assert!(st.mapping.is_some()); + assert_eq!(st.advertise, nonpublic, "{ip} {nonpublic}"); + assert_eq!(st.advertised().is_some(), nonpublic); + for _ in 0..3 { + r.advance(secs(3601)); + r.client.tick().await; + r.settle().await; + } + let creates = r + .creates_and_renewals() + .iter() + .filter(|(_, suggested, _)| *suggested == Some(0)) + .count(); + assert_eq!(creates, 1, "renewed, not recreated"); + assert_eq!(r.router.mappings().len(), 1); + r.client.close(secs(1)).await; + assert!(r.router.mappings().is_empty(), "released on close"); + r.router.shutdown().await; + } + } +} + +/// Live `only-tcp-443`: it aborts the in-flight attempt, refuses the next send at the +/// cap, withdraws the mapping, sends no delete and reports `Some(false)`; re-enabling on the same +/// network deletes leftovers first. +#[tokio::test] +async fn only_tcp_443_disables_all_lan_io() { + let r = rig().await; + r.map().await.unwrap(); + r.advance(secs(3601)); + let frozen = FrozenTime::new(); + let barrier = r.router.barrier(RequestKind::PmpMap); + r.client.get_cached_mapping_or_start_creating_one(); + barrier.reached().await; + let mut rx = r.client.subscribe(); + rx.borrow_and_update(); + r.client.disable(DisableReason::OnlyTcp443); + assert!(!r.client.attempt_in_flight(), "aborted"); + assert!(rx.has_changed().unwrap()); + let st = r.state(); + assert!(st.mapping.is_none()); + assert_eq!((st.facets.pmp, st.facets.pcp), (Some(false), Some(false))); + barrier.release(); + drop(frozen); + r.settle().await; + let total = r.router.counters().num_total_recv; + for _ in 0..5 { + r.advance(secs(700)); + r.client.tick().await; + r.client.cleanup().await; + r.settle().await; + } + r.client.close(secs(1)).await; + assert_eq!( + r.router.counters().num_total_recv, + total, + "no packet while disabled" + ); + assert_eq!(r.router.counters().num_deletes, 0); + // The cap itself refuses: a direct exchange through a built cap is denied by the policy. + let cap = r.caps.built().pop().unwrap(); + assert!(matches!( + oneshot::probe(&*cap, ProtoSet::BOTH).await, + Err(CapError::Denied(crate::DenyReason::Policy)) + )); + r.router.shutdown().await; + + // Re-enable on the same network: leftovers are deleted before anything else. + let r = rig().await; + r.map().await.unwrap(); + r.client.disable(DisableReason::OnlyTcp443); + assert_eq!(r.router.mappings().len(), 1, "left in place"); + r.client.enable(); + r.advance(secs(2)); + let n = r.router.requests().len(); + r.client.tick().await; + r.settle().await; + let after = r.router.requests(); + assert_eq!(after[n].lifetime, Some(0), "the delete went first"); + assert!(r.client.have_mapping()); + r.shutdown().await; +} + +/// Like Go's `TestUpdateEvent`, strengthened: state changes are observed for PMP and PCP success, +/// and for loss, expiry and disable. +#[tokio::test] +async fn notifications_for_every_protocol_and_loss() { + for pcp in [false, true] { + let r = rig_with(Opts { + router: RouterOptions { + pmp: !pcp, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + let mut rx = r.rx.clone(); + rx.borrow_and_update(); + r.map().await.unwrap(); + assert!(rx.has_changed().unwrap(), "{pcp}: success"); + assert_eq!( + rx.borrow_and_update().mapping.unwrap().proto, + if pcp { Proto::Pcp } else { Proto::Pmp } + ); + r.client.note_network_down(); + assert!(rx.has_changed().unwrap(), "{pcp}: loss"); + assert!(rx.borrow_and_update().mapping.is_none()); + r.router.set_lifetime_override(Some(30)); + r.client.check_network().await; + r.advance(secs(2)); + let got = r.map().await; + assert!( + got.is_some(), + "{pcp}: {:?} {:?}", + r.client.schedule(), + r.router.counters() + ); + rx.borrow_and_update(); + r.advance(secs(31)); + assert!(r.client.expire_if_due()); + assert!(rx.has_changed().unwrap(), "{pcp}: expiry"); + rx.borrow_and_update(); + r.client.disable(DisableReason::OnlyTcp443); + assert!(rx.has_changed().unwrap(), "{pcp}: disable"); + r.shutdown().await; + } +} + +/// Lifecycle lines at INFO, one WARN for 100 identical failures, no nonce in any line. +#[tokio::test] +#[tracing_test::traced_test] +async fn logging_is_informative_and_rate_limited() { + let r = rig_with(Opts { + router: RouterOptions { + pmp: false, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + r.router.set_assign_port(AssignPort::Fixed(6000)); + r.map().await.unwrap(); + let nonce = r.router.requests().last().unwrap().nonce.unwrap(); + r.router.set_assign_port(AssignPort::Fixed(6001)); + // Force a new port at renewal: delete the router's rule first. + r.router.reboot(); + r.router.set_epoch(u32::MAX); + r.advance(secs(3601)); + r.client.tick().await; + r.settle().await; + r.client.close(secs(1)).await; + r.router.shutdown().await; + // Left to expire: a network-down never deletes. + let r = rig().await; + r.map().await.unwrap(); + r.client.note_network_down(); + r.shutdown().await; + logs_assert(|lines: &[&str]| { + for want in [ + "mapping obtained", + "renewed with a new external address", + "mapping lost", + "router rule released", + "router rule left to expire", + ] { + if !lines + .iter() + .any(|l| l.contains(" INFO ") && l.contains(want)) + { + return Err(format!("no INFO line with {want:?}")); + } + } + Ok(()) + }); + + let r = rig().await; + r.router.set_result( + RequestKind::PmpMap, + Some(ResultOverride { + code: 3, + lifetime: 0, + }), + ); + r.client.set_local_port(PORT).unwrap(); + // A stale retry request never outlives the next attempt request. + r.client.tick().await; + r.settle().await; + r.client.disable(DisableReason::OnlyTcp443); + r.client.get_cached_mapping_or_start_creating_one(); + assert!(r.client.schedule().retry_at.is_none()); + r.client.enable(); + for _ in 0..100 { + r.advance(secs(700)); + r.client.tick().await; + r.settle().await; + } + r.client.note_network_down(); + r.shutdown().await; + logs_assert(|lines: &[&str]| { + let warns = lines + .iter() + .filter(|l| l.contains("WARN") && l.contains("mapping attempt failed")) + .count(); + if warns != 1 { + return Err(format!("{warns} warnings for 100 identical failures")); + } + let hex: String = nonce.iter().map(|b| format!("{b:02x}")).collect(); + let dec = format!("{nonce:?}"); + if lines.iter().any(|l| l.contains(&hex) || l.contains(&dec)) { + return Err("a nonce was logged".to_owned()); + } + Ok(()) + }); +} + +/// The `PortMapper` trait object drives the same client (the runtime holds only that). +#[tokio::test] +async fn trait_object_drives_the_client() { + let r = rig().await; + let m: Arc = Arc::new(r.client.clone()); + m.set_local_port(PORT).unwrap(); + assert_eq!(m.check_network().await, NetCheck::Changed); + m.tick().await; + r.settle().await; + assert!(m.subscribe().borrow().mapping.is_some()); + assert!(m.schedule().expires_at.is_some()); + m.close(secs(1)).await; + assert!(r.router.mappings().is_empty()); + assert!(r.lookup.calls() >= 1); + r.router.shutdown().await; +} + +// --------------------------------------------------------------------------------------------- +// Ownership, cleanup and cancellation regressions + +/// A failed create asks the driver for a retry at the backoff (so runner mode retries while +/// idle); the request is withdrawn while no attempt could start (disabled, no local port). +#[tokio::test] +async fn failed_create_asks_for_a_retry() { + let r = rig_with(Opts { + router: RouterOptions { + pcp: false, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + r.router.set_result( + RequestKind::PmpMap, + Some(ResultOverride { + code: 3, + lifetime: 0, + }), + ); + assert!(r.map().await.is_none()); + let s = r.client.schedule(); + assert!(s.backing_off); + assert_eq!(s.retry_at, s.next_attempt, "retry at the backoff"); + assert!(s.retry_at.unwrap() > r.clock.now()); + r.client.disable(DisableReason::OnlyTcp443); + assert_eq!(r.client.schedule().retry_at, None, "nothing can start"); + r.client.enable(); + assert!(r.client.schedule().retry_at.is_some()); + assert!(r.client.set_local_port(0).is_err()); + assert_eq!(r.client.schedule().retry_at, None, "no port to map"); + + // No service at all (NoServices) asks for a retry too. + let r2 = rig_with(Opts { + router: RouterOptions { + pmp: false, + pcp: false, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + assert!(r2.map().await.is_none()); + assert!(r2.client.schedule().retry_at.is_some()); + r2.shutdown().await; + r.shutdown().await; +} + +/// Same generation: a probe whose reply was built before a renewal committed, but +/// processed after it, cannot roll the renewed mapping back (renewals do not bump the +/// generation). For PCP, the next tick still has the mapping and the router holds one rule. +#[tokio::test] +async fn same_generation_probe_cannot_roll_back_a_renewal() { + for pcp in [false, true] { + let r = rig_with(Opts { + router: RouterOptions { + pmp: !pcp, + enforce_pcp_client_ip: true, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + r.router.set_epoch(5000); + r.map().await.unwrap(); + r.advance(secs(3601)); // renewal due; probe cache stale + let kind = if pcp { + RequestKind::PcpAnnounce + } else { + RequestKind::PmpPublicAddr + }; + let frozen = FrozenTime::new(); + let barrier = r.router.barrier(kind); + let c = r.client.clone(); + let probe = tokio::spawn(async move { c.probe().await }); + barrier.reached().await; // the probe's reply carries epoch 5000, held + tokio::time::advance(Duration::from_millis(1)).await; // renewal is strictly newer + r.router.set_epoch(5001); // a second later at the router + r.client.get_cached_mapping_or_start_creating_one(); // the renewal + while r.client.reap() != 0 { + tokio::task::yield_now().await; + } + assert!(r.client.have_mapping()); + barrier.release(); + drop(frozen); + probe.await.unwrap().unwrap(); + assert!(r.client.have_mapping(), "pcp={pcp}: renewal rolled back"); + assert_eq!(r.client.owned_rules(), 1, "pcp={pcp}"); + r.advance(secs(3)); + r.client.tick().await; + r.settle().await; + assert!(r.client.have_mapping(), "pcp={pcp}"); + assert!(!r.client.schedule().backing_off, "pcp={pcp}"); + assert_eq!(r.router.mappings().len(), 1, "pcp={pcp}"); + r.shutdown().await; + } +} + +/// An epoch rollback seen by a probe while an attempt of the same protocol is in flight keeps +/// that attempt's rule in the ledger as an orphan: the restarted router may commit it, and close +/// deletes it. +#[tokio::test] +async fn rollback_during_an_attempt_keeps_its_rule_deletable() { + // PCP only on both sides: a probe that also asked for NAT-PMP would wait out its whole + // 250 ms deadline for the silent NAT-PMP side, racing the held renewal's own 250 ms deadline, + // so the renewal could time out before the probe saw the rollback. + let r = rig_with(Opts { + router: RouterOptions { + pmp: false, + enforce_pcp_client_ip: true, + ..RouterOptions::default() + }, + protos: ProtoSet { + pmp: false, + pcp: true, + }, + ..Opts::default() + }) + .await; + r.router.set_epoch(5000); + r.map().await.unwrap(); + // The router reboots (table cleared, small epoch). The renewal falls due and is committed by + // the rebooted router, whose reply is held. + r.router.reboot(); + r.router.set_epoch(10); + r.advance(secs(3601)); + let frozen = FrozenTime::new(); + let barrier = r.router.barrier(RequestKind::PcpMap); + r.client.get_cached_mapping_or_start_creating_one(); + barrier.reached().await; + assert_eq!(r.router.mappings().len(), 1, "committed by the router"); + // A probe (another kick) sees the lower epoch while the renewal is in flight. + r.client.probe().await.unwrap(); + assert!(!r.client.have_mapping(), "rollback detected"); + assert_eq!(r.client.owned_rules(), 1, "the in-flight rule is kept"); + barrier.release(); + drop(frozen); + r.settle().await; + r.client.close(secs(1)).await; + assert!(r.router.mappings().is_empty(), "deleted on close"); + assert!(r.router.counters().num_deletes >= 1); + r.router.shutdown().await; +} + +/// A resume after the router's grant ended withdraws the mapping at once, even when the forced +/// renewal gets no answer; a shorter sleep keeps only what is left of the lease (Instant stops +/// during sleep, the router's clock does not). +#[tokio::test] +async fn resume_ages_the_lease() { + // Longer than the grant: withdrawn by check_clock itself, before any renewal. + let r = rig().await; + r.map().await.unwrap(); + assert!(!r.client.check_clock()); + let mut rx = r.client.subscribe(); + rx.borrow_and_update(); + r.clock.suspend(secs(5 * 3600)); + r.router.drop_next(10_000); + assert!(r.client.check_clock(), "resume noticed"); + assert!(!r.client.have_mapping()); + assert!(rx.has_changed().unwrap(), "published"); + assert_eq!(rx.borrow_and_update().advertised(), None); + r.advance(secs(3)); + r.client.tick().await; + r.settle().await; + assert_eq!(r.state().advertised(), None, "no renewal answered"); + r.shutdown().await; + + // Shorter than the grant: the lease is aged by the sleep, then expires on schedule. + let r = rig().await; + r.map().await.unwrap(); + let before = r.client.schedule().expires_at.unwrap(); + assert!(!r.client.check_clock()); + r.clock.suspend(secs(30 * 60)); + r.router.drop_next(10_000); + assert!(r.client.check_clock()); + let after = r.client.schedule().expires_at.unwrap(); + let aged = before.saturating_duration_since(after); + assert!( + aged > secs(30 * 60 - 1) && aged < secs(30 * 60 + 5), + "aged by {aged:?}" + ); + assert_eq!(r.state().mapping.unwrap().expires_at, after, "published"); + let entry = &r.client.owned_snapshot()[0]; + assert!( + entry.router_expiry() >= before, + "possible ownership remains conservative" + ); + assert!(r.client.have_mapping(), "still within the grant"); + r.advance(secs(3)); + r.client.tick().await; + r.settle().await; + assert!( + r.client.have_mapping(), + "a failed renewal keeps it until expiry" + ); + r.clock + .advance(after.saturating_duration_since(r.clock.now()) + secs(1)); + assert!(r.client.expire_if_due()); + assert_eq!(r.state().advertised(), None); + r.shutdown().await; +} + +/// A cleanup cancelled while its delete is in flight hands the entry back, so close sends the +/// delete again instead of leaving the rule on the router. +#[tokio::test] +async fn cancelled_cleanup_hands_the_rule_back() { + let r = rig().await; + r.map().await.unwrap(); + assert_eq!(r.router.mappings().len(), 1); + // The first delete is observable at a request barrier, but does not commit. + r.router.set_drop_deletes(true); + let frozen = FrozenTime::new(); + let barrier = r.router.barrier(RequestKind::PmpMap); + r.client.set_local_port(PORT + 1).unwrap(); + barrier.reached().await; + r.router.set_drop_deletes(false); + // Abort and join the cleanup before close: the RAII guard itself must return the claim. + let mut tasks = std::mem::take(&mut *lock(&r.client.inner.tasks)); + tasks.shutdown().await; + assert!( + !r.client.owned_snapshot()[0].deleting(), + "cancelled cleanup returned its claim" + ); + r.client.close(secs(1)).await; + barrier.release(); + drop(frozen); + assert!(r.router.mappings().is_empty(), "close re-sent the delete"); + assert_eq!(r.client.owned_rules(), 0); + r.router.shutdown().await; +} + +/// Simultaneous renewal: a renewal in flight when the local port changes or the socket is +/// rebound to a new port is stale; a rebind to the same port keeps the generation, and the +/// renewal commits. +#[tokio::test] +async fn renewal_racing_a_port_change_or_rebind() { + for case in ["port", "rebind", "rebind-same-port"] { + let r = rig().await; + r.map().await.unwrap(); + r.advance(secs(3601)); + let maps = r.maps(); + let frozen = FrozenTime::new(); + let barrier = r.router.barrier(RequestKind::PmpMap); + r.client.get_cached_mapping_or_start_creating_one(); + barrier.reached().await; + match case { + "port" => r.client.set_local_port(PORT + 1).unwrap(), + "rebind" => r.client.note_rebind(Some(PORT + 1)), + _ => r.client.note_rebind(Some(PORT)), + } + barrier.release(); + while r.client.reap() != 0 { + tokio::task::yield_now().await; + } + drop(frozen); + r.settle().await; + if case == "rebind-same-port" { + r.client.check_network().await; + assert_eq!(r.maps(), maps + 1, "{case}: one renewal, no delete"); + assert!(r.client.have_mapping(), "{case}: renewed"); + assert_eq!(r.client.owned_rules(), 1, "{case}"); + } else { + assert!( + r.state().mapping.is_none(), + "{case}: stale renewal installed" + ); + assert!( + r.router.mappings().iter().all(|m| m.internal != PORT), + "{case}: the old rule was deleted" + ); + r.advance(secs(2)); + r.client.tick().await; + r.settle().await; + assert_eq!( + r.router.mappings()[0].internal, + PORT + 1, + "{case}: newer state wins" + ); + } + r.shutdown().await; + } +} + +/// Through `tick`, repeated kicks (two ticks a second for ten simulated minutes) against a +/// router that stopped answering send a bounded number of datagrams, probes included. +#[tokio::test] +async fn repeated_kicks_never_storm() { + let r = rig().await; + r.router.set_lifetime_override(Some(7200)); + let ext = r.map().await.unwrap(); + r.advance(secs(3601)); + r.router.set_pmp(false); + r.router.set_pcp(false); + let total = r.router.counters().num_total_recv; + for _ in 0..600 { + r.advance(secs(1)); + r.client.tick().await; + r.client.tick().await; + r.settle().await; + } + let sent = r.router.counters().num_total_recv - total; + // Probes and attempts follow the backoff (10 + 30 + 60 + 120 + 300 s, ±20 %): a handful of + // two-datagram probes, not one per kick. + assert!(sent <= 16, "{sent} datagrams in 10 min"); + assert_eq!( + r.state().mapping.unwrap().external, + ext, + "kept until expiry" + ); + r.shutdown().await; +} + +/// Warn-once keys are bounded (no counts, reply text or unknown code values), and forgeable +/// router replies have their own budget, so they cannot use up the operational one. +#[tokio::test] +async fn warn_budgets_are_bounded_and_separate() { + let unknown = |c| MapError::Pmp { + code: pmp::ResultCode::Unknown(c), + epoch: 1, + }; + assert_eq!(warn_key(&unknown(100)), warn_key(&unknown(200))); + assert_eq!( + warn_key(&MapError::NoServices { + malformed: Some("3 datagram(s) from a source other than the gateway".to_owned()), + }), + warn_key(&MapError::NoServices { + malformed: Some("4 oversize datagram(s)".to_owned()), + }), + ); + assert!(warn_key(&unknown(1)).1, "NAT-PMP codes are forgeable"); + let pcp_refusal = MapError::Pcp { + code: pcp::ResultCode::NotAuthorized, + lifetime: 1, + epoch: 1, + }; + assert!( + !warn_key(&pcp_refusal).1, + "a nonce-verified refusal is operational" + ); + + let r = rig().await; + { + let mut st = r.client.state(); + for i in 0..1000 { + st.warn_once_router(format!("spoofed {i}")); + } + assert!(st.warned_router.len() <= MAX_WARNED); + assert!( + st.warn_once("lookup failed".to_owned()), + "operational budget untouched" + ); + assert!(!st.warn_once("lookup failed".to_owned())); + } + r.shutdown().await; +} + +/// Rules whose deletes never commit still count against admission. +#[tokio::test] +async fn outstanding_router_rules_are_bounded_when_deletes_fail() { + for refused in [false, true] { + let r = rig().await; + r.client.set_local_port(PORT).unwrap(); + r.client.check_network().await; + r.client.probe().await.unwrap(); + for i in 0..4 { + r.router.set_result(RequestKind::PmpMap, None); + r.client.get_cached_mapping_or_start_creating_one(); + r.settle().await; + assert!(r.client.have_mapping(), "slot {i}"); + r.router.set_drop_deletes(!refused); + if refused { + r.router.set_result( + RequestKind::PmpMap, + Some(ResultOverride { + code: 3, + lifetime: 0, + }), + ); + } + r.client.set_local_port(PORT + i + 1).unwrap(); + r.settle().await; + for _ in 0..4 { + r.client.cleanup().await; + } + r.advance(secs(3)); + } + assert_eq!(r.router.mappings().len(), 4); + r.router.set_result(RequestKind::PmpMap, None); + r.client.get_cached_mapping_or_start_creating_one(); + r.settle().await; + assert_eq!(r.router.mappings().len(), 4, "fifth MAP must be refused"); + assert_eq!(r.client.owned_rules(), 4); + assert!(!r.client.have_mapping()); + r.advance(secs(7201)); + r.client.tick().await; + r.settle().await; + assert_eq!(r.router.mappings().len(), 1, "expiry frees admission"); + r.router.set_drop_deletes(false); + r.shutdown().await; + } +} + +/// Discovery of PMP must not delete an aliased live PCP mapping. +#[tokio::test] +async fn renewal_keeps_pcp_when_pmp_becomes_available() { + let r = rig_with(Opts { + router: RouterOptions { + pmp: false, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + r.map().await.unwrap(); + let nonce = r.router.mappings()[0].nonce; + r.router.set_pmp(true); + r.advance(secs(3601)); + r.client.tick().await; + r.settle().await; + let table = r.router.mappings(); + assert_eq!( + table.len(), + 1, + "renewal must remain installed at the router" + ); + assert_eq!(table[0].nonce, nonce); + assert_eq!(r.state().mapping.unwrap().proto, Proto::Pcp); + assert_eq!(r.router.counters().num_deletes, 0); + r.shutdown().await; +} + +/// Losing a renewal reply must not lose the rule or its PCP nonce at expiry. +#[tokio::test] +async fn lost_renewal_preserves_ownership_through_expiry_and_close() { + for pcp in [false, true] { + let r = rig_with(Opts { + router: RouterOptions { + pmp: !pcp, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + r.router.set_lifetime_override(Some(60)); + r.map().await.unwrap(); + let nonce = r.router.mappings()[0].nonce; + r.advance(secs(31)); + r.router.set_lifetime_override(Some(7200)); + r.router.knobs_for(Some(if pcp { + RequestKind::PcpMap + } else { + RequestKind::PmpMap + })); + r.router.commit_then_drop_next(1); + r.client.get_cached_mapping_or_start_creating_one(); + r.settle().await; + assert_eq!(r.router.mappings()[0].nonce, nonce); + r.advance(secs(31)); + r.client.expire_if_due(); + assert!(!r.client.have_mapping()); + assert_eq!(r.client.owned_rules(), 1, "renewed router rule still owned"); + r.client.close(secs(1)).await; + assert!( + r.router.mappings().is_empty(), + "close deletes the renewed rule" + ); + r.router.shutdown().await; + } +} + +/// A nonce-less PMP fallback is not proof a PCP create was refused. +#[tokio::test] +async fn committed_pcp_with_only_pmp_fallback_is_cleaned_up() { + let r = rig_with(Opts { + router: RouterOptions { + pmp: false, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + r.router.knobs_for(Some(RequestKind::PcpMap)); + r.router + .spoof_before_next_reply(vec![0, 0x81, 0, 1, 0, 0, 0, 1]); + r.router.commit_then_drop_next(1); + assert!(r.map().await.is_none()); + assert!( + r.router.mappings().is_empty(), + "possibly committed PCP rule must be deleted" + ); + assert_eq!(r.router.counters().num_deletes, 1); + r.shutdown().await; +} + +/// A slow OS lookup/build cannot block a task heartbeat or timed release. +#[tokio::test] +async fn slow_discovery_does_not_block_heartbeat_or_shutdown() { + struct SlowLookup { + inner: Arc, + caps: Arc, + } + impl GatewayLookup for SlowLookup { + fn lookup(&self) -> Result, LookupError> { + self.caps.wait_if_paused(); + self.inner.lookup() + } + } + let r = rig().await; + let caps = Arc::new(PausedCleanupCaps { + inner: r.caps.clone(), + pause: std::sync::atomic::AtomicBool::new(false), + entered: tokio::sync::Notify::new(), + gate: (Mutex::new(false), std::sync::Condvar::new()), + }); + let opener = OpenCapsOnDrop(caps.clone()); + let (client, _rx) = Client::new(Config { + lookup: Arc::new(SlowLookup { + inner: r.lookup.clone(), + caps: caps.clone(), + }), + caps: caps.clone(), + clock: r.clock.clone(), + protos: ProtoSet::BOTH, + advertise_nonpublic_external: true, + lifetime: requested_lifetime(), + }); + client.set_local_port(PORT).unwrap(); + client.tick().await; + settle_checked(&client).await; + assert!(client.have_mapping()); + let frozen = FrozenTime::new(); + caps.pause.store(true, std::sync::atomic::Ordering::SeqCst); + let c = client.clone(); + let lookup = tokio::spawn(async move { c.check_network().await }); + caps.entered.notified().await; + // The heartbeat and release must finish while discovery is still blocked, rather than + // inferring overlap from a 10 ms sleep and comparing noisy wall-clock durations. + let heartbeat = tokio::spawn(async {}); + heartbeat.await.unwrap(); + let started = tokio::time::Instant::now(); + client.close(Duration::from_millis(80)).await; + assert_eq!(started.elapsed(), Duration::ZERO); + assert!( + !lookup.is_finished(), + "lookup must still be blocked during release" + ); + assert!( + r.router.mappings().is_empty(), + "release used its validated snapshot" + ); + drop(opener); + assert_eq!( + lookup.await.unwrap(), + NetCheck::Inactive, + "late discovery cannot reopen a closed client" + ); + drop(frozen); + r.shutdown().await; +} + +/// Force a completed I/O future through the stale commit branch, without aborting it. +#[tokio::test] +async fn completed_stale_attempt_cannot_publish_after_invalidation() { + for pcp in [false, true] { + let r = rig().await; + r.client.set_local_port(PORT).unwrap(); + r.client.check_network().await; + let rule = Rule::new( + if pcp { Proto::Pcp } else { Proto::Pmp }, + NonZeroU16::new(PORT).unwrap(), + ) + .unwrap(); + let ctx = { + let mut st = r.client.state(); + let n = st.net.as_ref().unwrap(); + let (cap, gw, ng) = (n.cap.clone(), n.gateway.clone(), n.net_gen); + let id = st + .owned + .add_pending(rule.clone(), ng, gw, r.clock.now()) + .unwrap(); + AttemptCtx { + generation: st.generation, + cap, + net_gen: ng, + rule, + rule_id: id, + existed: false, + evidence: st.owned.claim_map(id, r.clock.now(), false).unwrap(), + prev: None, + } + }; + let frozen = FrozenTime::new(); + let barrier = r.router.barrier(if pcp { + RequestKind::PcpMap + } else { + RequestKind::PmpMap + }); + let c = r.client.clone(); + // Deliberately outside the client's abort set: models I/O completing before abort wins. + let guard = SettlementGuard { + client: r.client.clone(), + event: FinalizerEvent::Map { + id: u64::MAX, + rule_id: ctx.rule_id, + }, + }; + let completion = tokio::spawn(async move { + let _guard = guard; + c.run_attempt(ctx).await + }); + barrier.reached().await; + r.client.policy().set_lan_sends_allowed(false); + r.client.set_local_port(PORT + 1).unwrap(); + barrier.release(); + assert!( + completion.await.unwrap(), + "stale completion requests orphan cleanup" + ); + assert!( + r.state().mapping.is_none(), + "stale result must not be published" + ); + assert_eq!( + r.client.owned_snapshot()[0].state(), + crate::owned::RuleState::Orphaned + ); + drop(frozen); + r.client.policy().set_lan_sends_allowed(true); + r.client.cleanup().await; + assert!(r.router.mappings().is_empty()); + r.shutdown().await; + } +} + +/// An unobserved grant may exceed the request: tombstones must not expire at the request alone. +#[tokio::test] +async fn lost_long_grants_still_count_after_requested_lifetime() { + let r = rig_with(Opts { + router: RouterOptions { + pmp: false, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + r.router.set_lifetime_override(Some(86_400)); + r.router.set_drop_deletes(true); + r.router.knobs_for(Some(RequestKind::PcpMap)); + r.router.commit_then_drop_next(1000); + for i in 0..4 { + r.client.set_local_port(PORT + i).unwrap(); + r.advance(secs(700)); + r.client.tick().await; + r.settle().await; + for _ in 0..4 { + r.client.cleanup().await; + } + } + assert_eq!(r.router.mappings().len(), 4); + r.advance(secs(7201)); + r.client.set_local_port(PORT + 4).unwrap(); + r.client.tick().await; + r.settle().await; + assert_eq!( + r.router.mappings().len(), + 4, + "unknown long grants still occupy their slots" + ); + assert_eq!(r.client.owned_rules(), 4); + r.shutdown().await; +} + +/// A lost adopted PCP create extends possible ownership and retains its nonce. +#[tokio::test] +async fn lost_adoption_preserves_nonce_until_close() { + let r = rig_with(Opts { + router: RouterOptions { + pmp: false, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + r.router.set_lifetime_override(Some(60)); + r.map().await.unwrap(); + let nonce = r.router.mappings()[0].nonce; + r.client.note_network_down(); + r.advance(secs(31)); + r.client.check_network().await; + r.client.probe().await.unwrap(); + r.router.set_lifetime_override(Some(7200)); + r.router.set_drop_deletes(true); + r.router.knobs_for(Some(RequestKind::PcpMap)); + r.router.commit_then_drop_next(1); + r.client.get_cached_mapping_or_start_creating_one(); + r.settle().await; + assert_eq!(r.router.mappings()[0].nonce, nonce); + r.advance(secs(31)); + assert_eq!( + r.router.mappings().len(), + 1, + "adopted grant outlives the original lease" + ); + r.router.set_drop_deletes(false); + r.client.close(secs(1)).await; + assert!( + r.router.mappings().is_empty(), + "adopted rule remains deletable after the old grant ends" + ); + r.router.shutdown().await; +} + +/// An adopted rule can predate a matched refusal: refusal does not prove the old rule is gone. +#[tokio::test] +async fn refused_adoption_keeps_existing_rule_owned() { + let r = rig_with(Opts { + router: RouterOptions { + pmp: false, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + r.map().await.unwrap(); + let nonce = r.router.mappings()[0].nonce; + r.client.note_network_down(); + r.advance(secs(3)); + r.router.set_result( + RequestKind::PcpMap, + Some(ResultOverride { + code: 7, + lifetime: 30, + }), + ); + r.client.tick().await; + r.settle().await; + assert_eq!( + r.client.owned_rules(), + 1, + "refused adoption retains pre-existing ownership" + ); + assert_eq!(r.router.mappings()[0].nonce, nonce); + r.router.set_result(RequestKind::PcpMap, None); + r.shutdown().await; +} + +/// A forward wall-clock correction must withdraw trust without pretending router rules expired. +#[tokio::test] +async fn wall_clock_jump_cannot_free_owned_rule_slots() { + let r = rig().await; + r.map().await.unwrap(); + r.client.check_clock(); + let expiry = r.client.owned_snapshot()[0].router_expiry(); + r.clock.suspend(secs(4 * 3600)); // also models a false positive wall-clock step + assert!(r.client.check_clock()); + assert!(!r.client.have_mapping()); + assert_eq!( + r.router.mappings().len(), + 1, + "router did not advance with the wall clock" + ); + assert_eq!( + r.client.owned_rules(), + 1, + "ownership is conservative across an unverified clock jump" + ); + assert_eq!(r.client.owned_snapshot()[0].router_expiry(), expiry); + r.shutdown().await; +} + +/// A lookup captured before synchronous invalidation cannot install its snapshot. +#[tokio::test] +async fn discovery_snapshot_is_rejected_after_port_generation_changes() { + struct PausedLookup { + entered: Arc, + gate: Arc<(Mutex, std::sync::Condvar)>, + } + impl GatewayLookup for PausedLookup { + fn lookup(&self) -> Result, LookupError> { + self.entered.notify_one(); + let (gate, cv) = &*self.gate; + let ready = lock(gate); + drop(cv.wait_while(ready, |open| !*open).unwrap()); + Ok(Some(gateway(GW1))) + } + } + let r = rig().await; + let entered = Arc::new(tokio::sync::Notify::new()); + let gate = Arc::new((Mutex::new(false), std::sync::Condvar::new())); + let (client, _rx) = Client::new(Config { + lookup: Arc::new(PausedLookup { + entered: entered.clone(), + gate: gate.clone(), + }), + caps: r.caps.clone(), + clock: r.clock.clone(), + protos: ProtoSet::BOTH, + advertise_nonpublic_external: true, + lifetime: requested_lifetime(), + }); + client.set_local_port(PORT).unwrap(); + let c = client.clone(); + let frozen = FrozenTime::new(); + let discovery = tokio::spawn(async move { c.check_network().await }); + tokio::time::timeout(secs(2), entered.notified()) + .await + .unwrap(); + client.set_local_port(PORT + 1).unwrap(); + *lock(&gate.0) = true; + gate.1.notify_one(); + assert_eq!(discovery.await.unwrap(), NetCheck::Inactive); + assert!(!client.has_network(), "stale gateway was never installed"); + drop(frozen); + client.close(secs(1)).await; + r.shutdown().await; +} + +/// Repeated validation failures without a network do not reset the warning budget. +#[tokio::test] +#[tracing_test::traced_test] +async fn offline_gateway_rejections_warn_once() { + let r = rig().await; + r.caps.set_present(GW1, false); + for _ in 0..10 { + assert_eq!(r.client.check_network().await, NetCheck::Down); + } + logs_assert(|lines: &[&str]| { + assert_eq!( + lines + .iter() + .filter(|line| line.contains("gateway rejected")) + .count(), + 1 + ); + Ok(()) + }); + r.shutdown().await; +} + +// Pause OS capability construction before cleanup can send its DELETE. +struct PausedCleanupCaps { + inner: Arc, + pause: std::sync::atomic::AtomicBool, + entered: tokio::sync::Notify, + gate: (Mutex, std::sync::Condvar), +} +impl PausedCleanupCaps { + fn wait_if_paused(&self) { + if self.pause.swap(false, std::sync::atomic::Ordering::SeqCst) { + self.entered.notify_one(); + let ready = lock(&self.gate.0); + drop(self.gate.1.wait_while(ready, |open| !*open).unwrap()); + } + } +} +impl CapFactory for PausedCleanupCaps { + fn build(&self, g: &Gateway, n: NetGen, p: &PolicyHandle) -> Result, CapError> { + self.wait_if_paused(); + self.inner.build(g, n, p) + } +} + +async fn alias_cleanup_case(initial_pcp: bool, port_roundtrip: bool, metadata_change: bool) { + let r = rig_with(Opts { + router: RouterOptions { + pmp: !initial_pcp, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + let caps = Arc::new(PausedCleanupCaps { + inner: r.caps.clone(), + pause: std::sync::atomic::AtomicBool::new(false), + entered: tokio::sync::Notify::new(), + gate: (Mutex::new(false), std::sync::Condvar::new()), + }); + let (client, _) = Client::new(Config { + lookup: r.lookup.clone(), + caps: caps.clone(), + clock: r.clock.clone(), + protos: ProtoSet::BOTH, + advertise_nonpublic_external: true, + lifetime: requested_lifetime(), + }); + client.set_local_port(PORT).unwrap(); + r.router.set_lifetime_override(Some(86_400)); + client.tick().await; + while client.attempt_in_flight() || client.reap() != 0 { + tokio::time::sleep(Duration::from_millis(5)).await; + } + assert!(client.have_mapping()); + if !port_roundtrip { + r.advance(secs(7201)); + assert!(client.expire_if_due()); + } + assert_eq!( + r.router.mappings().len(), + 1, + "router grant outlives local trust" + ); + r.router.set_pmp(true); + client.probe().await.unwrap(); + let frozen = FrozenTime::new(); + let opener = OpenCapsOnDrop(caps.clone()); + caps.pause.store(true, std::sync::atomic::Ordering::SeqCst); + if port_roundtrip { + client.set_local_port(PORT + 1).unwrap(); + } + let c = client.clone(); + let cleanup = tokio::spawn(async move { c.cleanup().await }); + tokio::time::timeout(secs(2), caps.entered.notified()) + .await + .unwrap(); + if port_roundtrip { + client.set_local_port(PORT).unwrap(); + r.advance(secs(3)); + } + if metadata_change { + let mut gw = gateway(GW1); + gw.prefix_len = 23; + r.lookup.set(gw); + assert_eq!(client.check_network().await, NetCheck::Changed); + } + // A driver tick must not create an alias while cleanup owns the wire identity. + client.tick().await; + while client.attempt_in_flight() { + tokio::task::yield_now().await; + } + let during_delete = client.subscribe().borrow().mapping; + drop(opener); + cleanup.await.unwrap(); + drop(frozen); + while client.reap() != 0 { + tokio::task::yield_now().await; + } + // Retirement permits the next create, including a protocol switch. Its following cleanup + // must leave the advertised rule intact. + r.advance(secs(3)); + client.tick().await; + while client.attempt_in_flight() || client.reap() != 0 { + tokio::task::yield_now().await; + } + client.cleanup().await; + let advertised = client.subscribe().borrow().mapping.unwrap(); + let table = r.router.mappings(); + client.close(secs(1)).await; + r.shutdown().await; + assert!( + during_delete.is_none(), + "MAP aliased a reserved delete identity" + ); + assert_eq!( + advertised.proto, + Proto::Pmp, + "retired PCP identity can switch" + ); + assert_eq!(table.len(), 1, "next cleanup must preserve advertisement"); + assert_eq!(table[0].internal, PORT); + assert_eq!(table[0].external, advertised.external.port()); +} +#[tokio::test] +async fn pcp_cleanup_cannot_delete_pmp_replacement() { + alias_cleanup_case(true, false, false).await; +} +#[tokio::test] +async fn pmp_cleanup_cannot_delete_pmp_replacement() { + alias_cleanup_case(false, false, false).await; +} + +#[tokio::test] +async fn port_away_and_back_during_cleanup() { + alias_cleanup_case(true, true, false).await; + alias_cleanup_case(false, true, false).await; +} + +#[tokio::test] +async fn pcp_outstanding_rules_bounded() { + for refused in [false, true] { + let r = rig_with(Opts { + router: RouterOptions { + pmp: false, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + r.client.set_local_port(PORT).unwrap(); + r.client.check_network().await; + r.client.probe().await.unwrap(); + for i in 0..4 { + r.router.set_result(RequestKind::PcpMap, None); + r.client.get_cached_mapping_or_start_creating_one(); + r.settle().await; + assert!(r.client.have_mapping(), "slot {i}"); + r.router.set_drop_deletes(!refused); + if refused { + r.router.set_result( + RequestKind::PcpMap, + Some(ResultOverride { + code: 3, + lifetime: 0, + }), + ); + } + r.client.set_local_port(PORT + i + 1).unwrap(); + r.settle().await; + for _ in 0..4 { + r.client.cleanup().await; + } + r.advance(secs(3)); + } + assert_eq!(r.router.mappings().len(), 4); + r.router.set_result(RequestKind::PcpMap, None); + r.client.get_cached_mapping_or_start_creating_one(); + r.settle().await; + assert_eq!(r.router.mappings().len(), 4, "fifth MAP must be refused"); + assert_eq!(r.client.owned_rules(), 4); + assert!(!r.client.have_mapping()); + r.advance(secs(7201)); + r.client.tick().await; + r.settle().await; + assert_eq!(r.router.mappings().len(), 1, "expiry frees admission"); + r.router.set_drop_deletes(false); + r.shutdown().await; + } +} + +#[tokio::test] +async fn failed_old_delete_cannot_delete_protocol_replacement() { + let r = rig_with(Opts { + router: RouterOptions { + pmp: false, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + r.router.set_lifetime_override(Some(86_400)); + r.map().await.unwrap(); + r.advance(secs(7201)); + assert!(r.client.expire_if_due()); + r.router.set_pmp(true); + r.router.set_drop_deletes(true); + r.client.tick().await; + r.settle().await; + let interim = r.state().mapping; + assert_eq!(r.router.mappings().len(), 1); + r.router.set_drop_deletes(false); + r.advance(secs(700)); + r.client.tick().await; + r.settle().await; + r.client.cleanup().await; + let table = r.router.mappings(); + let advertised = r.state().mapping.unwrap(); + r.shutdown().await; + assert!( + interim.is_none_or(|m| m.proto == Proto::Pcp), + "orphan must retain its protocol" + ); + assert_eq!(table.len(), 1, "next cleanup must preserve advertisement"); + assert_eq!(table[0].internal, PORT); + assert_eq!(table[0].external, advertised.external.port()); +} +#[tokio::test] +async fn refused_old_delete_cannot_delete_protocol_replacement() { + let r = rig_with(Opts { + router: RouterOptions { + pmp: false, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + r.router.set_lifetime_override(Some(86_400)); + r.map().await.unwrap(); + r.advance(secs(7201)); + assert!(r.client.expire_if_due()); + r.router.set_pmp(true); + r.router.set_result( + RequestKind::PcpMap, + Some(ResultOverride { + code: 7, + lifetime: 30, + }), + ); + r.client.tick().await; + r.settle().await; + let interim = r.state().mapping; + assert_eq!(r.router.mappings().len(), 1); + r.router.set_result(RequestKind::PcpMap, None); + r.advance(secs(700)); + r.client.tick().await; + r.settle().await; + r.client.cleanup().await; + let table = r.router.mappings(); + let advertised = r.state().mapping.unwrap(); + r.shutdown().await; + assert!( + interim.is_none_or(|m| m.proto == Proto::Pcp), + "orphan must retain its protocol" + ); + assert_eq!(table.len(), 1, "next cleanup must preserve advertisement"); + assert_eq!(table[0].internal, PORT); + assert_eq!(table[0].external, advertised.external.port()); +} + +#[tokio::test] +async fn pcp_variant_network_change_releases_only_to_a_validating_old_network() { + for old_validates in [true, false] { + let r = rig_with(Opts { + router: RouterOptions { + pmp: false, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + let router2 = FakeRouter::start(RouterOptions { + pmp: false, + ..RouterOptions::default() + }) + .await + .unwrap(); + r.caps.add(GW2, router2.endpoint()); + r.map().await.unwrap(); + assert_eq!(r.router.mappings().len(), 1); + r.caps.set_present(GW1, old_validates); + r.lookup.set(gateway(GW2)); + assert_eq!(r.client.check_network().await, NetCheck::Changed); + assert!(r.state().mapping.is_none()); + r.settle().await; + r.advance(secs(2)); + r.client.tick().await; + r.settle().await; + assert_eq!( + r.router.counters().num_deletes, + u64::from(old_validates), + "old_validates={old_validates}" + ); + assert_eq!(r.router.mappings().len(), usize::from(!old_validates)); + assert_eq!( + router2.counters().num_deletes, + 0, + "nothing for the old rule" + ); + assert_eq!( + router2.mappings().len(), + 1, + "new mapping via the new router" + ); + assert!(r.state().mapping.is_some()); + assert_eq!(r.client.owned_rules(), if old_validates { 1 } else { 2 }); + router2.shutdown().await; + r.shutdown().await; + } +} + +#[tokio::test] +async fn pcp_variant_down_invalidates_without_release() { + for how in ["no-route", "error", "down"] { + let r = rig_with(Opts { + router: RouterOptions { + pmp: false, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + r.map().await.unwrap(); + match how { + "no-route" => { + r.lookup.set_no_route(); + assert_eq!(r.client.check_network().await, NetCheck::Down); + } + "error" => { + r.lookup.set_error(); + assert_eq!(r.client.check_network().await, NetCheck::Down); + } + _ => r.client.note_network_down(), + } + r.settle().await; + r.client.cleanup().await; + assert!(r.state().mapping.is_none(), "{how}"); + assert_eq!(r.router.counters().num_deletes, 0, "{how}: no release"); + // Kept (never deleted) only so a create on the same network can adopt it. + assert!( + r.client.owned_snapshot().iter().all(|e| !e.releasable()), + "{how}" + ); + assert!(!r.client.has_network()); + r.client.close(secs(1)).await; + assert_eq!( + r.router.counters().num_deletes, + 0, + "{how}: not even on close" + ); + r.router.shutdown().await; + } +} + +#[tokio::test] +async fn pcp_variant_rebind_keeps_or_replaces_the_mapping() { + let r = rig_with(Opts { + router: RouterOptions { + pmp: false, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + let ext = r.map().await.unwrap(); + let mut rx = r.client.subscribe(); + rx.borrow_and_update(); + let before = r.router.requests().len(); + r.client.note_rebind(Some(PORT)); + assert!( + r.state().mapping.is_none(), + "withdrawn before OS validation" + ); + r.client.check_network().await; + assert!(rx.has_changed().unwrap(), "re-published for the magicsock"); + assert_eq!(r.state().mapping.unwrap().external, ext); + r.settle().await; + assert_eq!(r.router.requests().len(), before, "no I/O"); + + r.client.note_rebind(Some(PORT + 2)); + assert!(r.state().mapping.is_none()); + r.settle().await; + r.advance(secs(2)); + let before = r.creates_and_renewals().len(); + r.client.get_cached_mapping_or_start_creating_one(); + r.settle().await; + assert_eq!( + r.creates_and_renewals().len(), + before, + "changed-port rebind requires fresh network validation" + ); + r.client.tick().await; + r.settle().await; + let table: Vec = r.router.mappings().iter().map(|m| m.internal).collect(); + assert_eq!(table, [PORT + 2]); + + r.client.note_rebind(None); + assert!(r.state().mapping.is_none()); + r.settle().await; + assert!(r.router.mappings().is_empty(), "released: the port is gone"); + r.shutdown().await; +} + +#[tokio::test] +async fn pcp_variant_close_releases_within_budget() { + let r = rig_with(Opts { + router: RouterOptions { + pmp: false, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + r.map().await.unwrap(); + r.client.close(secs(1)).await; + assert!(r.router.mappings().is_empty()); + let deletes = r.router.counters().num_deletes; + assert_eq!(deletes, 1); + r.client.close(secs(1)).await; + r.client.release_blocking_best_effort(); + assert_eq!(r.router.counters().num_deletes, deletes, "idempotent"); + assert!(r.client.tick().await.is_none()); + assert!( + r.client + .get_cached_mapping_or_start_creating_one() + .is_none() + ); + assert_eq!(r.router.counters().num_deletes, deletes); + r.router.shutdown().await; + + // Router gone: close still ends within its budget. + let r = rig_with(Opts { + router: RouterOptions { + pmp: false, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + r.map().await.unwrap(); + r.router.drop_next(1000); + let t = std::time::Instant::now(); + r.client.close(Duration::from_millis(300)).await; + assert!( + t.elapsed() < Duration::from_millis(600), + "{:?}", + t.elapsed() + ); + r.router.shutdown().await; + + // Close while an attempt waits on the router: no new work starts after it began. + let r = rig_with(Opts { + router: RouterOptions { + pmp: false, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + r.client.set_local_port(PORT).unwrap(); + r.client.check_network().await; + r.client.probe().await.unwrap(); + let barrier = r.router.barrier(RequestKind::PcpMap); + r.client.get_cached_mapping_or_start_creating_one(); + barrier.reached().await; + let c = r.client.clone(); + let close = tokio::spawn(async move { c.close(secs(2)).await }); + wait_for_close(&r.client).await; + let maps = r.creates_and_renewals().len(); + assert!( + r.client + .get_cached_mapping_or_start_creating_one() + .is_none() + ); + barrier.release(); + close.await.unwrap(); + assert_eq!( + r.creates_and_renewals().len(), + maps, + "nothing started after close began" + ); + assert!( + r.router.mappings().is_empty(), + "the in-flight rule was deleted" + ); + r.router.shutdown().await; +} + +/// Invalidation clears the single-flight handle before a completed send's future is dropped. +/// Keep the ledger reservation until that future actually finishes, not until invalidation. +#[tokio::test] +async fn map_reservation_survives_invalidation() { + for pcp in [false, true] { + let r = rig_with(Opts { + router: RouterOptions { + pmp: !pcp, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + r.client.set_local_port(PORT).unwrap(); + r.client.check_network().await; + r.client.probe().await.unwrap(); + let frozen = FrozenTime::new(); + let barrier = r.router.barrier(if pcp { + RequestKind::PcpMap + } else { + RequestKind::PmpMap + }); + r.client.get_cached_mapping_or_start_creating_one(); + barrier.reached().await; + // Model an operation whose completion has won the abort race. It remains in the JoinSet. + r.client.state().inflight.as_mut().unwrap().abort.take(); + r.client.set_local_port(PORT + 1).unwrap(); + r.client.set_local_port(PORT).unwrap(); + r.advance(secs(3)); + // PCP-only discovery left PMP uncached. Answer the tick's probe immediately: a silent + // probe would need time to advance, and race the MAP's equally long exchange deadline. + r.router.set_pmp(true); + r.client.tick().await; + assert!(r.client.owned_snapshot()[0].mapping(), "MAP is still live"); + assert_owned_invariant(&r.client); + let during = r.state().mapping; + let sends = r.router.requests(); + barrier.release(); + while r.client.reap() != 0 { + tokio::task::yield_now().await; + } + drop(frozen); + r.settle().await; + r.advance(secs(3)); + r.client.tick().await; + r.settle().await; + r.client.cleanup().await; + let advertised = r.state().mapping.unwrap(); + let table = r.router.mappings(); + r.shutdown().await; + assert!(during.is_none(), "pending MAP identity was aliased"); + assert_eq!( + sends + .iter() + .filter( + |q| matches!(q.kind, RequestKind::PmpMap | RequestKind::PcpMap) + && q.lifetime == Some(0) + ) + .count(), + 0, + "delete raced pending MAP" + ); + assert_eq!( + sends + .iter() + .filter(|q| q.lifetime.is_some_and(|t| t > 0)) + .count(), + 1, + "second MAP raced pending MAP" + ); + assert_eq!(table.len(), 1); + assert_eq!(table[0].internal, PORT); + assert_eq!(table[0].external, advertised.external.port()); + } +} + +#[tokio::test] +async fn timed_out_map_returns_reservation_before_late_reply() { + for pcp in [false, true] { + let r = rig_with(Opts { + protos: ProtoSet { pmp: !pcp, pcp }, + ..Opts::default() + }) + .await; + r.client.set_local_port(PORT).unwrap(); + r.client.check_network().await; + r.client.probe().await.unwrap(); + let frozen = FrozenTime::new(); + let reply = r.router.barrier(if pcp { + RequestKind::PcpMap + } else { + RequestKind::PmpMap + }); + r.client.get_cached_mapping_or_start_creating_one(); + reply.reached().await; + r.client.state().inflight.as_mut().unwrap().abort.take(); + r.client.set_local_port(PORT + 1).unwrap(); + r.client.set_local_port(PORT).unwrap(); + assert!(r.client.owned_snapshot()[0].mapping()); + assert_owned_invariant(&r.client); + assert_eq!(delete_requests(&r.router), 0); + + // The server's reply is still held, but the client operation has timed out. Cleanup + // may now delete; a reply barrier alone cannot establish a live MAP reservation. + tokio::time::advance(crate::cap::EXCHANGE_TIMEOUT + Duration::from_millis(1)).await; + while r.client.reap() != 0 { + assert_owned_invariant(&r.client); + tokio::task::yield_now().await; + } + assert_eq!(delete_requests(&r.router), 1); + assert!(r.client.owned_snapshot().is_empty()); + assert!(r.router.mappings().is_empty()); + assert_router_inventory(&r.client, GW1, &r.router); + reply.release(); + drop(frozen); + r.shutdown().await; + } +} + +#[tokio::test] +async fn unknown_slots_recover_when_router_recovers() { + let r = rig().await; + r.router.set_drop_deletes(true); + r.router.knobs_for(Some(RequestKind::PmpMap)); + r.router.commit_then_drop_next(1000); + for i in 0..4 { + r.client.set_local_port(PORT + i).unwrap(); + r.advance(secs(700)); + r.client.tick().await; + r.settle().await; + for _ in 0..4 { + r.client.cleanup().await; + } + } + assert_eq!(r.client.owned_rules(), 4); + r.advance(secs(7201)); + assert!( + r.router.mappings().is_empty(), + "all actual router grants expired" + ); + r.router.set_drop_deletes(false); + r.router.commit_then_drop_next(0); + r.client.set_local_port(PORT + 4).unwrap(); + r.client.tick().await; + r.settle().await; + let recovered = r.client.have_mapping(); + let tombstones = r.client.owned_rules(); + r.shutdown().await; + assert!( + recovered, + "healthy empty router remains blocked by {tombstones} tombstones" + ); +} + +fn delete_requests(router: &FakeRouter) -> usize { + router + .requests() + .iter() + .filter(|q| { + matches!(q.kind, RequestKind::PmpMap | RequestKind::PcpMap) && q.lifetime == Some(0) + }) + .count() +} + +// Lose a committed create and all its initial deletes, leaving an unknown grant. +async fn exhaust_unknown_slots(r: &Rig, pcp: bool) { + r.router.set_drop_deletes(true); + r.router.knobs_for(Some(if pcp { + RequestKind::PcpMap + } else { + RequestKind::PmpMap + })); + r.router.commit_then_drop_next(1); + r.client.set_local_port(PORT).unwrap(); + r.client.tick().await; + r.settle().await; + for _ in 0..4 { + r.client.cleanup().await; + } + assert!(!r.client.have_mapping()); + assert_eq!(r.client.owned_rules(), 1); + assert_eq!(r.router.mappings().len(), 1); + assert_eq!(delete_requests(&r.router), 3); +} + +#[tokio::test] +async fn lost_cleanup_retries_are_infrequent_and_keep_slots() { + for pcp in [false, true] { + let r = rig_with(Opts { + router: RouterOptions { + pmp: !pcp, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + exhaust_unknown_slots(&r, pcp).await; + let before = delete_requests(&r.router); + for _ in 0..599 { + r.advance(secs(1)); + r.client.cleanup().await; + } + assert_eq!(delete_requests(&r.router), before, "no hot retry loop"); + r.advance(secs(1)); + r.client.cleanup().await; + let after = delete_requests(&r.router); + assert_eq!( + after, + before + 1, + "exhaustion must allow a recovery attempt" + ); + for _ in 0..599 { + r.advance(secs(1)); + r.client.cleanup().await; + } + assert_eq!( + delete_requests(&r.router), + after, + "one attempt per cooldown" + ); + assert_eq!(r.client.owned_rules(), 1, "loss is not retirement evidence"); + assert_eq!(r.router.mappings().len(), 1); + r.advance(secs(1)); + // A refused delete is also insufficient evidence; subsequent success may retire it. + r.router.set_drop_deletes(false); + r.router.set_result( + if pcp { + RequestKind::PcpMap + } else { + RequestKind::PmpMap + }, + Some(ResultOverride { + code: 2, + lifetime: 0, + }), + ); + r.client.cleanup().await; + assert_eq!(r.client.owned_rules(), 1); + assert_eq!(r.router.mappings().len(), 1); + r.router.set_result( + if pcp { + RequestKind::PcpMap + } else { + RequestKind::PmpMap + }, + None, + ); + r.caps.set_present(GW1, false); + r.advance(secs(600)); + assert_eq!( + r.client.cleanup().await, + 0, + "failed fresh validation sends nothing" + ); + assert_eq!(r.client.owned_rules(), 1); + r.caps.set_present(GW1, true); + r.advance(secs(599)); + assert_eq!( + r.client.cleanup().await, + 0, + "failed validation also has a cooldown" + ); + r.advance(secs(1)); + assert_eq!( + r.client.cleanup().await, + 1, + "validation failure must remain recoverable" + ); + assert_eq!(r.client.owned_rules(), 0); + assert!(r.router.mappings().is_empty()); + r.shutdown().await; + } +} + +#[tokio::test] +async fn network_down_fences_a_late_recovery_validation() { + let r = rig().await; + let caps = Arc::new(PausedCleanupCaps { + inner: r.caps.clone(), + pause: std::sync::atomic::AtomicBool::new(false), + entered: tokio::sync::Notify::new(), + gate: (Mutex::new(false), std::sync::Condvar::new()), + }); + let (client, _) = Client::new(Config { + lookup: r.lookup.clone(), + caps: caps.clone(), + clock: r.clock.clone(), + protos: ProtoSet::BOTH, + advertise_nonpublic_external: true, + lifetime: requested_lifetime(), + }); + r.router.set_drop_deletes(true); + r.router.knobs_for(Some(RequestKind::PmpMap)); + r.router.commit_then_drop_next(1); + client.set_local_port(PORT).unwrap(); + client.tick().await; + while client.attempt_in_flight() || client.reap() != 0 { + tokio::time::sleep(Duration::from_millis(5)).await; + } + for _ in 0..4 { + client.cleanup().await; + } + assert_eq!(delete_requests(&r.router), 3); + r.advance(secs(600)); + r.router.set_drop_deletes(false); + let frozen = FrozenTime::new(); + let opener = OpenCapsOnDrop(caps.clone()); + caps.pause.store(true, std::sync::atomic::Ordering::SeqCst); + let c = client.clone(); + let cleanup = tokio::spawn(async move { c.cleanup().await }); + let entered = tokio::time::timeout(secs(2), caps.entered.notified()).await; + client.note_network_down(); + drop(opener); + assert_eq!(cleanup.await.unwrap(), 0); + drop(frozen); + entered.expect("exhausted entry must reach fresh validation before retry"); + for _ in 0..5 { + r.advance(secs(7200)); + client.cleanup().await; + } + assert!(client.owned_snapshot().iter().all(|e| !e.releasable())); + assert_eq!( + client.owned_rules(), + 1, + "network loss is not retirement evidence" + ); + assert!(!client.have_mapping()); + client.close(secs(1)).await; + assert_eq!( + delete_requests(&r.router), + 3, + "network-down entries stay unsent, even after validation finishes or on close" + ); + r.shutdown().await; +} + +/// A cleanup capability must join synchronous invalidation, just like the active capability. +#[tokio::test] +async fn network_down_invalidates_validated_cleanup_cap() { + let r = rig().await; + exhaust_unknown_slots(&r, false).await; + r.advance(secs(600)); + r.router.set_drop_deletes(false); + let frozen = FrozenTime::new(); + let barrier = r.router.barrier(RequestKind::PmpMap); + let c = r.client.clone(); + let cleanup = tokio::spawn(async move { c.cleanup().await }); + tokio::time::timeout(secs(2), barrier.reached()) + .await + .unwrap(); + let cap = r.caps.built().last().unwrap().clone(); + assert!(cap.is_valid()); + r.client.note_network_down(); + let still_valid = cap.is_valid(); + barrier.release(); + assert_eq!( + cleanup.await.unwrap(), + 1, + "a matched acknowledgement may still retire the already sent delete" + ); + drop(frozen); + r.client.cleanup().await; + assert!(r.router.mappings().is_empty()); + assert!(!r.client.have_mapping()); + r.shutdown().await; + assert!( + !still_valid, + "network-down must invalidate an already validated cleanup cap before its next send" + ); +} + +#[tokio::test] +async fn prefix_change_does_not_split_a_wire_identity() { + alias_cleanup_case(true, false, true).await; + alias_cleanup_case(false, false, true).await; +} + +#[tokio::test] +async fn cancel_initial_pcp_by_same_port_rebind() { + let r = rig_with(Opts { + router: RouterOptions { + pmp: false, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + r.client.set_local_port(PORT).unwrap(); + r.client.check_network().await; + r.client.probe().await.unwrap(); + let frozen = FrozenTime::new(); + let barrier = r.router.barrier(RequestKind::PcpMap); + r.client.get_cached_mapping_or_start_creating_one(); + barrier.reached().await; + let original = r.client.owned_snapshot()[0].clone(); + r.client.note_rebind(Some(PORT)); + barrier.release(); + while r.client.reap() != 0 { + tokio::task::yield_now().await; + } + drop(frozen); + r.settle().await; + let orphan = &r.client.owned_snapshot()[0]; + assert_eq!(orphan.state(), crate::owned::RuleState::Orphaned); + assert_eq!(orphan.rule, original.rule, "retain the original PCP nonce"); + assert_eq!(orphan.router_expiry(), original.router_expiry()); + r.client.check_network().await; + for _ in 0..3 { + r.advance(secs(700)); + r.client.tick().await; + r.settle().await; + } + let recovered = r.client.have_mapping(); + let entries = r.client.owned_snapshot(); + r.shutdown().await; + assert!( + recovered, + "cancelled PCP identity never recovered: {entries:?}" + ); +} + +// Drop during a live MAP claim, extended to renewals and an operation runtime torn down +// outside any runtime context. Request barriers replace timing sleeps. +async fn drop_claim_case(pcp: bool, renewal: bool, no_runtime: bool) { + let r = rig_with(Opts { + router: RouterOptions { + pmp: !pcp, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + if renewal { + r.map().await.unwrap(); + r.advance(secs(3601)); + } else { + r.client.set_local_port(PORT).unwrap(); + r.client.check_network().await; + r.client.probe().await.unwrap(); + } + let kind = if pcp { + RequestKind::PcpMap + } else { + RequestKind::PmpMap + }; + let map_reply = r.router.barrier(kind); + let delete = r.router.barrier(kind); + let c = r.client.clone(); + if no_runtime { + tokio::task::spawn_blocking(move || { + std::thread::spawn(move || { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + rt.block_on(async { + let _frozen = FrozenTime::new(); + c.get_cached_mapping_or_start_creating_one(); + map_reply.reached().await; + assert!(c.owned_snapshot()[0].mapping()); + }); + assert!(tokio::runtime::Handle::try_current().is_err()); + c.release_blocking_best_effort(); + drop(rt); // aborts/drains the operation finalizer without an entered runtime + drop(map_reply); + }) + .join() + .unwrap() + }) + .await + .unwrap(); + } else { + let _frozen = FrozenTime::new(); + c.get_cached_mapping_or_start_creating_one(); + map_reply.reached().await; + assert!(c.owned_snapshot()[0].mapping()); + c.release_blocking_best_effort(); + map_reply.release(); + } + r.settle().await; + let released = tokio::time::timeout(secs(1), delete.reached()) + .await + .is_ok(); + delete.release(); + let table = r.router.mappings(); + let sends = delete_requests(&r.router); + let creates = r.creates_and_renewals().len(); + r.client.release_blocking_best_effort(); + r.client.cleanup().await; + assert!(!r.client.have_mapping()); + r.shutdown().await; + assert!( + released, + "finalizer skipped delete: pcp={pcp}, renewal={renewal}, no_runtime={no_runtime}" + ); + assert!(table.is_empty()); + assert_eq!(sends, 1); + assert_eq!( + creates, + if renewal { 2 } else { 1 }, + "no post-release create" + ); +} + +#[tokio::test] +async fn drop_during_live_map_claim() { + for pcp in [false, true] { + for renewal in [false, true] { + drop_claim_case(pcp, renewal, false).await; + } + } +} + +#[tokio::test] +async fn drop_claim_during_no_runtime_teardown() { + for pcp in [false, true] { + for renewal in [false, true] { + drop_claim_case(pcp, renewal, true).await; + } + } +} + +#[tokio::test] +async fn close_during_external_cleanup_validation() { + let mut observed = Vec::new(); + for pcp in [false, true] { + let r = rig_with(Opts { + router: RouterOptions { + pmp: !pcp, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + let caps = Arc::new(PausedCleanupCaps { + inner: r.caps.clone(), + pause: std::sync::atomic::AtomicBool::new(false), + entered: tokio::sync::Notify::new(), + gate: (Mutex::new(false), std::sync::Condvar::new()), + }); + let (client, _) = Client::new(Config { + lookup: r.lookup.clone(), + caps: caps.clone(), + clock: r.clock.clone(), + protos: ProtoSet::BOTH, + advertise_nonpublic_external: true, + lifetime: requested_lifetime(), + }); + r.router.set_lifetime_override(Some(86_400)); + client.set_local_port(PORT).unwrap(); + client.tick().await; + while client.attempt_in_flight() || client.reap() != 0 { + tokio::task::yield_now().await; + } + r.advance(secs(7201)); + client.expire_if_due(); + let frozen = FrozenTime::new(); + let opener = OpenCapsOnDrop(caps.clone()); + caps.pause.store(true, std::sync::atomic::Ordering::SeqCst); + let c = client.clone(); + let cleanup = tokio::spawn(async move { c.cleanup().await }); + tokio::time::timeout(secs(1), caps.entered.notified()) + .await + .unwrap(); + let c = client.clone(); + let close = tokio::spawn(async move { c.close(secs(1)).await }); + wait_for_close(&client).await; + drop(opener); + cleanup.await.unwrap(); + close.await.unwrap(); + drop(frozen); + let deletes = delete_requests(&r.router); + let entries = client.owned_snapshot(); + r.shutdown().await; + observed.push((pcp, deletes, entries)); + } + assert!( + observed.iter().all(|(_, deletes, _)| *deletes == 1), + "close skipped externally claimed rule: {observed:?}" + ); +} + +#[tokio::test] +async fn cancel_cleanup_reservations_and_recovery_spacing() { + for pcp in [false, true] { + for exhausted in [false, true] { + let r = rig_with(Opts { + router: RouterOptions { + pmp: !pcp, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + r.map().await.unwrap(); + // Make a real owned orphan without scheduling a background cleanup. + let id = r.client.owned_snapshot()[0].id; + { + let mut st = r.client.state(); + st.mapping = None; + st.owned.orphan(id); + } + if exhausted { + r.router.set_drop_deletes(true); + for _ in 0..3 { + r.client.cleanup().await; + } + r.advance(secs(600)); + } + let frozen = FrozenTime::new(); + let before = delete_requests(&r.router); + let barrier = r.router.barrier(if pcp { + RequestKind::PcpMap + } else { + RequestKind::PmpMap + }); + let c = r.client.clone(); + let job = tokio::spawn(async move { c.cleanup().await }); + barrier.reached().await; + job.abort(); + drop(job.await); + barrier.release(); + drop(frozen); + assert!( + r.client + .owned_snapshot() + .iter() + .all(|e| !e.deleting() && !e.mapping()) + ); + assert_eq!(delete_requests(&r.router), before + 1); + r.router.set_drop_deletes(false); + if exhausted { + let sent = delete_requests(&r.router); + for _ in 0..599 { + r.advance(secs(1)); + r.client.cleanup().await; + } + assert_eq!(delete_requests(&r.router), sent); + r.advance(secs(1)); + } + assert_eq!(r.client.cleanup().await, 1); + assert_eq!(r.client.owned_rules(), 0); + r.shutdown().await; + } + } +} + +#[tokio::test] +async fn reset_on_other_gateway_must_not_retire_old_rule() { + let mut observed = Vec::new(); + for pcp in [false, true] { + let r = rig_with(Opts { + router: RouterOptions { + pmp: !pcp, + ..RouterOptions::default() + }, + protos: ProtoSet { pmp: !pcp, pcp }, + ..Opts::default() + }) + .await; + exhaust_unknown_slots(&r, pcp).await; + let original = r.client.owned_snapshot()[0].clone(); + let old = original.id; + r.caps.set_present(GW1, false); + let router2 = FakeRouter::start(RouterOptions { + pmp: !pcp, + ..RouterOptions::default() + }) + .await + .unwrap(); + r.caps.add(GW2, router2.endpoint()); + r.lookup.set(gateway(GW2)); + r.client.check_network().await; + r.settle().await; + r.advance(secs(3)); + r.client.tick().await; + r.settle().await; + assert!(r.client.have_mapping()); + assert_eq!(r.client.owned_rules(), 2); + assert_router_inventories(&r.client, &[(GW1, &r.router), (GW2, &router2)]); + router2.reboot(); + router2.set_epoch(10); + r.advance(secs(601)); + r.client.probe().await.unwrap(); + r.settle().await; + assert_router_inventories(&r.client, &[(GW1, &r.router), (GW2, &router2)]); + let retained = r.client.owned_snapshot().iter().any(|e| e.id == old); + if let Some(retained) = r.client.owned_snapshot().iter().find(|e| e.id == old) { + assert_eq!( + retained.rule, original.rule, + "other-router reset must retain the PCP nonce" + ); + assert_eq!(retained.router_expiry(), original.router_expiry()); + } + assert!( + r.client + .owned_snapshot() + .iter() + .all(|e| e.gateway.gw == GW1), + "B's lost rules should retire" + ); + let old_still_exists = r.router.mappings().len(); + observed.push((pcp, retained, old_still_exists)); + router2.shutdown().await; + r.shutdown().await; + } + assert!( + observed + .iter() + .all(|(_, retained, actual)| *retained && *actual == 1), + "other-router reset freed unrelated ownership: {observed:?}" + ); +} + +/// Conservation of possible router ownership, not just endpoint trust: an active lease is +/// owned by its held mapping; a MAP or +/// delete claim is owned by its future; all other entries must be cleanup orphans. Fenced and +/// terminal unacknowledged orphans remain counted, never classified as acknowledged gone. +fn assert_owned_invariant(client: &Client) { + use crate::owned::{MAX_OWNED, RuleState}; + let st = client.state(); + st.owned.model.check(st.owned.entries()); + assert!(st.owned.len() <= MAX_OWNED); + for (i, e) in st.owned.entries().iter().enumerate() { + assert!(!(e.mapping() && e.deleting()), "two live operations: {e:?}"); + for other in &st.owned.entries()[..i] { + assert!( + !(e.gateway.gw == other.gateway.gw + && e.gateway.self_ip == other.gateway.self_ip + && e.gateway.if_index == other.gateway.if_index + && e.rule.internal() == other.rule.internal()), + "aliased ownership" + ); + } + if e.deleting() { + assert_eq!(e.state(), RuleState::Orphaned); + } else if !e.mapping() { + match e.state() { + RuleState::Pending => panic!("unclaimed pending rule: {e:?}"), + RuleState::Active => assert!( + st.mapping.as_ref().is_some_and(|h| h.rule_id == e.id), + "active rule has no lease owner" + ), + RuleState::Orphaned => { + if st.phase == Phase::Running && e.releasable() && st.disabled.is_none() { + assert!( + st.owned.next_cleanup(client.now()).is_some(), + "orphan has no cleanup schedule" + ); + } + } + } + } + } + if let Some(h) = &st.mapping { + let e = st + .owned + .get(h.rule_id) + .expect("held mapping lost ownership"); + assert_eq!(e.state(), RuleState::Active); + assert_eq!(e.rule, h.m.rule()); + } + if st.phase == Phase::Closed { + assert!(st.mapping.is_none()); + } +} + +/// Check conservation against all participating routers. Only use real fake-router reboots +/// here: older unit tests also inject an epoch rollback without actually clearing its table. +fn assert_router_inventory(client: &Client, gw: Ipv4Addr, router: &FakeRouter) { + assert_router_inventories(client, &[(gw, router)]); +} + +#[tokio::test] +async fn cancel_close_preserves_final_release() { + for pcp in [false, true] { + let r = rig_with(Opts { + router: RouterOptions { + pmp: !pcp, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + r.map().await.unwrap(); + r.router.set_drop_deletes(true); + let kind = if pcp { + RequestKind::PcpMap + } else { + RequestKind::PmpMap + }; + let frozen = FrozenTime::new(); + let first = r.router.barrier(kind); + let c = r.client.clone(); + let close = tokio::spawn(async move { c.close(secs(10)).await }); + first.reached().await; + assert_owned_invariant(&r.client); + let final_delete = r.router.barrier(kind); + r.router.set_drop_deletes(false); + close.abort(); + assert!(close.await.unwrap_err().is_cancelled()); + first.release(); + let phase = r.client.state().phase; + let released = tokio::time::timeout(secs(1), final_delete.reached()) + .await + .is_ok(); + final_delete.release(); + drop(frozen); + assert_router_inventory(&r.client, GW1, &r.router); + let table = r.router.mappings(); + r.client.release_blocking_best_effort(); // clean up if the final release was lost + r.shutdown().await; + assert_eq!( + phase, + Phase::Closed, + "cancelled close left a half-closed owner" + ); + assert!(released, "cancelled close discarded release duty"); + assert!(table.is_empty()); + } +} + +#[tokio::test] +async fn close_deadline_preserves_unpolled_cleanup_handoff() { + for pcp in [false, true] { + let r = rig_with(Opts { + router: RouterOptions { + pmp: !pcp, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + let caps = Arc::new(PausedCleanupCaps { + inner: r.caps.clone(), + pause: std::sync::atomic::AtomicBool::new(false), + entered: tokio::sync::Notify::new(), + gate: (Mutex::new(false), std::sync::Condvar::new()), + }); + let (client, _) = Client::new(Config { + lookup: r.lookup.clone(), + caps: caps.clone(), + clock: r.clock.clone(), + protos: ProtoSet::BOTH, + advertise_nonpublic_external: true, + lifetime: requested_lifetime(), + }); + client.set_local_port(PORT).unwrap(); + client.tick().await; + while client.attempt_in_flight() || client.reap() != 0 { + tokio::task::yield_now().await; + } + { + let mut st = client.state(); + let id = st.mapping.take().unwrap().rule_id; + st.owned.orphan(id); + client.publish(&mut st); + } + let frozen = FrozenTime::new(); + let opener = OpenCapsOnDrop(caps.clone()); + caps.pause.store(true, std::sync::atomic::Ordering::SeqCst); + let mut cleanup = Box::pin(client.cleanup()); + tokio::select! { _ = caps.entered.notified() => {}, _ = &mut cleanup => panic!("cleanup did not pause") } + assert_owned_invariant(&client); + let budget = Duration::from_millis(20); + let start = tokio::time::Instant::now(); + finish_at_deadline(client.close(budget), budget).await; + assert_eq!(start.elapsed(), budget + Duration::from_millis(1)); + assert_eq!(client.state().phase, Phase::Closed); + assert!( + client.owned_snapshot()[0].deleting(), + "close stole a live reservation" + ); + assert_owned_invariant(&client); + let delete = r.router.barrier(if pcp { + RequestKind::PcpMap + } else { + RequestKind::PmpMap + }); + drop(cleanup); // a caller which was not polled during close now returns the claim + drop(opener); + let released = tokio::time::timeout(secs(1), delete.reached()) + .await + .is_ok(); + delete.release(); + drop(frozen); + assert_router_inventory(&client, GW1, &r.router); + assert!(!client.have_mapping()); + let table = r.router.mappings(); + r.shutdown().await; + assert!(released, "deadline discarded a deferred release"); + assert!(table.is_empty()); + } +} + +/// Exhaustive MAP cancellation cuts: before the future's first poll and after the router +/// commits but before its reply, for create/renew/adopt, every invalidation and both protocols. +#[tokio::test] +async fn exhaustive_map_cancellation_points() { + for pcp in [false, true] { + for mode in 0..3 { + for sent in [false, true] { + for action in 0..8 { + let r = rig_with(Opts { + router: RouterOptions { + pmp: !pcp, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + if mode != 0 { + r.map().await.unwrap(); + r.advance(secs(if mode == 1 { 3601 } else { 3 })); + if mode == 2 { + let mut st = r.client.state(); + let id = st.mapping.take().unwrap().rule_id; + st.owned.orphan(id); + r.client.publish(&mut st); + } + } else { + r.client.set_local_port(PORT).unwrap(); + r.client.check_network().await; + r.client.probe().await.unwrap(); + } + assert_router_inventory(&r.client, GW1, &r.router); + let kind = if pcp { + RequestKind::PcpMap + } else { + RequestKind::PmpMap + }; + let frozen = FrozenTime::new(); + let reply = sent.then(|| r.router.barrier(kind)); + r.client.get_cached_mapping_or_start_creating_one(); + assert!( + r.client.owned_snapshot()[0].mapping(), + "mode={mode},sent={sent},action={action}" + ); + assert_owned_invariant(&r.client); + if let Some(reply) = &reply { + reply.reached().await; + } + assert_router_inventory(&r.client, GW1, &r.router); + let router2 = if action == 3 { + Some( + FakeRouter::start(RouterOptions { + pmp: !pcp, + ..RouterOptions::default() + }) + .await + .unwrap(), + ) + } else { + None + }; + let possible_rule = sent || mode != 0; + let final_delete = + (action == 5 && possible_rule).then(|| r.router.barrier(kind)); + let before = delete_requests(&r.router); + match action { + 0 => r.client.note_rebind(Some(PORT)), + 1 => { + r.client.set_local_port(PORT + 1).unwrap(); + } + 2 => r.client.note_network_down(), + 3 => { + r.router.set_drop_deletes(true); + r.caps.add(GW2, router2.as_ref().unwrap().endpoint()); + r.lookup.set(gateway(GW2)); + r.client.check_network().await; + } + 4 => r.client.disable(DisableReason::OnlyTcp443), + 5 => r.client.release_blocking_best_effort(), + 6 => r.client.close(secs(1)).await, + _ => r + .client + .state() + .inflight + .as_ref() + .unwrap() + .abort + .as_ref() + .unwrap() + .abort(), + } + assert_owned_invariant(&r.client); + if let Some(reply) = reply { + reply.release(); + } + // Once the MAP guard settled, unrelated cleanup may legitimately need its + // timeout (the network-change cut deliberately loses DELETE replies). + while r.client.owned_snapshot().iter().any(|e| e.mapping()) { + assert_owned_invariant(&r.client); + tokio::task::yield_now().await; + } + drop(frozen); + r.settle().await; + if let Some(delete) = final_delete { + tokio::time::timeout(secs(1), delete.reached()) + .await + .unwrap(); + delete.release(); + } + assert_router_inventory(&r.client, GW1, &r.router); + if !possible_rule && action != 3 { + assert_eq!( + r.client.owned_rules(), + 0, + "unsent fresh reservation must retire" + ); + assert_eq!( + delete_requests(&r.router), + before, + "unsent identity must not be deleted" + ); + } + if action == 2 || action == 4 { + assert_eq!(delete_requests(&r.router), before, "fenced release sent"); + } + if action == 2 { + // A down network remains unsent even through terminal teardown. + r.client.close(secs(1)).await; + assert_eq!(delete_requests(&r.router), before); + } else if action != 5 && action != 6 { + if action == 4 { + r.client.enable(); + } + r.client.check_network().await; + assert_owned_invariant(&r.client); + r.advance(secs(700)); + r.client.tick().await; + r.settle().await; + assert!( + r.client.have_mapping(), + "recovery failed: pcp={pcp}, mode={mode}, sent={sent}, action={action}" + ); + r.client.cleanup().await; + assert_router_inventory(&r.client, GW1, &r.router); + } + if let Some(router2) = router2 { + assert_router_inventories(&r.client, &[(GW1, &r.router), (GW2, &router2)]); + let originals: Vec<_> = r + .client + .owned_snapshot() + .into_iter() + .filter(|e| e.gateway.gw == GW1) + .map(|e| (e.id, e.rule)) + .collect(); + router2.reboot(); + router2.set_epoch(10); + r.advance(secs(601)); + r.client.probe().await.unwrap(); + r.settle().await; + for (id, rule) in originals { + assert_eq!(r.client.state().owned.get(id).unwrap().rule, rule); + } + assert_router_inventory(&r.client, GW1, &r.router); + assert_router_inventories(&r.client, &[(GW1, &r.router), (GW2, &router2)]); + r.client.close(secs(1)).await; + assert_router_inventories(&r.client, &[(GW1, &r.router), (GW2, &router2)]); + router2.shutdown().await; + } + assert_owned_invariant(&r.client); + r.shutdown().await; + } + } + } + } +} + +#[tokio::test] +async fn network_down_fences_deferred_drop_release() { + for pcp in [false, true] { + let r = rig_with(Opts { + router: RouterOptions { + pmp: !pcp, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + r.map().await.unwrap(); + r.advance(secs(3601)); + r.client.get_cached_mapping_or_start_creating_one(); // guard captured, future not polled + assert!(r.client.owned_snapshot()[0].mapping()); + r.client.release_blocking_best_effort(); + assert_owned_invariant(&r.client); + r.client.note_network_down(); // must fence finalizers even though phase is Closed + r.settle().await; + // Drain loopback work without relying on a fixed delay. + r.client.cleanup().await; + assert_router_inventory(&r.client, GW1, &r.router); + let down = r.client.owned_snapshot().iter().all(|e| !e.releasable()); + let cap_valid = r.caps.built().iter().any(|cap| cap.is_valid()); + let deletes = delete_requests(&r.router); + r.shutdown().await; + assert!( + down && !cap_valid, + "closed client did not fence its pending release capability" + ); + assert_eq!(deletes, 0); + } +} + +struct OpenCapsOnDrop(Arc); +impl Drop for OpenCapsOnDrop { + fn drop(&mut self) { + *lock(&self.0.gate.0) = true; + self.0.gate.1.notify_all(); + } +} + +async fn settle_checked(client: &Client) { + tokio::time::timeout(secs(5), async { + loop { + assert_owned_invariant(client); + if !client.attempt_in_flight() && client.reap() == 0 { + break; + } + tokio::time::sleep(Duration::from_millis(1)).await; + } + }) + .await + .expect("operation failed to return ownership"); +} + +/// Each cleanup await cut, with one/three simultaneous claims and every teardown route. +#[tokio::test] +async fn exhaustive_cleanup_claim_handoffs() { + for pcp in [false, true] { + for batch in [1, 3] { + for validation in [false, true] { + for action in 0..4 { + let r = rig_with(Opts { + router: RouterOptions { + pmp: !pcp, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + let caps = Arc::new(PausedCleanupCaps { + inner: r.caps.clone(), + pause: std::sync::atomic::AtomicBool::new(false), + entered: tokio::sync::Notify::new(), + gate: (Mutex::new(false), std::sync::Condvar::new()), + }); + let opener = OpenCapsOnDrop(caps.clone()); + let (client, _) = Client::new(Config { + lookup: r.lookup.clone(), + caps: caps.clone(), + clock: r.clock.clone(), + protos: ProtoSet::BOTH, + advertise_nonpublic_external: true, + lifetime: requested_lifetime(), + }); + client.check_network().await; + client.probe().await.unwrap(); + // Build several real router rules without scheduling background cleanup between them. + for i in 0..batch { + { + let mut st = client.state(); + st.local_port = NonZeroU16::new(PORT + i); + } + r.advance(secs(3)); + assert_owned_invariant(&client); + client.get_cached_mapping_or_start_creating_one(); + settle_checked(&client).await; + { + let mut st = client.state(); + let id = st.mapping.take().unwrap().rule_id; + st.owned.orphan(id); + client.publish(&mut st); + } + assert_router_inventory(&client, GW1, &r.router); + } + let frozen = FrozenTime::new(); + let reply = (!validation).then(|| { + r.router.barrier(if pcp { + RequestKind::PcpMap + } else { + RequestKind::PmpMap + }) + }); + caps.pause + .store(validation, std::sync::atomic::Ordering::SeqCst); + let c = client.clone(); + let cleanup = tokio::spawn(async move { c.cleanup().await }); + if let Some(reply) = &reply { + reply.reached().await; + } else { + tokio::time::timeout(secs(1), caps.entered.notified()) + .await + .unwrap(); + } + assert!(client.owned_snapshot().iter().all(|e| e.deleting())); + assert_router_inventory(&client, GW1, &r.router); + let before = delete_requests(&r.router); + match action { + 0 => cleanup.abort(), + 1 => client.release_blocking_best_effort(), + 2 => client.close(secs(1)).await, + _ => client.note_network_down(), + } + assert_owned_invariant(&client); + drop(opener); + if let Some(reply) = reply { + reply.release(); + } + drop(cleanup.await); + drop(frozen); + assert!( + client + .owned_snapshot() + .iter() + .all(|e| !e.deleting() && !e.mapping()) + ); + assert_router_inventory(&client, GW1, &r.router); + if action == 0 { + assert_eq!(client.cleanup().await, usize::from(batch)); + assert!(client.owned_snapshot().is_empty()); + } else if action == 1 { + tokio::time::timeout(secs(1), async { + while delete_requests(&r.router) < before + usize::from(batch) { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!( + delete_requests(&r.router), + before + usize::from(batch), + "one final send per claimed rule" + ); + } else if action == 2 { + assert!( + client.owned_snapshot().is_empty(), + "close did not retire handed-back claims" + ); + } else { + assert_eq!( + delete_requests(&r.router), + before, + "network-down sent another delete" + ); + } + client.cleanup().await; + assert_router_inventory(&client, GW1, &r.router); + if action != 3 { + assert!(r.router.mappings().is_empty()); + } + assert!(!client.have_mapping()); + client.close(secs(1)).await; + assert_router_inventory(&client, GW1, &r.router); + r.shutdown().await; + } + } + } + } +} + +#[tokio::test] +async fn drop_during_close_consumes_final_duty_once() { + for pcp in [false, true] { + let r = rig_with(Opts { + router: RouterOptions { + pmp: !pcp, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + r.map().await.unwrap(); + let kind = if pcp { + RequestKind::PcpMap + } else { + RequestKind::PmpMap + }; + r.router.set_drop_deletes(true); + let frozen = FrozenTime::new(); + let in_close = r.router.barrier(kind); + let c = r.client.clone(); + let close = tokio::spawn(async move { c.close(secs(2)).await }); + in_close.reached().await; + assert_owned_invariant(&r.client); + assert!(r.client.owned_snapshot()[0].deleting()); + let final_delete = r.router.barrier(kind); + let duplicate = r.router.barrier(kind); + r.client.release_blocking_best_effort(); + r.router.set_drop_deletes(false); + in_close.release(); + tokio::time::advance(crate::cap::EXCHANGE_TIMEOUT + Duration::from_millis(1)).await; + close.await.unwrap(); + tokio::time::timeout(secs(1), final_delete.reached()) + .await + .unwrap(); + final_delete.release(); + drop(frozen); + let duplicated = tokio::time::timeout(secs(1), duplicate.reached()) + .await + .is_ok(); + duplicate.release(); + assert_router_inventory(&r.client, GW1, &r.router); + assert!(!r.client.have_mapping()); + assert!(r.router.mappings().is_empty()); + let count = delete_requests(&r.router); + r.shutdown().await; + assert!( + !duplicated, + "close rearmed a final duty already consumed by Drop" + ); + assert_eq!(count, 2, "one interrupted close send plus one final send"); + } +} + +async fn expired_renewal_cancel_case(pcp: bool, clock_aging: bool, unpolled: bool) { + let r = rig_with(Opts { + router: RouterOptions { + pmp: !pcp, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + r.map().await.unwrap(); + r.advance(secs(3601)); + r.client.check_clock(); + let frozen = FrozenTime::new(); + let reply = (!unpolled).then(|| { + r.router.barrier(if pcp { + RequestKind::PcpMap + } else { + RequestKind::PmpMap + }) + }); + r.client.get_cached_mapping_or_start_creating_one(); + if let Some(reply) = &reply { + reply.reached().await; + } + let original = r.client.owned_snapshot()[0].clone(); + if clock_aging { + r.clock.suspend(secs(3600)); + r.lease_clock.advance(3600); + assert!(r.client.check_clock()); + } else { + r.advance(secs(3600)); + assert!(r.client.expire_if_due()); + } + let expired = &r.client.owned_snapshot()[0]; + assert!( + expired.mapping(), + "expiry must retain the operation reservation" + ); + assert_eq!( + expired.rule, original.rule, + "expiry must retain the PCP nonce" + ); + assert_eq!( + expired.router_expiry(), + original.router_expiry(), + "trust expiry is not ownership expiry" + ); + assert!(!r.client.have_mapping()); + r.client.note_rebind(Some(PORT)); + if let Some(reply) = reply { + reply.release(); + } + while r.client.reap() != 0 { + tokio::task::yield_now().await; + } + drop(frozen); + let state = r.client.owned_snapshot()[0].state(); + let cleanup_due = r.client.schedule().cleanup_due; + r.client.check_network().await; + for _ in 0..3 { + r.advance(secs(700)); + r.client.tick().await; + while r.client.reap() != 0 { + tokio::task::yield_now().await; + } + } + let recovered = r.client.have_mapping(); + r.shutdown().await; + assert_eq!( + state, + crate::owned::RuleState::Orphaned, + "unowned Active; cleanup={cleanup_due:?}, recovered={recovered}" + ); + assert!( + cleanup_due.is_some(), + "ownerless entry has no cleanup deadline" + ); + assert!(recovered); +} + +#[tokio::test] +async fn expired_pcp_renewal_cancel() { + expired_renewal_cancel_case(true, false, false).await; +} +#[tokio::test] +async fn expired_pmp_renewal_cancel() { + expired_renewal_cancel_case(false, false, false).await; +} + +#[tokio::test] +async fn cancel_close_waiting_external_cleanup_then_foreign_drop() { + use std::{ + future::{Future, poll_fn}, + task::Poll, + }; + for pcp in [false, true] { + for network_down in [false, true] { + let r = rig_with(Opts { + router: RouterOptions { + pmp: !pcp, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + let caps = Arc::new(PausedCleanupCaps { + inner: r.caps.clone(), + pause: std::sync::atomic::AtomicBool::new(false), + entered: tokio::sync::Notify::new(), + gate: (Mutex::new(false), std::sync::Condvar::new()), + }); + let opener = OpenCapsOnDrop(caps.clone()); + let (client, _) = Client::new(Config { + lookup: r.lookup.clone(), + caps: caps.clone(), + clock: r.clock.clone(), + protos: ProtoSet::BOTH, + advertise_nonpublic_external: true, + lifetime: requested_lifetime(), + }); + client.set_local_port(PORT).unwrap(); + client.tick().await; + settle_checked(&client).await; + { + let mut st = client.state(); + let id = st.mapping.take().unwrap().rule_id; + st.owned.orphan(id); + client.publish(&mut st); + } + let frozen = FrozenTime::new(); + caps.pause.store(true, std::sync::atomic::Ordering::SeqCst); + let mut cleanup = Box::pin(client.cleanup()); + tokio::select! { + _ = caps.entered.notified() => {}, + _ = &mut cleanup => panic!("cleanup did not pause"), + } + let mut close = Box::pin(client.close(secs(10))); + poll_fn(|cx| { + assert!(close.as_mut().poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + assert_eq!(client.state().phase, Phase::Closing); + assert!(client.owned_snapshot()[0].deleting()); + drop(close); + assert_eq!(client.state().phase, Phase::Closed); + if network_down { + client.note_network_down(); + } + let delete = (!network_down).then(|| { + r.router.barrier(if pcp { + RequestKind::PcpMap + } else { + RequestKind::PmpMap + }) + }); + std::thread::scope(|scope| { + scope + .spawn(move || { + assert!(tokio::runtime::Handle::try_current().is_err()); + drop(cleanup); + }) + .join() + .unwrap(); + }); + drop(opener); + if let Some(delete) = delete { + tokio::time::timeout(secs(1), delete.reached()) + .await + .unwrap(); + delete.release(); + } + drop(frozen); + assert_owned_invariant(&client); + assert_eq!(delete_requests(&r.router), usize::from(!network_down)); + client.release_blocking_best_effort(); + r.shutdown().await; + } + } +} + +#[tokio::test] +async fn clock_aged_renewal_cancel() { + for pcp in [false, true] { + expired_renewal_cancel_case(pcp, true, false).await; + } +} + +#[tokio::test] +async fn expired_unpolled_renewal_cancel() { + for pcp in [false, true] { + for aged in [false, true] { + expired_renewal_cancel_case(pcp, aged, true).await; + } + } +} + +#[tokio::test] +async fn final_release_fenced_before_socket_send() { + for pcp in [false, true] { + for disable in [false, true] { + let r = rig_with(Opts { + router: RouterOptions { + pmp: !pcp, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + r.map().await.unwrap(); + let before = delete_requests(&r.router); + let entered = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let (c, hit) = (r.client.clone(), entered.clone()); + let hook = crate::cap::send_tests::set_send_hook(move || { + if disable { + c.disable(DisableReason::OnlyTcp443); + } else { + c.note_network_down(); + } + hit.store(true, std::sync::atomic::Ordering::SeqCst); + }); + r.client.release_blocking_best_effort(); + drop(hook); + r.settle().await; + assert!( + entered.load(std::sync::atomic::Ordering::SeqCst), + "final-send cut not reached" + ); + // A loopback-only marker round drains earlier datagrams; its independent policy + // cannot un-fence the client's release capability. Count every post-fence delete. + let marker = TestCap::new(gateway(GW1), &r.router, PolicyHandle::new(true)); + oneshot::probe( + &*marker, + ProtoSet { + pmp: true, + pcp: false, + }, + ) + .await + .unwrap(); + let deletes = delete_requests(&r.router); + assert_router_inventory(&r.client, GW1, &r.router); + assert!(!r.client.have_mapping()); + let table = r.router.mappings(); + r.shutdown().await; + assert_eq!( + deletes, before, + "final release emitted a datagram after its fence returned" + ); + assert_eq!(table.len(), 1); + } + } +} + +/// Aggregate physical router state, deduplicating aliases of the same fake endpoint. The +/// assignment must be injective: one ledger entry cannot explain rules on two routers, even +/// when their logical gateway/source/port/nonce values coincide. Unknown ledger slots need +/// not have a confirmed router rule, but they remain counted. +fn assert_router_inventories(client: &Client, routers: &[(Ipv4Addr, &FakeRouter)]) { + assert_owned_invariant(client); + let entries = client.owned_snapshot(); + let mut physical = HashMap::new(); + for (gw, router) in routers { + let (aliases, _) = physical + .entry(router.endpoint().addr) + .or_insert_with(|| (Vec::new(), router.mappings())); + aliases.push(*gw); + } + let actual: Vec<_> = physical + .values() + .flat_map(|(aliases, rules)| rules.iter().map(move |rule| (aliases, rule))) + .collect(); + assert!( + actual.len() <= entries.len() && actual.len() <= crate::owned::MAX_OWNED, + "aggregate router ownership exceeds counted slots" + ); + let candidates: Vec> = actual + .iter() + .map(|(aliases, rule)| { + entries + .iter() + .enumerate() + .filter_map(|(i, e)| { + (aliases.contains(&e.gateway.gw) + && rule.proto == 17 + && e.gateway.self_ip == rule.client_ip + && e.rule.internal().get() == rule.internal + && match &e.rule { + Rule::Pcp { nonce, .. } => rule.nonce == Some(*nonce.as_bytes()), + Rule::Pmp { .. } => rule.nonce.is_none(), + }) + .then_some(i) + }) + .collect() + }) + .collect(); + fn assign(row: usize, choices: &[Vec], used: &mut HashSet) -> bool { + if row == choices.len() { + return true; + } + for &owner in &choices[row] { + if used.insert(owner) { + if assign(row + 1, choices, used) { + return true; + } + used.remove(&owner); + } + } + false + } + assert!( + assign(0, &candidates, &mut HashSet::new()), + "router rules have no injective ledger attribution: {actual:?}; ledger={entries:?}" + ); +} + +#[tokio::test] +async fn oracle_rejects_unexplained_retirement() { + use crate::ownership_model::{RefusalEvidence, Retirement}; + for fault in 0..7 { + let r = rig().await; + r.map().await.unwrap(); + r.router.reboot(); // true router state alone is not accepted client reset evidence + { + let mut st = r.client.state(); + let h = st.mapping.take().unwrap(); + let reason = match fault { + 0 => None, + 1 => Some(Retirement::Delete { matched: false }), + 2 => Some(Retirement::Expiry { + observed_at: r.clock.now(), + }), + 3 => Some(Retirement::Reset { + observation: crate::ownership_model::ResetId::MISSING, + gateway: gateway(GW2), + proto: Proto::Pmp, + }), + 4 => Some(Retirement::Reset { + observation: crate::ownership_model::ResetId::MISSING, + gateway: gateway(GW1), + proto: Proto::Pmp, + }), + 5 => Some(Retirement::Refusal(RefusalEvidence::Unproven)), + _ => Some(Retirement::ProvenUnsent), + }; + if let Some(reason) = reason { + st.owned.record_retirement(h.rule_id, reason); + } + st.owned.remove(h.rule_id); // inject a disappearance, with missing/false evidence + r.client.publish(&mut st); + } + let observed = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + assert_owned_invariant(&r.client) + })); + r.shutdown().await; + assert!( + observed.is_err(), + "oracle accepted unexplained/false retirement: fault={fault}" + ); + } +} + +#[tokio::test] +async fn oracle_injective_across_routers() { + let r = rig().await; + r.map().await.unwrap(); + let original = r.client.owned_snapshot()[0].rule.clone(); + { + let mut st = r.client.state(); + let id = st.mapping.take().unwrap().rule_id; + st.owned.orphan(id); + st.local_port = NonZeroU16::new(PORT + 1); + r.client.publish(&mut st); + } + r.advance(secs(3)); + r.client.get_cached_mapping_or_start_creating_one(); + r.settle().await; + let unused = r + .client + .owned_snapshot() + .into_iter() + .find(|e| e.rule.internal().get() == PORT + 1) + .unwrap() + .rule; + let cap1 = TestCap::new(gateway(GW1), &r.router, r.client.policy().clone()); + oneshot::delete(&*cap1, &unused).await.unwrap(); + // Two possible owners and two actual rules: a count bound alone cannot catch reuse. + let router2 = FakeRouter::start(RouterOptions::default()).await.unwrap(); + let cap2 = TestCap::new(gateway(GW1), &router2, r.client.policy().clone()); + let rule = original; + oneshot::map(&*cap2, &rule, requested_lifetime(), None) + .await + .unwrap(); + // The old per-router oracle attributes both physical rules to the same one entry. + let observed = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + assert_router_inventories(&r.client, &[(GW1, &r.router), (GW1, &router2)]) + })); + oneshot::delete(&*cap2, &rule).await.unwrap(); + router2.shutdown().await; + r.shutdown().await; + assert!( + observed.is_err(), + "oracle reused one ledger entry for two physical router rules" + ); +} + +#[tokio::test] +async fn ownership_oracle_rejects_reset_before_admission() { + use crate::ownership_model::Retirement; + let r = rig().await; + r.map().await.unwrap(); + { + let mut st = r.client.state(); + let h = st.mapping.take().unwrap(); + let committed = h.lease.committed_at - secs(20); + let observation = st.owned.model.accept_reset( + gateway(GW1), + Proto::Pmp, + 100, + 10, + committed, + committed + secs(1), + ); + st.owned.record_retirement( + h.rule_id, + Retirement::Reset { + observation, + gateway: gateway(GW1), + proto: Proto::Pmp, + }, + ); + st.owned.remove(h.rule_id); + } + let rejected = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let st = r.client.state(); + st.owned.model.check(st.owned.entries()); + })); + assert_eq!(r.router.mappings().len(), 1, "the rule is still live"); + r.shutdown().await; + assert!( + rejected.is_err(), + "historical reset cannot retire a later rule" + ); +} + +#[tokio::test] +async fn expired_renewal_success_preserves_identity() { + for pcp in [false, true] { + for aged in [false, true] { + let r = rig_with(Opts { + router: RouterOptions { + pmp: !pcp, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + r.map().await.unwrap(); + r.advance(secs(3601)); + r.client.check_clock(); + let frozen = FrozenTime::new(); + let reply = r.router.barrier(if pcp { + RequestKind::PcpMap + } else { + RequestKind::PmpMap + }); + r.client.get_cached_mapping_or_start_creating_one(); + reply.reached().await; + let before = r.client.owned_snapshot()[0].clone(); + if aged { + r.clock.suspend(secs(3600)); + r.lease_clock.advance(3600); + assert!(r.client.check_clock()); + } else { + r.advance(secs(3600)); + assert!(r.client.expire_if_due()); + } + assert!(!r.client.have_mapping()); + assert_owned_invariant(&r.client); + reply.release(); + while r.client.reap() != 0 { + tokio::task::yield_now().await; + } + drop(frozen); + r.settle().await; + assert!(r.client.have_mapping()); + let after = r.client.owned_snapshot()[0].clone(); + assert_eq!(before.id, after.id); + assert_eq!(before.rule, after.rule); + assert_router_inventory(&r.client, GW1, &r.router); + r.shutdown().await; + } + } +} + +/// Close cannot replace an invalid cached capability: retain and fence ownership without I/O. +#[tokio::test] +async fn close_with_invalid_cached_capability_fences_retained_rule() { + for pcp in [false, true] { + let r = rig_with(Opts { + router: RouterOptions { + pmp: !pcp, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + r.map().await.unwrap(); + let builds = r.caps.builds(); + let lookups = r.lookup.calls(); + let received = r.router.counters().num_total_recv; + let original = r.client.owned_snapshot()[0].clone(); + for cap in r.client.state().release_caps.values() { + cap.invalidate(); + } + + r.client.close(secs(1)).await; + assert_eq!(r.client.state().phase, Phase::Closed); + assert!(!r.client.have_mapping()); + assert_eq!(r.client.owned_rules(), 1); + let e = r.client.owned_snapshot()[0].clone(); + assert_eq!(e.id, original.id); + assert_eq!(e.rule, original.rule); + assert_eq!(e.state(), crate::owned::RuleState::Orphaned); + assert!(!e.releasable(), "close must terminally fence the orphan"); + assert!(!e.mapping() && !e.deleting(), "reservation returned"); + assert!( + e.retry_at.is_none(), + "terminal validation failure must not schedule recovery" + ); + assert_owned_invariant(&r.client); + + r.client.release_blocking_best_effort(); + assert_eq!(r.client.cleanup().await, 0); + assert_eq!( + r.caps.builds(), + builds, + "no replacement capability discovery" + ); + assert_eq!(r.lookup.calls(), lookups, "no fresh gateway discovery"); + assert_eq!(r.router.counters().num_total_recv, received, "no sends"); + assert_eq!(delete_requests(&r.router), 0); + assert_eq!(r.router.mappings().len(), 1, "router rule remains owned"); + assert_router_inventory(&r.client, GW1, &r.router); + r.shutdown().await; + } +} + +#[tokio::test] +async fn fresh_bind_failure_releases_reservation() { + let mut observed = Vec::new(); + for pcp in [false, true] { + let r = rig_with(Opts { + router: RouterOptions { + pmp: !pcp, + ..RouterOptions::default() + }, + protos: ProtoSet { pmp: !pcp, pcp }, + ..Opts::default() + }) + .await; + r.client.set_local_port(PORT).unwrap(); + r.client.check_network().await; + r.client.probe().await.unwrap(); + for cap in r.caps.built() { + cap.inject_bind_error(Some(ADDR_NOT_AVAILABLE)); + } + // A failed setup must not depend on successful cleanup to recover its slot. + r.caps.set_present(GW1, false); + let received = r.router.counters().num_total_recv; + for offset in 0..5 { + r.client.set_local_port(PORT + offset).unwrap(); + r.advance(secs(700)); + r.client.get_cached_mapping_or_start_creating_one(); + let started = r.client.attempt_in_flight(); + if started { + assert_eq!( + r.client.owned_snapshot().last().unwrap().rule.proto(), + if pcp { Proto::Pcp } else { Proto::Pmp } + ); + } + r.settle().await; + observed.push((pcp, offset, started, r.client.owned_rules())); + assert!(r.router.mappings().is_empty()); + assert_router_inventory(&r.client, GW1, &r.router); + } + assert_eq!(delete_requests(&r.router), 0, "never delete an unsent rule"); + assert_eq!( + r.router.counters().num_total_recv, + received, + "no MAP or DELETE" + ); + r.shutdown().await; + } + assert!( + observed + .iter() + .all(|(_, _, started, slots)| *started && *slots == 0), + "failed setup pinned admission, including the fifth identity: {observed:?}" + ); +} + +#[tokio::test] +async fn unsent_adoption_preserves_prior_expiry() { + for pcp in [false, true] { + for renewal in [false, true] { + let r = rig_with(Opts { + router: RouterOptions { + pmp: !pcp, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + r.map().await.unwrap(); + let before = r.client.owned_snapshot()[0].clone(); + r.advance(secs(if renewal { 3601 } else { 3 })); + if !renewal { + r.client.note_network_down(); + r.client.check_network().await; + } + for cap in r.caps.built() { + cap.inject_bind_error(Some(ADDR_NOT_AVAILABLE)); + } + r.caps.set_present(GW1, false); + let received = r.router.counters().num_total_recv; + r.client.get_cached_mapping_or_start_creating_one(); + r.settle().await; + let after = r.client.owned_snapshot()[0].clone(); + assert_eq!(after.id, before.id); + assert_eq!(after.rule, before.rule, "retain protocol and PCP nonce"); + assert_eq!( + after.router_expiry(), + before.router_expiry(), + "unsent operation inflated grant" + ); + assert_eq!( + r.client.have_mapping(), + renewal, + "do not extend advertisement trust" + ); + assert_eq!(r.router.counters().num_total_recv, received); + assert_router_inventory(&r.client, GW1, &r.router); + r.client.note_network_down(); + r.shutdown().await; + } + } +} + +#[tokio::test] +async fn send_evidence_survives_cancellation() { + for pcp in [false, true] { + for sent in [false, true] { + let r = rig_with(Opts { + router: RouterOptions { + pmp: !pcp, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + r.client.set_local_port(PORT).unwrap(); + r.client.check_network().await; + r.client.probe().await.unwrap(); + let kind = if pcp { + RequestKind::PcpMap + } else { + RequestKind::PmpMap + }; + let frozen = FrozenTime::new(); + let reply = sent.then(|| r.router.barrier(kind)); + r.client.get_cached_mapping_or_start_creating_one(); + let rule = r.client.owned_snapshot()[0].rule.clone(); + if let Some(reply) = &reply { + reply.reached().await; + } + // Same-port rebind cancels the operation without deleting its possible grant. + r.client.note_rebind(Some(PORT)); + if let Some(reply) = reply { + reply.release(); + } + while r.client.reap() != 0 { + tokio::task::yield_now().await; + } + drop(frozen); + r.settle().await; + assert_eq!( + r.client.owned_rules(), + usize::from(sent), + "pcp={pcp},sent={sent}" + ); + assert_eq!(r.router.mappings().len(), usize::from(sent)); + if sent { + let entry = r.client.owned_snapshot()[0].clone(); + assert_eq!(entry.rule, rule); + assert!(entry.router_expiry() >= r.clock.now() + secs(u64::from(u32::MAX) - 1)); + assert!(!entry.mapping() && !entry.deleting()); + } + assert_eq!(delete_requests(&r.router), 0); + assert_router_inventory(&r.client, GW1, &r.router); + r.shutdown().await; + } + } +} diff --git a/ts_portmapper/src/clock.rs b/ts_portmapper/src/clock.rs new file mode 100644 index 00000000..9155ccc4 --- /dev/null +++ b/ts_portmapper/src/clock.rs @@ -0,0 +1,52 @@ +//! The time source the lifecycle client schedules against. +//! +//! [`SystemClock`] is the monotonic clock plus the wall clock. Tests use a manual clock (in +//! `testing`) that adds an offset to the real monotonic clock, so lease arithmetic can jump hours +//! while the loopback exchanges still run against real 250 ms deadlines. +//! +//! Neither macOS (`CLOCK_UPTIME_RAW`) nor Linux (`CLOCK_MONOTONIC`) advances [`Instant`] while the +//! host sleeps, so the client compares the wall clock's progress with the monotonic clock's to +//! notice a resume or a wall-clock jump. + +use std::time::SystemTime; + +use tokio::time::Instant; + +/// A monotonic clock and a wall clock. +pub trait Clock: Send + Sync + 'static { + /// The monotonic time the schedule runs on. + fn now(&self) -> Instant; + + /// The wall-clock time, only for noticing suspend/resume and clock jumps. + fn wall(&self) -> SystemTime; + + /// The real monotonic instant that corresponds to `t` on this clock (for arming timers and for + /// consumers that compare against the real clock). + fn to_real(&self, t: Instant) -> Instant { + let now = self.now(); + let real = Instant::now(); + if t >= now { + real + (t - now) + } else { + real.checked_sub(now - t).unwrap_or(real) + } + } +} + +/// The real clocks. +#[derive(Debug, Clone, Copy, Default)] +pub struct SystemClock; + +impl Clock for SystemClock { + fn now(&self) -> Instant { + Instant::now() + } + + fn wall(&self) -> SystemTime { + SystemTime::now() + } + + fn to_real(&self, t: Instant) -> Instant { + t + } +} diff --git a/ts_portmapper/src/lib.rs b/ts_portmapper/src/lib.rs index 6ccf1bc4..2a7e07df 100644 --- a/ts_portmapper/src/lib.rs +++ b/ts_portmapper/src/lib.rs @@ -19,17 +19,25 @@ //! `/proc/net/route`, `Unsupported` elsewhere). //! - [`pmp`] and [`pcp`]: pure, strictly validating codecs. //! - [`oneshot`]: single-attempt probe, map and delete. +//! - [`client`]: the lifecycle client (create, renew, expire, release) with generation-guarded +//! transitions; [`schedule`] and `owned` are its pure scheduling and owned-rule ledger. //! - [`probe_tool`]: the `--portmap-probe` diagnostic used to validate a deployment. //! - `testing` (feature `test-harness`): a fake NAT-PMP/PCP router and `TestCap`, a loopback //! transport that production code cannot construct. mod blocking; mod cap; +pub mod client; +pub mod clock; pub mod gateway; pub mod oneshot; +mod owned; pub mod pcp; pub mod pmp; pub mod probe_tool; +pub mod schedule; +#[cfg(test)] +mod test_time; #[cfg(any(test, feature = "test-harness"))] pub mod testing; @@ -47,8 +55,14 @@ pub const TEST_HARNESS_ENABLED: bool = cfg!(feature = "test-harness"); pub use cap::{ BindMode, CapError, DenyReason, EXCHANGE_TIMEOUT, ExchangeStats, Flow, GatewayCap, LanDest, - MAX_EXAMINED, NetGen, PolicyHandle, ProbeListener, PxpTransport, RECV_BUF, errno_name, + MAX_EXAMINED, NetGen, PolicyHandle, ProbeListener, PxpTransport, RECV_BUF, SendEvidence, + errno_name, +}; +pub use client::{ + CapFactory, Client, Config, DisableReason, NetCheck, PortMapper, PortmapEndpoint, PortmapState, + ProbeFacets, ScheduleSnapshot, SystemCaps, }; +pub use clock::{Clock, SystemClock}; pub use gateway::{Gateway, GatewayLookup, LookupError}; /// The UDP port NAT-PMP and PCP servers listen on (RFC 6886 §3, RFC 6887 §19.1). @@ -114,3 +128,6 @@ mod tests { } } } + +#[cfg(test)] +mod ownership_model; diff --git a/ts_portmapper/src/lifecycle.rs b/ts_portmapper/src/lifecycle.rs new file mode 100644 index 00000000..975b7224 --- /dev/null +++ b/ts_portmapper/src/lifecycle.rs @@ -0,0 +1,944 @@ +//! Serialized settlement owner. See the transition table in `docs/PORTMAPPER.md`. +//! +//! This module performs no asynchronous I/O. A caller holds the client state mutex for one +//! event; reservations outlive handle cancellation and are returned only by finalizers. + +use super::*; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum Phase { + Running, + Closing, + Closed, +} + +pub(super) struct NetSnapshot { + pub(super) net_gen: NetGen, + pub(super) cap: Arc, + pub(super) release_cap: Arc, + pub(super) gateway: Gateway, +} + +#[derive(Debug, Default)] +pub(super) struct ProbeCache { + pub(super) pmp_seen_at: Option, + pub(super) pmp_public_ip: Option, + pub(super) pcp_seen_at: Option, + pub(super) last_probe_at: Option, + /// The last probe that sent anything found no service. Until the attempt backoff allows the + /// next attempt, a probe then sends nothing either, so repeated kicks cannot turn a router + /// without NAT-PMP or PCP into a probe storm. + pub(super) found_nothing: bool, +} + +impl ProbeCache { + pub(super) fn fresh(t: Option, now: Instant) -> bool { + t.is_some_and(|t| now.saturating_duration_since(t) < PROBE_CACHE) + } + + pub(super) fn saw_pmp(&self, now: Instant) -> bool { + Self::fresh(self.pmp_seen_at, now) + } + + pub(super) fn saw_pcp(&self, now: Instant) -> bool { + Self::fresh(self.pcp_seen_at, now) + } +} + +#[derive(Debug, Clone)] +pub(super) struct Held { + pub(super) m: oneshot::Mapping, + pub(super) lease: Lease, + pub(super) net_gen: NetGen, + pub(super) rule_id: RuleId, +} + +pub(super) struct Inflight { + pub(super) id: u64, + pub(super) rule_id: RuleId, + pub(super) abort: Option, +} + +/// Evidence available when withdrawing the held owner. Reservations survive every variant. +#[derive(Debug, Clone, PartialEq, Eq)] +enum Invalidation { + /// None: every orphan is deleted once allowed. + Orphan, + /// All of them: a delete could reach the wrong router (network down). They stay in the + /// ledger, never deleted, so a create for the same rule on the same network can adopt them. + Fence, + /// This protocol's: the router lost them (epoch rollback). The rule of an attempt in flight + /// (the `RuleId`) is kept: its MAP may still be committed by the restarted router. + Reset(Proto, Gateway, Option), +} + +pub(super) struct Lifecycle { + data: View, +} + +/// Read projections only. Production callers cannot bypass settlement with a mutable view. +pub(super) struct View { + pub(super) generation: u64, + pub(super) phase: Phase, + pub(super) local_port: Option, + pub(super) net: Option>, + pub(super) next_net_gen: u64, + pub(super) probe: ProbeCache, + pub(super) facets: ProbeFacets, + pub(super) mapping: Option, + pub(super) sched: Schedule, + pub(super) owned: OwnedRules, + pub(super) release_caps: HashMap>, + pub(super) rebind_pending: bool, + pub(super) inflight: Option, + pub(super) next_attempt_id: u64, + /// A caller wanted an attempt that spacing refused or that failed; the driver may retry at + /// the earliest start. + pub(super) retry_wanted: bool, + pub(super) disabled: Option, + pub(super) last_clock: Option<(Instant, SystemTime)>, + /// Distinct operational warnings (lookup, gateway, OS errors, ledger, nonce-verified PCP + /// refusals) already logged in this network generation. + pub(super) warned: HashSet, + /// Distinct warnings about router replies anyone on the LAN can forge (NAT-PMP codes, + /// unverified PCP headers, malformed replies): a separate budget, so a noisy router or a + /// spoofer cannot use up the operational one. + pub(super) warned_router: HashSet, + /// Info lines already logged in this network generation (suppressed advertisements). + pub(super) noted: HashSet, +} + +impl std::ops::Deref for Lifecycle { + type Target = View; + fn deref(&self) -> &Self::Target { + &self.data + } +} + +// Fault-injection tests deliberately violate transitions to audit the independent oracle. +#[cfg(test)] +impl std::ops::DerefMut for Lifecycle { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.data + } +} + +impl Lifecycle { + pub(super) fn warn_once(&mut self, key: String) -> bool { + Self::once(&mut self.data.warned, key) + } + + /// [`warn_once`](Self::warn_once) for warnings about forgeable router replies. + pub(super) fn warn_once_router(&mut self, key: String) -> bool { + Self::once(&mut self.data.warned_router, key) + } + + pub(super) fn once(set: &mut HashSet, key: String) -> bool { + if set.contains(&key) || set.len() >= MAX_WARNED { + return false; + } + set.insert(key); + true + } + + pub(super) fn note_once(&mut self, key: String) -> bool { + if self.data.noted.contains(&key) || self.data.noted.len() >= MAX_WARNED { + return false; + } + self.data.noted.insert(key); + true + } +} +impl Lifecycle { + pub(super) fn new() -> Self { + Self { + data: View { + generation: 0, + phase: Phase::Running, + local_port: None, + net: None, + next_net_gen: 1, + probe: ProbeCache::default(), + facets: ProbeFacets::default(), + mapping: None, + sched: Schedule::default(), + owned: OwnedRules::default(), + release_caps: HashMap::new(), + rebind_pending: false, + inflight: None, + next_attempt_id: 0, + retry_wanted: false, + disabled: None, + last_clock: None, + warned: HashSet::new(), + warned_router: HashSet::new(), + noted: HashSet::new(), + }, + } + } + fn invalidate(&mut self, why: &str, evidence: Invalidation) { + self.cancel_map_handle(); + let had = self.data.mapping.take(); + self.data.owned.orphan_all(); + let gone = match evidence { + Invalidation::Orphan => Vec::new(), + Invalidation::Fence => self.data.owned.forget_orphans(), + Invalidation::Reset(p, gateway, keep) => self.data.owned.drop_proto(p, &gateway, keep), + }; + for e in &gone { + log_left(e, LeftReason::NetworkGone); + } + if let Some(h) = had { + tracing::info!( + proto = %h.m.proto, + external = %h.m.external, + reason = why, + "portmapper: mapping lost" + ); + } + } + + pub(super) fn network_down(&mut self, why: &str) { + self.invalidate(why, Invalidation::Fence); + if let Some(net) = self.data.net.take() { + net.cap.invalidate(); + } + for cap in self.data.release_caps.values() { + cap.invalidate(); + } + self.data.rebind_pending = false; + self.data.probe = ProbeCache::default(); + if self.data.disabled.is_none() { + self.data.facets = ProbeFacets::default(); + } + self.data.warned.clear(); + self.data.warned_router.clear(); + self.data.noted.clear(); + } + + pub(super) fn lease_expired(&mut self, now: Instant) { + let Some(h) = self.data.mapping.take() else { + return; + }; + tracing::info!( + proto = %h.m.proto, + external = %h.m.external, + "portmapper: mapping expired" + ); + // Removing the held lease removes its owner, even while a renewal reserves the wire + // identity. Keep that reservation and the possible grant; a success may still commit. + self.data.owned.orphan(h.rule_id); + let renewing = self + .data + .inflight + .as_ref() + .is_some_and(|i| i.rule_id == h.rule_id); + if renewing { + return; + } + // The router may hold it longer than we trusted it (PCP may grant more than asked): then + // it is deleted like any leftover; otherwise it is simply gone. + if self + .data + .owned + .get(h.rule_id) + .is_some_and(|e| e.mapping() || e.router_expiry() > now) + { + self.data.owned.orphan(h.rule_id); + } else { + #[cfg(test)] + self.data.owned.record_retirement( + h.rule_id, + crate::ownership_model::Retirement::Expiry { observed_at: now }, + ); + self.data.owned.remove(h.rule_id); + } + } + + pub(super) fn router_reset( + &mut self, + proto: Proto, + epoch: u32, + started: Instant, + gateway: &Gateway, + ) -> bool { + if epoch == 0 { + return false; + } + let Some(h) = &self.data.mapping else { + return false; + }; + if h.m.proto != proto || epoch >= h.m.epoch || h.lease.committed_at > started { + return false; + } + #[cfg(test)] + let reset_owner = (h.m.epoch, h.lease.committed_at); + tracing::info!( + %proto, + epoch, + stored = h.m.epoch, + "portmapper: router epoch went back; it restarted and lost the mapping" + ); + let inflight = self.data.inflight.as_ref().map(|i| i.rule_id).filter(|id| { + self.data + .owned + .get(*id) + .is_some_and(|e| e.rule.proto() == proto) + }); + #[cfg(test)] + self.data.owned.model.accept_reset( + gateway.clone(), + proto, + reset_owner.0, + epoch, + reset_owner.1, + started, + ); + self.invalidate( + "router restarted", + Invalidation::Reset(proto, gateway.clone(), inflight), + ); + if let Some(id) = inflight { + // The in-flight request may receive an unobserved grant longer than requested. + self.data.owned.orphan(id); + } + match proto { + Proto::Pmp => { + self.data.probe.pmp_seen_at = None; + self.data.probe.pmp_public_ip = None; + } + Proto::Pcp => self.data.probe.pcp_seen_at = None, + } + true + } + + pub(super) fn map_success( + &mut self, + ctx: &AttemptCtx, + m: oneshot::Mapping, + now: Instant, + ) -> (bool, bool) { + let mut need_cleanup = false; + let mut publish = false; + let stale = self.attempt_stale(ctx); + + let lease = Lease::new(now, m.lifetime); + if stale { + self.data + .owned + .orphan_committed(ctx.rule_id, lease.router_expiry); + tracing::info!( + proto = %m.proto, + external = %m.external, + "portmapper: discarding a mapping that arrived after an \ + invalidation; deleting it" + ); + need_cleanup = true; + } else { + self.data.owned.commit(ctx.rule_id, lease.router_expiry); + let old = self.data.mapping.take(); + if let Some(old) = &old + && old.rule_id != ctx.rule_id + { + // A protocol switch or a new internal port: the old rule is released. + self.data.owned.orphan(old.rule_id); + need_cleanup = true; + } + match &old { + Some(o) if o.m.external == m.external => tracing::debug!( + proto = %m.proto, + external = %m.external, + lease = m.lifetime, + "portmapper: mapping renewed" + ), + Some(o) => tracing::info!( + proto = %m.proto, + external = %m.external, + previous = %o.m.external, + lease = m.lifetime, + "portmapper: mapping renewed with a new external address" + ), + None => tracing::info!( + proto = %m.proto, + external = %m.external, + lease = m.lifetime, + "portmapper: mapping obtained" + ), + } + self.data.mapping = Some(Held { + m, + lease, + net_gen: ctx.net_gen, + rule_id: ctx.rule_id, + }); + self.data.sched.note_success(); + publish = true; + } + (publish, need_cleanup) + } + + fn attempt_stale(&self, ctx: &AttemptCtx) -> bool { + ctx.generation != self.data.generation + || self.data.phase != Phase::Running + || self.data.disabled.is_some() + } + pub(super) fn map_refused(&mut self, ctx: &AttemptCtx, e: MapError, now: Instant) -> bool { + let mut need_cleanup = false; + let stale = self.attempt_stale(ctx); + // A renewal whose mapping is gone (expired meanwhile) settles like a create. + let renewal = ctx.prev.is_some() + && self + .data + .mapping + .as_ref() + .is_some_and(|h| h.rule_id == ctx.rule_id); + + self.data.owned.discard_unsent(ctx.rule_id); + if !renewal { + match &e { + MapError::PmpAddr { map: Some(r), .. } => self + .data + .owned + .orphan_committed(ctx.rule_id, lease_deadline(now, r.lifetime)), + e if ctx.existed || may_exist(e) => self.data.owned.orphan(ctx.rule_id), + _ => { + #[cfg(test)] + self.data.owned.record_retirement( + ctx.rule_id, + crate::ownership_model::Retirement::Refusal(match &e { + MapError::Pmp { .. } => { + crate::ownership_model::RefusalEvidence::MatchedMap(Proto::Pmp) + } + MapError::Pcp { .. } => { + crate::ownership_model::RefusalEvidence::MatchedMap(Proto::Pcp) + } + _ => crate::ownership_model::RefusalEvidence::Unproven, + }), + ); + self.data.owned.remove(ctx.rule_id); + } + } + need_cleanup |= self.data.owned.has_orphans(); + } + if !stale { + self.data + .sched + .note_failure(now, failure_kind(&e), jitter()); + // The wanted mapping is still missing (or due): retry at the backoff. + self.data.retry_wanted = true; + let (key, router) = warn_key(&e); + let key = format!("map {key}"); + let first = if router { + self.warn_once_router(key) + } else { + self.warn_once(key) + }; + if first { + tracing::warn!( + proto = %ctx.rule.proto(), + error = %e, + "portmapper: mapping attempt failed" + ); + } else { + tracing::debug!(error = %e, "portmapper: mapping attempt failed"); + } + } + need_cleanup + } +} + +/// The guard carries an event, not a second copy of lifecycle state. +pub(super) enum FinalizerEvent { + Map { id: u64, rule_id: RuleId }, + Delete(Vec), + FinalRelease(Vec), + Close, +} + +impl Lifecycle { + fn cancel_map_handle(&mut self) { + self.data.generation += 1; + if let Some(i) = self.data.inflight.take() + && let Some(a) = i.abort + { + a.abort(); + } + } + pub(super) fn map_task_attached(&mut self, id: u64, abort: AbortHandle) { + if let Some(i) = &mut self.data.inflight + && i.id == id + { + i.abort = Some(abort); + } + } + pub(super) fn map_finished(&mut self, id: u64, rule_id: RuleId) { + self.data.owned.finish_map(rule_id); + if self.data.inflight.as_ref().is_some_and(|i| i.id == id) { + self.data.inflight = None; + } + } + pub(super) fn cleanup_cancelled(&mut self, ids: &[RuleId]) { + for id in ids { + self.data.owned.unclaim(*id); + } + } + pub(super) fn final_release_finished(&mut self, ids: &[RuleId]) { + // No acknowledgement was awaited: release the reservation, never the admission slot. + self.cleanup_cancelled(ids); + } + pub(super) fn delete_validation_failed(&mut self, id: RuleId, closing: bool, now: Instant) { + if closing { + self.data.owned.abandon(id); + } else { + self.data.owned.defer_delete(id, now); + } + } + pub(super) fn delete_fenced(&mut self, id: RuleId) { + self.data.owned.abandon(id); + } + pub(super) fn final_release_started( + &mut self, + policy: &PolicyHandle, + now: Instant, + ) -> Vec { + if !policy.lan_sends_allowed() { + return Vec::new(); + } + self.data.owned.claim_final_releases(now) + } + pub(super) fn rebind_failed(&mut self) { + self.data.local_port = None; + self.invalidate("rebind failed", Invalidation::Orphan); + } + pub(super) fn map_started( + &mut self, + cfg: &Config, + policy: &PolicyHandle, + now: Instant, + ) -> Result<(u64, AttemptCtx), Error> { + // Only a refusal by attempt spacing or a failure asks the driver to come back; any other + // outcome of this request (started, busy, disabled, offline) does not. + self.data.retry_wanted = false; + if self.data.phase != Phase::Running || tokio::runtime::Handle::try_current().is_err() { + return Err(Error::Closed); + } + if self.data.disabled.is_some() || !policy.lan_sends_allowed() { + return Err(Error::Disabled); + } + let local_port = self.data.local_port.ok_or(Error::NoLocalPort)?; + let Some(net) = &self.data.net else { + return Err(Error::NoNetwork); + }; + let (cap, net_gen, gateway) = (net.cap.clone(), net.net_gen, net.gateway.clone()); + if self.data.inflight.is_some() || !cap.is_valid() || self.data.rebind_pending { + return Err(Error::Busy); + } + if !self.data.sched.may_start(now) { + self.data.retry_wanted = true; + return Err(Error::Backoff); + } + let protos = cfg.protos; + let recent_pmp = self.data.probe.saw_pmp(now); + let recent_pcp = self.data.probe.saw_pcp(now); + let probed_recently = self + .data + .probe + .last_probe_at + .is_some_and(|t| now.saturating_duration_since(t) < RECENT_PROBE); + if (!protos.pmp && !protos.pcp) || (probed_recently && !recent_pmp && !recent_pcp) { + self.data.sched.note_start(now); + self.data + .sched + .note_failure(now, Failure::Transient, jitter()); + self.data.retry_wanted = true; + return Err(Error::NoServices); + } + let prefer_pcp = protos.pcp && (!protos.pmp || (!recent_pmp && recent_pcp)); + let mut proto = if prefer_pcp { Proto::Pcp } else { Proto::Pmp }; + // An orphan still owns the same UDP wire identity. Adopt its protocol and nonce; + // never create an alias while any MAP or delete (including OS validation) owns it. + if let Some(e) = self.data.owned.for_identity(local_port, &gateway) { + if e.mapping() || e.deleting() { + self.data.retry_wanted = true; + return Err(Error::Busy); + } + proto = e.rule.proto(); + if !match proto { + Proto::Pmp => protos.pmp, + Proto::Pcp => protos.pcp, + } { + return Err(Error::Busy); + } + } + + // A renewal: the same rule (protocol, internal port, nonce) on the same network. + let prev = self + .data + .mapping + .as_ref() + .filter(|h| h.net_gen == net_gen && h.m.proto == proto && h.m.internal == local_port) + .cloned(); + let adopted = match &prev { + Some(_) => None, + None => self.data.owned.adopt(proto, local_port, &gateway, net_gen), + }; + let rule = match (&prev, &adopted) { + (Some(h), _) => h.m.rule(), + (None, Some((_, rule))) => rule.clone(), + (None, None) => Rule::new(proto, local_port).map_err(Error::Nonce)?, + }; + let existing = adopted + .map(|(id, _)| id) + .or_else(|| self.data.owned.live_for(&rule, net_gen)); + let existed = existing.is_some(); + let rule_id = match existing { + Some(id) => id, + None => match self + .data + .owned + .add_pending(rule.clone(), net_gen, gateway, now) + { + Some(id) => id, + None => { + if self.warn_once("ledger full".to_owned()) { + tracing::warn!( + "portmapper: too many router rules outstanding; not mapping until \ + they are deleted or expire" + ); + } + return Err(Error::LedgerFull); + } + }, + }; + let ids: HashSet<_> = self.data.owned.entries().iter().map(|e| e.id).collect(); + self.data.release_caps.retain(|id, _| ids.contains(id)); + if let Some(net) = &self.data.net { + self.data + .release_caps + .insert(rule_id, net.release_cap.clone()); + } + let evidence = self + .data + .owned + .claim_map(rule_id, now, existed) + .ok_or(Error::Busy)?; + self.data.sched.note_start(now); + let id = self.data.next_attempt_id; + self.data.next_attempt_id += 1; + let ctx = AttemptCtx { + generation: self.data.generation, + cap, + net_gen, + rule, + rule_id, + existed, + evidence, + prev: prev.map(|h| h.m), + }; + self.data.inflight = Some(Inflight { + id, + rule_id, + abort: None, + }); + Ok((id, ctx)) + } + + pub(super) fn network_validated( + &mut self, + gw: Gateway, + net_gen: NetGen, + cap: Arc, + release_cap: Arc, + ) -> bool { + let same = self + .data + .net + .as_ref() + .is_some_and(|n| n.gateway.same_network(&gw)); + if let Some(old) = self.data.net.take() { + old.cap.invalidate(); + } + if !same { + self.invalidate("network changed", Invalidation::Orphan); + self.data.probe = ProbeCache::default(); + if self.data.disabled.is_none() { + self.data.facets = ProbeFacets::default(); + } + self.data.sched.note_network_change(); + self.data.warned.clear(); + self.data.warned_router.clear(); + self.data.noted.clear(); + tracing::info!(gateway = %gw, "portmapper: gateway found or changed"); + } + // A same-network rebind restores the trusted mapping only after this fresh validation. + let ids: Vec<_> = self + .data + .owned + .entries() + .iter() + .filter(|e| e.gateway.same_network(&gw)) + .map(|e| e.id) + .collect(); + for id in ids { + self.data.release_caps.insert(id, release_cap.clone()); + } + self.data.rebind_pending = false; + self.data.net = Some(NetSnapshot { + net_gen, + cap, + release_cap, + gateway: gw, + }); + same + } + + pub(super) fn clock_aged(&mut self, now: Instant, slept: Duration) -> bool { + self.data.probe = ProbeCache::default(); + // Wall-clock movement may be a correction rather than real sleep. Age trusted + // advertisements, but never free ownership slots using that unverified evidence. + let mut changed = false; + if let Some(h) = &mut self.data.mapping { + if !slept.is_zero() { + h.lease.age(slept, now); + changed = true; + } + h.lease.renew_after = h.lease.renew_after.min(now); + } + if self + .data + .mapping + .as_ref() + .is_some_and(|h| h.lease.expired(now)) + { + self.lease_expired(now); + } + changed + } + + pub(super) fn port_changed(&mut self, new: Option) -> bool { + if self.data.local_port == new { + false + } else { + let had = self.data.local_port.is_some() + || self.data.mapping.is_some() + || self.data.inflight.is_some(); + self.data.local_port = new; + if had { + self.invalidate("local port changed", Invalidation::Orphan); + } + had + } + } + + pub(super) fn rebind_started(&mut self, same: bool) { + if same { + self.cancel_map_handle(); + } + self.data.rebind_pending = true; + if let Some(net) = &self.data.net { + net.cap.invalidate(); + } + } + + pub(super) fn policy_disabled(&mut self, why: DisableReason) -> bool { + if self.data.disabled == Some(why) { + return false; + } + self.data.disabled = Some(why); + self.invalidate("disabled", Invalidation::Orphan); + self.data.facets.pmp = Some(false); + self.data.facets.pcp = Some(false); + if self.data.owned.has_orphans() { + tracing::info!( + rules = self.data.owned.len(), + "portmapper: disabled; router rules are left until re-enabled or until they expire" + ); + } else { + tracing::info!("portmapper: disabled"); + } + true + } + + pub(super) fn policy_enabled(&mut self, policy: &PolicyHandle) -> bool { + if self.data.disabled.is_none() { + return false; + } + self.data.disabled = None; + if self.data.phase == Phase::Running { + policy.set_lan_sends_allowed(true); + } + self.data.facets = ProbeFacets::default(); + tracing::info!("portmapper: re-enabled"); + true + } + + pub(super) fn cleanup_started( + &mut self, + closing: bool, + policy: &PolicyHandle, + now: Instant, + ) -> Vec { + for e in self.data.owned.drop_expired(now) { + log_left(&e, LeftReason::Expired); + } + if !policy.lan_sends_allowed() + || self.data.phase + != if closing { + Phase::Closing + } else { + Phase::Running + } + { + return Vec::new(); + } + if closing { + self.data.owned.claim_for_close(now) + } else { + self.data.owned.claim_orphans(now) + } + } + + pub(super) fn delete_acknowledged(&mut self, id: RuleId, now: Instant) { + self.data.owned.finish_delete(id, true, now); + } + + pub(super) fn delete_unacknowledged(&mut self, id: RuleId, now: Instant) { + if let Some(gone) = self.data.owned.finish_delete(id, false, now) { + tracing::info!(internal = gone.rule.internal().get(), proto = %gone.rule.proto(), + "portmapper: rule retained; cleanup retries slowed to ten minutes"); + } + } + + pub(super) fn close_started(&mut self) -> bool { + if self.data.phase != Phase::Running { + return false; + } + self.data.phase = Phase::Closing; + self.invalidate("closing", Invalidation::Orphan); + if let Some(net) = &self.data.net { + net.cap.invalidate(); + } + true + } + + pub(super) fn close_finished(&mut self) { + if self.data.phase == Phase::Closing { + self.data.phase = Phase::Closed; + // An operation may hand back its claim just as the deadline expires. Give every + // unresolved entry the final duty, rather than testing a transient busy flag here. + // A concurrent Drop may already have consumed that duty: never rearm it. + self.data.owned.request_final_release(); + } + if !self.data.owned.is_empty() { + tracing::info!( + rules = self.data.owned.len(), + "portmapper: closed with router rules left to expire" + ); + } + } + + pub(super) fn drop_requested(&mut self) -> bool { + if self.data.phase == Phase::Closed { + return false; + } + self.data.phase = Phase::Closed; + self.invalidate("closing", Invalidation::Orphan); + if let Some(net) = &self.data.net { + net.cap.invalidate(); + } + self.data.owned.request_final_release(); + true + } +} + +impl Lifecycle { + pub(super) fn probe_observed( + &mut self, + r: &oneshot::ProbeReport, + want: ProtoSet, + protos: ProtoSet, + started: Instant, + now: Instant, + gateway: &Gateway, + ) -> bool { + let mut rolled_back = false; + let before = self.data.facets; + if want.pmp { + if r.pmp { + self.data.probe.pmp_seen_at = Some(now); + self.data.probe.pmp_public_ip = r.pmp_public_ip; + if let Some(e) = r.pmp_epoch { + rolled_back |= self.router_reset(Proto::Pmp, e, started, gateway); + } + } + self.data.facets.pmp = Some(r.pmp); + } else if protos.pmp { + self.data.facets.pmp = Some(true); + } + if want.pcp { + if r.pcp { + self.data.probe.pcp_seen_at = Some(now); + if let Some(e) = r.pcp_epoch { + rolled_back |= self.router_reset(Proto::Pcp, e, started, gateway); + } + } + self.data.facets.pcp = Some(r.pcp); + if let Some(code @ (pcp::ResultCode::NotAuthorized | pcp::ResultCode::AddressMismatch)) = + r.pcp_result + && self.warn_once_router(format!("announce {}", pcp_code_key(code))) + { + tracing::warn!(result = %code, "portmapper: PCP server will not map for us"); + } + } else if protos.pcp { + self.data.facets.pcp = Some(true); + } + self.data.probe.last_probe_at = Some(now); + self.data.probe.found_nothing = + !self.data.probe.saw_pmp(now) && !self.data.probe.saw_pcp(now); + if self.data.facets != before { + tracing::info!( + pmp = ?self.data.facets.pmp, + pcp = ?self.data.facets.pcp, + "portmapper: probe result changed" + ); + } + rolled_back + } + pub(super) fn clock_observed(&mut self, now: Instant, wall: SystemTime) -> Option { + // (whether the clocks disagree, how much further the wall clock moved) + let seen = self.data.last_clock.map(|(n0, w0)| { + let mono = now.saturating_duration_since(n0); + match wall.duration_since(w0) { + Ok(real) => (real.abs_diff(mono) > CLOCK_SKEW, real.saturating_sub(mono)), + Err(e) => (e.duration() + mono > CLOCK_SKEW, Duration::ZERO), + } + }); + self.data.last_clock = Some((now, wall)); + match seen { + Some((true, slept)) => Some(slept), + _ => None, + } + } + pub(super) fn delete_validated(&mut self, id: RuleId, closing: bool, cap: Arc) -> bool { + // A fresh lookup cannot revive a fenced entry or a closed operation. + if (!closing && self.data.phase != Phase::Running) + || !self + .data + .owned + .get(id) + .is_some_and(|r| r.releasable() && r.deleting()) + { + return false; + } + self.data.release_caps.insert(id, cap); + true + } + pub(super) fn network_lookup_started(&mut self) -> NetGen { + let g = NetGen(self.data.next_net_gen); + self.data.next_net_gen += 1; + g + } + pub(super) fn probe_cached(&mut self, now: Instant) { + self.data.probe.last_probe_at = Some(now); + } +} diff --git a/ts_portmapper/src/oneshot.rs b/ts_portmapper/src/oneshot.rs index 694e2acf..68cd2369 100644 --- a/ts_portmapper/src/oneshot.rs +++ b/ts_portmapper/src/oneshot.rs @@ -434,11 +434,33 @@ pub async fn map( rule: &Rule, lifetime: NonZeroU32, prev: Option<&Mapping>, +) -> Result { + map_inner(t, rule, lifetime, prev, None).await +} + +pub(crate) async fn map_with_evidence( + t: &T, + rule: &Rule, + lifetime: NonZeroU32, + prev: Option<&Mapping>, + evidence: &crate::SendEvidence, +) -> Result { + map_inner(t, rule, lifetime, prev, Some(evidence)).await +} + +async fn map_inner( + t: &T, + rule: &Rule, + lifetime: NonZeroU32, + prev: Option<&Mapping>, + evidence: Option<&crate::SendEvidence>, ) -> Result { let prev = prev.filter(|p| p.rule() == *rule); match rule { - Rule::Pmp { internal } => map_pmp(t, *internal, lifetime, prev).await, - Rule::Pcp { internal, nonce } => map_pcp(t, *internal, lifetime, prev, nonce.clone()).await, + Rule::Pmp { internal } => map_pmp(t, *internal, lifetime, prev, evidence).await, + Rule::Pcp { internal, nonce } => { + map_pcp(t, *internal, lifetime, prev, nonce.clone(), evidence).await + } } } @@ -447,6 +469,7 @@ async fn map_pmp( internal: NonZeroU16, lifetime: NonZeroU32, prev: Option<&Mapping>, + evidence: Option<&crate::SendEvidence>, ) -> Result { let suggested = prev.map_or(0, |p| p.external.port()); let addr_req = pmp::public_address_request(); @@ -497,7 +520,7 @@ async fn map_pmp( } }; let stats = t - .exchange(LanDest::Pxp, &reqs, deadline(), &mut on_reply) + .exchange_with_evidence(LanDest::Pxp, &reqs, deadline(), &mut on_reply, evidence) .await?; if let Some(f) = map_refused { return Err(f); @@ -531,6 +554,7 @@ async fn map_pcp( lifetime: NonZeroU32, prev: Option<&Mapping>, nonce: pcp::Nonce, + evidence: Option<&crate::SendEvidence>, ) -> Result { let suggested = prev.map_or(0, |p| p.external.port()); let req = pcp::map_request(t.gateway().self_ip, internal, suggested, lifetime, &nonce); @@ -552,7 +576,7 @@ async fn map_pcp( } }; let stats = t - .exchange(LanDest::Pxp, &[&req], deadline(), &mut on_reply) + .exchange_with_evidence(LanDest::Pxp, &[&req], deadline(), &mut on_reply, evidence) .await?; finish_notes(&stats, &mut notes); match outcome { diff --git a/ts_portmapper/src/owned.rs b/ts_portmapper/src/owned.rs new file mode 100644 index 00000000..1be2be4e --- /dev/null +++ b/ts_portmapper/src/owned.rs @@ -0,0 +1,728 @@ +//! The owned-rule ledger: every rule this process may have created on a router, so that a rule +//! whose reply was lost, or whose commit arrived after an invalidation, can still be deleted. +//! +//! An entry is added **before** each MAP is sent. A rule is enough to delete it: NAT-PMP needs +//! only the internal port, PCP also the nonce it was created with. The ledger is bounded +//! ([`MAX_OWNED`]), which bounds the router state one process can leave behind, not just its tasks. + +use std::sync::Arc; + +use tokio::time::Instant; + +use crate::{ + cap::{NetGen, SendEvidence}, + gateway::Gateway, + oneshot::{Proto, Rule}, +}; + +/// The most rules one client tracks at once. New attempts are refused while it is full. +pub(crate) const MAX_OWNED: usize = 4; +/// Initial unacknowledged deletes before switching to infrequent recovery attempts. +pub(crate) const MAX_DELETE_TRIES: u32 = 3; +/// After the initial attempts, at most one delete per rule per ten minutes, with fresh gateway +/// validation each time. Unknown grants never age out at the requested lifetime: they keep an +/// admission slot until per-rule evidence (a matched reply/delete or accepted reset) resolves it. +pub(crate) const DELETE_RETRY_INTERVAL: std::time::Duration = std::time::Duration::from_secs(600); + +/// Where an owned rule stands. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum RuleState { + /// A MAP for it was (or is about to be) sent; no reply committed yet. + Pending, + /// A committed reply backs the current mapping. + Active, + /// No longer wanted: to be deleted (when allowed) or left to router expiry. + Orphaned, +} + +/// Opaque handle to a ledger entry. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub(crate) struct RuleId(u64); + +/// One rule this process may own on a router. +#[derive(Debug, Clone)] +pub(crate) struct OwnedRule { + /// Handle. + pub(crate) id: RuleId, + /// What it takes to delete it. + pub(crate) rule: Rule, + /// The network it was created on. + pub(crate) net_gen: NetGen, + /// That network's gateway, self address and interface. + pub(crate) gateway: Gateway, + ownership: Ownership, + release: Release, + /// Admission time, used only by the independent ownership journal. + #[cfg(test)] + pub(crate) sent_at: Instant, + /// Prior settled possible ownership. A live MAP can expand this bound only with syscall + /// evidence; a matched grant replaces it. Fresh unsent reservations start with no grant. + settled_expiry: Instant, + /// Deletes already sent without an acknowledgement. + pub(crate) delete_tries: u32, + /// Recovery/validation cooldown, independent of possible router expiry. + pub(crate) retry_at: Option, +} + +/// Reservations are exclusive by construction; a delete can only own an orphan. +#[derive(Debug, Clone)] +enum Ownership { + Idle(RuleState), + Map(RuleState, MapGrant), + Delete, +} + +#[derive(Debug, Clone)] +enum MapGrant { + Pending { + evidence: Arc, + existed: bool, + }, + Settled, +} + +#[derive(Debug, Clone, Copy)] +enum FinalDuty { + None, + Owed, +} + +#[derive(Debug, Clone, Copy)] +enum Release { + Allowed(FinalDuty), + Fenced(FinalDuty), +} + +impl Release { + fn duty(self) -> FinalDuty { + match self { + Self::Allowed(d) | Self::Fenced(d) => d, + } + } +} + +impl OwnedRule { + pub(crate) fn state(&self) -> RuleState { + match &self.ownership { + Ownership::Idle(s) | Ownership::Map(s, _) => *s, + Ownership::Delete => RuleState::Orphaned, + } + } + + pub(crate) fn mapping(&self) -> bool { + matches!(self.ownership, Ownership::Map(..)) + } + pub(crate) fn deleting(&self) -> bool { + matches!(self.ownership, Ownership::Delete) + } + pub(crate) fn releasable(&self) -> bool { + matches!(self.release, Release::Allowed(_)) + } + fn reserved(&self) -> bool { + self.mapping() || self.deleting() + } + fn set_state(&mut self, state: RuleState) { + match &mut self.ownership { + Ownership::Map(s, _) | Ownership::Idle(s) => *s = state, + Ownership::Delete => debug_assert_eq!(state, RuleState::Orphaned), + } + } + + /// A live reservation owns no additional grant until its MAP syscall may have run. + pub(crate) fn router_expiry(&self) -> Instant { + match &self.ownership { + Ownership::Map(_, MapGrant::Pending { evidence, .. }) => evidence + .possible_until() + .map_or(self.settled_expiry, |until| self.settled_expiry.max(until)), + _ => self.settled_expiry, + } + } + + fn grant_observed(&mut self, until: Instant) { + self.settled_expiry = until; + if let Ownership::Map(_, grant) = &mut self.ownership { + *grant = MapGrant::Settled; + } + } + + fn unclaim_delete(&mut self) { + if self.deleting() { + self.ownership = Ownership::Idle(RuleState::Orphaned); + } + } + fn fence(&mut self) { + self.release = Release::Fenced(self.release.duty()); + } +} + +/// Why a rule left the ledger without a delete. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum LeftReason { + /// Its network no longer validates, or is down: a delete could reach the wrong router. + NetworkGone, + /// The router's grant already ended. + Expired, +} + +/// The ledger. See the module docs. +#[derive(Debug, Default)] +pub(crate) struct OwnedRules { + entries: Vec, + next_id: u64, + #[cfg(test)] + pub(crate) model: crate::ownership_model::OwnershipModel, +} + +// Prefix length and interface name are validation metadata, not fields on a router's UDP +// mapping key. A rename/prefix update must not split a live operation's reservation. +fn same_wire_gateway(a: &Gateway, b: &Gateway) -> bool { + a.gw == b.gw && a.self_ip == b.self_ip && a.if_index == b.if_index +} + +impl OwnedRules { + /// All entries. + pub(crate) fn entries(&self) -> &[OwnedRule] { + &self.entries + } + + /// Number of entries in any state. + pub(crate) fn len(&self) -> usize { + self.entries.len() + } + + /// Whether the ledger is empty. + pub(crate) fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + /// Whether a new entry fits. + pub(crate) fn has_room(&self) -> bool { + self.entries.len() < MAX_OWNED + } + + fn find(&self, id: RuleId) -> Option { + self.entries.iter().position(|e| e.id == id) + } + + /// The entry with this id. + pub(crate) fn get(&self, id: RuleId) -> Option<&OwnedRule> { + self.find(id).map(|i| &self.entries[i]) + } + + /// The live (pending or active) entry for exactly this rule on this network, if any: a renewal + /// reuses it in place. + pub(crate) fn live_for(&self, rule: &Rule, net_gen: NetGen) -> Option { + self.entries + .iter() + .find(|e| e.state() != RuleState::Orphaned && e.net_gen == net_gen && e.rule == *rule) + .map(|e| e.id) + } + + /// The UDP wire identity, independent of protocol, nonce and local network generation. + /// A router may share one table between PCP and PMP. + pub(crate) fn for_identity( + &self, + internal: std::num::NonZeroU16, + gateway: &Gateway, + ) -> Option<&OwnedRule> { + self.entries + .iter() + .find(|e| e.rule.internal() == internal && same_wire_gateway(&e.gateway, gateway)) + } + + /// Reserve a MAP through completion/cancellation. All callers hold the client state lock. + pub(crate) fn claim_map( + &mut self, + id: RuleId, + now: Instant, + existed: bool, + ) -> Option> { + let i = self.find(id)?; + let e = &mut self.entries[i]; + if e.reserved() { + return None; + } + let evidence = Arc::new(SendEvidence::new( + if e.rule.proto() == Proto::Pmp { 1 } else { 0 }, + now, + )); + e.ownership = Ownership::Map( + e.state(), + MapGrant::Pending { + evidence: evidence.clone(), + existed, + }, + ); + #[cfg(test)] + self.model.map_started(id, evidence.clone()); + Some(evidence) + } + + /// Settle only after the transport future ended or was dropped, so its evidence cannot + /// change concurrently. Local absence affects this attempt only; retain prior ownership. + pub(crate) fn discard_unsent(&mut self, id: RuleId) { + let Some(i) = self.find(id) else { + return; + }; + let Ownership::Map(_, MapGrant::Pending { evidence, existed }) = &self.entries[i].ownership + else { + return; + }; + if evidence.possible_until().is_some() { + return; + } + if *existed { + if let Ownership::Map(_, grant) = &mut self.entries[i].ownership { + *grant = MapGrant::Settled; + } + } else { + #[cfg(test)] + self.record_retirement(id, crate::ownership_model::Retirement::ProvenUnsent); + self.entries.remove(i); + } + } + + /// Release a completed/cancelled MAP. Retire fresh never-sent identities; otherwise return + /// unsettled entries to cleanup, materializing any attempted MAP's possible grant. Active + /// renewals retain their held mapping and no unsent attempt extends ownership. + pub(crate) fn finish_map(&mut self, id: RuleId) { + self.discard_unsent(id); + if let Some(i) = self.find(id) { + let e = &mut self.entries[i]; + if e.mapping() { + e.settled_expiry = e.router_expiry(); + e.ownership = Ownership::Idle(match e.state() { + RuleState::Pending => RuleState::Orphaned, + s => s, + }); + } + } + } + + /// Reserve an identity before I/O. No router grant is recorded until a MAP syscall is + /// attempted. Returns `None` when the ledger is full. + pub(crate) fn add_pending( + &mut self, + rule: Rule, + net_gen: NetGen, + gateway: Gateway, + now: Instant, + ) -> Option { + if !self.has_room() || self.for_identity(rule.internal(), &gateway).is_some() { + return None; + } + let id = RuleId(self.next_id); + self.next_id += 1; + self.entries.push(OwnedRule { + id, + rule, + net_gen, + gateway, + ownership: Ownership::Idle(RuleState::Pending), + release: Release::Allowed(FinalDuty::None), + #[cfg(test)] + sent_at: now, + settled_expiry: now, + delete_tries: 0, + retry_at: None, + }); + #[cfg(test)] + self.model.admit(self.entries.last().unwrap().clone()); + Some(id) + } + + /// A reply committed for `id`: it is active until `router_expiry`. + pub(crate) fn commit(&mut self, id: RuleId, router_expiry: Instant) { + if let Some(i) = self.find(id) { + let e = &mut self.entries[i]; + e.set_state(RuleState::Active); + e.grant_observed(router_expiry); + } + } + + /// A reply arrived but is stale: the router holds the rule until `router_expiry`, and nobody + /// wants it. + pub(crate) fn orphan_committed(&mut self, id: RuleId, router_expiry: Instant) { + if let Some(i) = self.find(id) { + let e = &mut self.entries[i]; + e.set_state(RuleState::Orphaned); + e.grant_observed(router_expiry); + } + } + + /// Mark one entry orphaned. + pub(crate) fn orphan(&mut self, id: RuleId) { + if let Some(i) = self.find(id) { + self.entries[i].set_state(RuleState::Orphaned); + } + } + + /// Mark every pending and active entry orphaned (an invalidation: nothing is in use any more). + pub(crate) fn orphan_all(&mut self) { + for e in &mut self.entries { + e.set_state(RuleState::Orphaned); + } + } + + /// Remove `id` without a delete (the router refused the create, or lost the rule). + pub(crate) fn remove(&mut self, id: RuleId) -> Option { + self.find(id).map(|i| self.entries.remove(i)) + } + + #[cfg(test)] + pub(crate) fn record_retirement( + &mut self, + id: RuleId, + reason: crate::ownership_model::Retirement, + ) { + if let Some(e) = self.get(id).cloned() { + self.model.retire(e, reason); + } + } + + /// Mark every orphaned entry as never to be deleted (a network-down); returns those that + /// changed, for logging. + pub(crate) fn forget_orphans(&mut self) -> Vec { + let mut out = Vec::new(); + for e in &mut self.entries { + if e.state() == RuleState::Orphaned && e.releasable() { + e.fence(); + out.push(e.clone()); + } + } + out + } + + /// An orphaned entry for the same protocol and internal port on the same network + /// (`gateway`), not being deleted: a new create adopts it (and, for PCP, its nonce) instead of + /// asking for a second rule. It becomes pending again, on `net_gen`. + pub(crate) fn adopt( + &mut self, + proto: Proto, + internal: std::num::NonZeroU16, + gateway: &Gateway, + net_gen: NetGen, + ) -> Option<(RuleId, Rule)> { + let e = self.entries.iter_mut().find(|e| { + e.state() == RuleState::Orphaned + && !e.reserved() + && e.rule.proto() == proto + && e.rule.internal() == internal + && same_wire_gateway(&e.gateway, gateway) + })?; + e.set_state(RuleState::Pending); + e.release = Release::Allowed(e.release.duty()); + e.delete_tries = 0; + e.retry_at = None; + e.net_gen = net_gen; + e.gateway = gateway.clone(); + Some((e.id, e.rule.clone())) + } + + /// Remove this reporting router's entries for `proto` on an accepted epoch rollback, + /// except `keep`. A different gateway's reset says nothing about this rule's ownership: + /// the rule of an attempt in flight, whose MAP may reach the restarted router and be committed + /// there. It stays, so an invalidation orphans it and a cleanup deletes it. + pub(crate) fn drop_proto( + &mut self, + proto: Proto, + gateway: &Gateway, + keep: Option, + ) -> Vec { + let (gone, kept): (Vec, Vec) = std::mem::take(&mut self.entries) + .into_iter() + .partition(|e| { + e.rule.proto() == proto + && same_wire_gateway(&e.gateway, gateway) + && !e.reserved() + && Some(e.id) != keep + }); + #[cfg(test)] + for e in &gone { + self.model.retire( + e.clone(), + crate::ownership_model::Retirement::Reset { + observation: self.model.latest_reset(), + gateway: gateway.clone(), + proto, + }, + ); + } + self.entries = kept; + gone + } + + /// Remove every orphaned entry whose router grant has ended by `now`. + pub(crate) fn drop_expired(&mut self, now: Instant) -> Vec { + let (gone, keep): (Vec, Vec) = std::mem::take(&mut self.entries) + .into_iter() + .partition(|e| { + e.state() == RuleState::Orphaned && !e.reserved() && now >= e.router_expiry() + }); + #[cfg(test)] + for e in &gone { + self.model.retire( + e.clone(), + crate::ownership_model::Retirement::Expiry { observed_at: now }, + ); + } + self.entries = keep; + gone + } + + /// Releasable orphaned entries that are not being deleted yet; each is marked as being + /// deleted. + pub(crate) fn claim_orphans(&mut self, now: Instant) -> Vec { + self.claim_orphans_inner(now, false) + } + + /// Shutdown gets one final attempt, even when ordinary recovery is cooling down. + pub(crate) fn claim_for_close(&mut self, now: Instant) -> Vec { + self.claim_orphans_inner(now, true) + } + + fn claim_orphans_inner(&mut self, now: Instant, closing: bool) -> Vec { + let mut out = Vec::new(); + for e in &mut self.entries { + if e.state() == RuleState::Orphaned + && e.releasable() + && !e.reserved() + && (closing || e.retry_at.is_none_or(|at| now >= at)) + { + e.ownership = Ownership::Delete; + // Reserve the recovery interval before awaiting OS work or sending, so + // cancellation cannot bypass its spacing. + if e.delete_tries >= MAX_DELETE_TRIES { + e.retry_at = Some(now + DELETE_RETRY_INTERVAL); + } + out.push(e.clone()); + } + } + out + } + + /// Transfer final-release duty to each operation (or claim idle entries immediately). + pub(crate) fn request_final_release(&mut self) { + for e in &mut self.entries { + e.release = match e.release { + Release::Allowed(_) => Release::Allowed(FinalDuty::Owed), + Release::Fenced(_) => Release::Fenced(FinalDuty::Owed), + }; + } + } + + /// Any operation still holds a reservation. + pub(crate) fn has_claims(&self) -> bool { + self.entries.iter().any(|e| e.reserved()) + } + + /// Claim one final best-effort send per rule, independent of normal recovery cooldowns. + /// Existing operations keep their reservations until their finalizers hand them back. + pub(crate) fn claim_final_releases(&mut self, now: Instant) -> Vec { + let mut out = Vec::new(); + for e in &mut self.entries { + if matches!(e.release.duty(), FinalDuty::Owed) + && e.releasable() + && !e.reserved() + && e.state() == RuleState::Orphaned + && now < e.router_expiry() + { + e.release = Release::Allowed(FinalDuty::None); + e.ownership = Ownership::Delete; + out.push(e.clone()); + } + } + out + } + + /// Stop deleting an unreachable rule, retaining its admission slot until expiry. + pub(crate) fn abandon(&mut self, id: RuleId) { + if let Some(i) = self.find(id) { + self.entries[i].fence(); + self.entries[i].unclaim_delete(); + } + } + + /// A delete finished. Only an acknowledgement retires ownership. After the initial + /// attempts, keep the admission slot and try once per recovery interval, indefinitely. + /// Returns an entry just once when its fast attempts are exhausted (for logging). + pub(crate) fn finish_delete( + &mut self, + id: RuleId, + acked: bool, + now: Instant, + ) -> Option { + let i = self.find(id)?; + if acked { + #[cfg(test)] + self.record_retirement( + id, + crate::ownership_model::Retirement::Delete { matched: acked }, + ); + self.entries.remove(i); + return None; + } + let e = &mut self.entries[i]; + e.unclaim_delete(); + e.delete_tries = e.delete_tries.saturating_add(1); + if e.delete_tries >= MAX_DELETE_TRIES { + e.retry_at = Some(now + DELETE_RETRY_INTERVAL); + } + (e.delete_tries == MAX_DELETE_TRIES).then(|| e.clone()) + } + + /// Validation failed or its worker timed out. Keep ownership and allow another fresh + /// validation later; an unavailable worker is not evidence that the network is gone. + pub(crate) fn defer_delete(&mut self, id: RuleId, now: Instant) { + if let Some(i) = self.find(id) { + let e = &mut self.entries[i]; + e.unclaim_delete(); + e.retry_at = Some(now + DELETE_RETRY_INTERVAL); + } + } + + /// Earliest cleanup worth waking for, independent of node activity or peers. Network-down + /// rules and reservations owned by other operations cannot be sent by this timer. + pub(crate) fn next_cleanup(&self, now: Instant) -> Option { + self.entries + .iter() + .filter(|e| e.state() == RuleState::Orphaned && e.releasable() && !e.reserved()) + .map(|e| e.retry_at.unwrap_or(now)) + .min() + } + + /// Give a claimed entry back without a delete attempt. + pub(crate) fn unclaim(&mut self, id: RuleId) { + if let Some(i) = self.find(id) { + self.entries[i].unclaim_delete(); + } + } + + /// Whether any entry is orphaned and may be deleted. + pub(crate) fn has_orphans(&self) -> bool { + self.entries + .iter() + .any(|e| e.state() == RuleState::Orphaned && e.releasable()) + } +} + +#[cfg(test)] +mod tests { + use std::{num::NonZeroU16, time::Duration}; + + use crate::oneshot::Proto; + + use super::*; + + fn gw() -> Gateway { + Gateway { + gw: "192.168.1.1".parse().unwrap(), + self_ip: "192.168.1.20".parse().unwrap(), + prefix_len: 24, + if_index: 4, + if_name: "en0".to_owned(), + } + } + + fn pmp(port: u16) -> Rule { + Rule::Pmp { + internal: NonZeroU16::new(port).unwrap(), + } + } + + #[test] + fn bounded_and_renewals_reuse_the_entry() { + let now = Instant::now(); + let mut l = OwnedRules::default(); + let a = l.add_pending(pmp(1), NetGen(1), gw(), now).unwrap(); + assert_eq!(l.live_for(&pmp(1), NetGen(1)), Some(a)); + assert_eq!(l.live_for(&pmp(1), NetGen(2)), None); + assert_eq!(l.live_for(&pmp(2), NetGen(1)), None); + for p in 2..=4 { + assert!(l.add_pending(pmp(p), NetGen(1), gw(), now).is_some()); + } + assert!(!l.has_room()); + assert!(l.add_pending(pmp(5), NetGen(1), gw(), now).is_none()); + assert_eq!(l.len(), MAX_OWNED); + } + + #[test] + fn orphans_are_claimed_once_and_retried_a_bounded_number_of_times() { + let now = Instant::now(); + let mut l = OwnedRules::default(); + let a = l.add_pending(pmp(1), NetGen(1), gw(), now).unwrap(); + l.commit(a, now + Duration::from_secs(60)); + assert!(l.claim_orphans(now).is_empty()); + l.orphan_all(); + assert_eq!(l.claim_orphans(now).len(), 1); + // Claimed: a second cleanup does not see it. + assert!(l.claim_orphans(now).is_empty()); + assert!(l.finish_delete(a, false, now).is_none()); + for _ in 1..MAX_DELETE_TRIES - 1 { + assert_eq!(l.claim_orphans(now).len(), 1); + assert!(l.finish_delete(a, false, now).is_none()); + } + assert_eq!(l.claim_orphans(now).len(), 1); + assert!(l.finish_delete(a, false, now).is_some()); + assert_eq!(l.len(), 1, "unacknowledged rule remains a tombstone"); + assert!(l.claim_orphans(now).is_empty()); + assert_eq!(l.drop_expired(now + Duration::from_secs(60)).len(), 1); + + let b = l.add_pending(pmp(2), NetGen(1), gw(), now).unwrap(); + l.orphan(b); + l.claim_orphans(now); + assert!(l.finish_delete(b, true, now).is_none()); + assert!(l.is_empty()); + } + + #[test] + fn forgotten_rules_are_never_claimed_but_can_be_adopted_and_count() { + let now = Instant::now(); + let mut l = OwnedRules::default(); + let a = l.add_pending(pmp(1), NetGen(1), gw(), now).unwrap(); + l.orphan_all(); + assert_eq!(l.forget_orphans().len(), 1); + assert!(!l.has_orphans()); + assert!(l.claim_orphans(now).is_empty()); + // Same rule, same network, a later generation: adopted. + let p1 = NonZeroU16::new(1).unwrap(); + let mut other = gw(); + other.gw = "10.0.0.1".parse().unwrap(); + assert!(l.adopt(Proto::Pmp, p1, &other, NetGen(2)).is_none()); + assert!(l.adopt(Proto::Pcp, p1, &gw(), NetGen(2)).is_none()); + let mut refreshed = gw(); + refreshed.prefix_len = 23; + refreshed.if_name = "renamed-interface".to_owned(); + let (id, rule) = l.adopt(Proto::Pmp, p1, &refreshed, NetGen(2)).unwrap(); + assert_eq!( + l.get(id).unwrap().gateway, + refreshed, + "adoption uses the fresh validated snapshot" + ); + assert_eq!((id, rule), (a, pmp(1))); + assert_eq!(l.live_for(&pmp(1), NetGen(2)), Some(a)); + // Forgotten rules still count against admission until expiry. + l.orphan_all(); + l.forget_orphans(); + for p in 2..=4 { + l.add_pending(pmp(p), NetGen(2), gw(), now + Duration::from_secs(1)) + .unwrap(); + } + assert!(l.add_pending(pmp(5), NetGen(2), gw(), now).is_none()); + assert!(l.get(a).is_some()); + assert!(l.add_pending(pmp(6), NetGen(2), gw(), now).is_none()); + } + + #[test] + fn stale_commit_records_the_grant() { + let now = Instant::now(); + let mut l = OwnedRules::default(); + let a = l.add_pending(pmp(1), NetGen(1), gw(), now).unwrap(); + l.orphan_committed(a, now + Duration::from_secs(86_400)); + let e = l.get(a).unwrap(); + assert_eq!(e.state(), RuleState::Orphaned); + assert_eq!(e.router_expiry(), now + Duration::from_secs(86_400)); + assert!(l.drop_expired(now + Duration::from_secs(60)).is_empty()); + assert_eq!(l.drop_expired(now + Duration::from_secs(86_400)).len(), 1); + } +} diff --git a/ts_portmapper/src/ownership_model.rs b/ts_portmapper/src/ownership_model.rs new file mode 100644 index 00000000..e9c33c01 --- /dev/null +++ b/ts_portmapper/src/ownership_model.rs @@ -0,0 +1,215 @@ +//! Test-only conservation model. Admission and retirement are separate observations: removing +//! an entry cannot manufacture its own evidence. Fenced/terminal unacknowledged orphans remain +//! possible ownership and count against admission; sending a final delete is not retirement. + +use std::{ + collections::{HashMap, HashSet}, + sync::Arc, +}; + +use tokio::time::Instant; + +use crate::{ + Gateway, + oneshot::Proto, + owned::{OwnedRule, RuleId}, +}; + +#[derive(Debug, Clone)] +pub(crate) enum RefusalEvidence { + MatchedMap(Proto), + Unproven, +} + +#[derive(Debug, Clone, Copy)] +pub(crate) struct ResetId(usize); + +impl ResetId { + pub(crate) const MISSING: Self = Self(usize::MAX); +} + +#[derive(Debug, Clone)] +pub(crate) enum Retirement { + /// A matched refusal of the MAP request. + Refusal(RefusalEvidence), + /// Every recorded MAP attempt for a fresh identity stopped before its syscall boundary. + ProvenUnsent, + Delete { + matched: bool, + }, + Expiry { + observed_at: Instant, + }, + Reset { + observation: ResetId, + gateway: Gateway, + proto: Proto, + }, +} + +#[derive(Debug)] +struct AcceptedReset { + sequence: usize, + gateway: Gateway, + proto: Proto, + previous: u32, + reported: u32, + committed: Instant, + probe_started: Instant, +} + +#[derive(Debug, Default)] +pub(crate) struct OwnershipModel { + admitted: HashMap, + retired: HashMap, + last_map: HashMap, + sequence: usize, + attempts: HashMap>>, + resets: Vec, +} + +fn wire_gateway(a: &Gateway, b: &Gateway) -> bool { + a.gw == b.gw && a.self_ip == b.self_ip && a.if_index == b.if_index +} + +impl OwnershipModel { + pub(crate) fn admit(&mut self, entry: OwnedRule) { + self.sequence += 1; + self.last_map.insert(entry.id, self.sequence); + assert!( + self.admitted.insert(entry.id, entry).is_none(), + "reused admission id" + ); + } + + pub(crate) fn map_started(&mut self, id: RuleId, evidence: Arc) { + self.attempts.entry(id).or_default().push(evidence); + self.sequence += 1; + self.last_map.insert(id, self.sequence); + } + + pub(crate) fn latest_reset(&self) -> ResetId { + self.resets + .len() + .checked_sub(1) + .map(ResetId) + .unwrap_or(ResetId::MISSING) + } + + pub(crate) fn retire(&mut self, entry: OwnedRule, evidence: Retirement) { + self.sequence += 1; + assert!( + self.retired + .insert(entry.id, (entry, evidence, self.sequence)) + .is_none(), + "double retirement" + ); + } + + pub(crate) fn accept_reset( + &mut self, + gateway: Gateway, + proto: Proto, + previous: u32, + reported: u32, + committed: Instant, + probe_started: Instant, + ) -> ResetId { + self.sequence += 1; + let id = ResetId(self.resets.len()); + self.resets.push(AcceptedReset { + sequence: self.sequence, + gateway, + proto, + previous, + reported, + committed, + probe_started, + }); + id + } + + pub(crate) fn check(&self, entries: &[OwnedRule]) { + let live: HashSet<_> = entries.iter().map(|e| e.id).collect(); + assert_eq!(live.len(), entries.len(), "duplicate live id"); + assert_eq!( + self.admitted.len(), + live.len() + self.retired.len(), + "possible ownership disappeared without retirement evidence" + ); + for (id, admitted) in &self.admitted { + let entry = if let Some(entry) = entries.iter().find(|e| e.id == *id) { + assert!( + !self.retired.contains_key(id), + "retired entry is still live" + ); + entry + } else { + let (entry, reason, retired_sequence) = + self.retired.get(id).expect("unexplained retirement"); + match reason { + Retirement::Refusal(evidence) => match evidence { + RefusalEvidence::MatchedMap(proto) => { + assert_eq!(*proto, entry.rule.proto(), "foreign MAP refusal") + } + RefusalEvidence::Unproven => panic!("unmatched refusal"), + }, + Retirement::ProvenUnsent => { + let attempts = self + .attempts + .get(id) + .expect("unsent retirement has no attempt evidence"); + assert!( + !attempts.is_empty() + && attempts.iter().all(|e| e.possible_until().is_none()), + "attempted MAP cannot retire as proven unsent" + ); + } + Retirement::Delete { matched } => assert!(*matched, "unacknowledged delete"), + Retirement::Expiry { observed_at } => assert!( + entry.router_expiry() <= *observed_at, + "requested lifetime/trust expiry is not proven router expiry" + ), + Retirement::Reset { + observation, + gateway, + proto, + } => { + assert!( + wire_gateway(gateway, &entry.gateway) && *proto == entry.rule.proto(), + "foreign router/protocol reset cannot retire this entry" + ); + let r = self + .resets + .get(observation.0) + .expect("unknown reset observation"); + assert!( + wire_gateway(&r.gateway, gateway) + && r.proto == *proto + && r.reported > 0 + && r.reported < r.previous + && r.committed <= r.probe_started + && admitted.sent_at <= r.probe_started + && self.last_map[id] <= r.sequence + && r.sequence < *retired_sequence, + "reset observation falls outside this ownership lifetime" + ); + } + } + entry + }; + assert_eq!( + admitted.rule, entry.rule, + "ownership changed protocol/nonce in place" + ); + assert!( + wire_gateway(&admitted.gateway, &entry.gateway), + "ownership moved to another wire gateway" + ); + } + assert!( + self.retired.keys().all(|id| self.admitted.contains_key(id)), + "retired unknown id" + ); + } +} diff --git a/ts_portmapper/src/schedule.rs b/ts_portmapper/src/schedule.rs new file mode 100644 index 00000000..939803c2 --- /dev/null +++ b/ts_portmapper/src/schedule.rs @@ -0,0 +1,354 @@ +//! Retry spacing, renewal and expiry for the lifecycle client: pure arithmetic over instants, so +//! every rule here is unit-tested with a manual clock. +//! +//! Three instants are kept apart on purpose: +//! +//! - [`Schedule::next_attempt_at`]: the earliest time **any** create or renew may start, whatever +//! the trigger (driver timer, kick, rebind, ReSTUN, `get_cached…`). Failures push it out by a +//! jittered backoff; router refusals push it further. +//! - [`Lease::renew_after`]: when a healthy mapping should be renewed. +//! - [`Lease::expires_at`]: the end of local trust. After it the mapping is dropped and its endpoint +//! withdrawn, whether or not anything else happens. +//! +//! [`Lease::router_expiry`] records what the router actually granted, which may be longer than we +//! trust (PCP may grant more than requested). + +use std::time::Duration; + +use tokio::time::Instant; + +use crate::oneshot::lease_deadline; + +/// The lifetime requested for every mapping (Go's `pmpMapLifetimeSec` / `pcpMapLifetimeSec`). +pub const REQUESTED_LIFETIME: u32 = 7200; +/// The longest lease trusted locally, whatever the router grants. +pub const MAX_TRUSTED: Duration = Duration::from_secs(7200); +/// Bounds on the renewal point, as a delay after the commit. +pub const MIN_RENEW_DELAY: Duration = Duration::from_secs(10); +/// See [`MIN_RENEW_DELAY`]. +pub const MAX_RENEW_DELAY: Duration = Duration::from_secs(3600); +/// Backoff after consecutive failures; the last step is the cap. +pub const BACKOFF_STEPS: [Duration; 6] = [ + Duration::from_secs(10), + Duration::from_secs(30), + Duration::from_secs(60), + Duration::from_secs(120), + Duration::from_secs(300), + Duration::from_secs(600), +]; +/// The least time between any two attempt starts, even across a network change. +pub const ATTEMPT_FLOOR: Duration = Duration::from_secs(2); +/// How long a "not authorized" refusal holds off the next attempt. +pub const NOT_AUTHORIZED_HOLD: Duration = Duration::from_secs(30 * 60); +/// The longest hold taken from a PCP error lifetime (RFC 6887 §7.4, §11.4). +pub const MAX_ERROR_HOLD: Duration = Duration::from_secs(30 * 60); + +/// A mapping's lease, computed once at commit. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Lease { + /// When the router's reply was committed. + pub committed_at: Instant, + /// The lifetime the router granted, in seconds (never 0 for a live mapping). + pub granted: u32, + /// `min(granted, MAX_TRUSTED)`. + pub trusted: Duration, + /// When to renew: `committed_at + clamp(trusted / 2, 10 s, 3600 s)`. + pub renew_after: Instant, + /// When local trust ends: `committed_at + trusted`. + pub expires_at: Instant, + /// When the router's grant ends: `committed_at + granted` (the full wire lifetime). + pub router_expiry: Instant, +} + +impl Lease { + /// The lease for a mapping granted `granted` seconds at `now`. + pub fn new(now: Instant, granted: u32) -> Self { + let trusted = Duration::from_secs(u64::from(granted)).min(MAX_TRUSTED); + let renew_delay = (trusted / 2).clamp(MIN_RENEW_DELAY, MAX_RENEW_DELAY); + let at = |d: Duration| now.checked_add(d).unwrap_or(now); + Self { + committed_at: now, + granted, + trusted, + // A lease shorter than 20 s would renew at or after its own expiry; renewal still + // starts no later than expiry. + renew_after: at(renew_delay.min(trusted)), + expires_at: at(trusted), + router_expiry: lease_deadline(now, granted), + } + } + + /// Whether local trust has ended at `now`. + pub fn expired(&self, now: Instant) -> bool { + now >= self.expires_at + } + + /// Whether the mapping is due for renewal at `now`. + pub fn renew_due(&self, now: Instant) -> bool { + now >= self.renew_after + } + + /// The host slept for `by` (or the wall clock jumped forward) and it is now `now`: the + /// router's clock kept running while [`Instant`] did not, so the renewal point, local trust and + /// the router's grant all move earlier by `by`. `committed_at` stays: it orders events on the + /// monotonic clock. + pub fn age(&mut self, by: Duration, now: Instant) { + self.renew_after = earlier(self.renew_after, by, now); + self.expires_at = earlier(self.expires_at, by, now); + self.router_expiry = earlier(self.router_expiry, by, now); + } +} + +/// `t - by`. When that is not representable it lies before the clock's origin, so before `now`, +/// and `now` stands in for it (every "has it passed" comparison then holds). +pub(crate) fn earlier(t: Instant, by: Duration, now: Instant) -> Instant { + t.checked_sub(by).unwrap_or(now) +} + +/// Why an attempt failed, for spacing the next one. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Failure { + /// Timeout, no responder, no services, an OS error or a malformed exchange. + Transient, + /// NAT-PMP result 2 or PCP `NOT_AUTHORIZED`: hold off for at least [`NOT_AUTHORIZED_HOLD`]. + NotAuthorized, + /// Another router refusal; `hold` is a PCP error lifetime, if any. + Refused { + /// How long the router says the error stays current. + hold: Option, + }, +} + +/// Attempt spacing. See the module docs. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Schedule { + /// The earliest next attempt start, or `None` for "now". + pub next_attempt_at: Option, + /// When the last attempt started (for the [`ATTEMPT_FLOOR`]). + pub last_start: Option, + /// Consecutive failures since the last success or network change. + pub failures: u32, +} + +/// The backoff after `n` consecutive failures (`n ≥ 1`), before jitter. +pub fn backoff(n: u32) -> Duration { + let i = usize::try_from(n.saturating_sub(1)).unwrap_or(usize::MAX); + BACKOFF_STEPS[i.min(BACKOFF_STEPS.len() - 1)] +} + +/// Scale `d` by `jitter`, clamped to the ±20 % band. +pub fn jittered(d: Duration, jitter: f64) -> Duration { + d.mul_f64(jitter.clamp(0.8, 1.2)) +} + +impl Schedule { + /// The earliest instant an attempt may start: `max(next_attempt_at, last_start + floor)`. + pub fn earliest_start(&self) -> Option { + let floor = self.last_start.map(|t| t + ATTEMPT_FLOOR); + match (self.next_attempt_at, floor) { + (Some(a), Some(b)) => Some(a.max(b)), + (a, b) => a.or(b), + } + } + + /// Whether an attempt may start at `now`. + pub fn may_start(&self, now: Instant) -> bool { + self.earliest_start().is_none_or(|t| now >= t) + } + + /// Record an attempt start. + pub fn note_start(&mut self, now: Instant) { + self.last_start = Some(now); + } + + /// Record a success: the failure count resets and nothing holds off the next attempt but + /// the floor. + pub fn note_success(&mut self) { + self.failures = 0; + self.next_attempt_at = None; + } + + /// Record a failure at `now`; `jitter` is a factor in `[0.8, 1.2]`. + pub fn note_failure(&mut self, now: Instant, why: Failure, jitter: f64) { + self.failures = self.failures.saturating_add(1); + let mut hold = jittered(backoff(self.failures), jitter); + match why { + Failure::Transient => {} + Failure::NotAuthorized => hold = hold.max(NOT_AUTHORIZED_HOLD), + Failure::Refused { hold: Some(h) } => hold = hold.max(h.min(MAX_ERROR_HOLD)), + Failure::Refused { hold: None } => {} + } + self.next_attempt_at = Some(now.checked_add(hold).unwrap_or(now)); + } + + /// A network change: the failure count and any hold reset, the floor stays. + pub fn note_network_change(&mut self) { + self.failures = 0; + self.next_attempt_at = None; + } + + /// When the driver should try a renewal: `max(renew_after, earliest_start)`, never a bare + /// past instant that would make it spin. + pub fn renewal_due(&self, lease: &Lease) -> Instant { + self.earliest_start() + .map_or(lease.renew_after, |t| t.max(lease.renew_after)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn secs(n: u64) -> Duration { + Duration::from_secs(n) + } + + #[test] + fn lease_uses_min_of_granted_and_two_hours() { + let t0 = Instant::now(); + let l = Lease::new(t0, 7200); + assert_eq!(l.trusted, secs(7200)); + assert_eq!(l.renew_after, t0 + secs(3600)); + assert_eq!(l.expires_at, t0 + secs(7200)); + assert_eq!(l.router_expiry, t0 + secs(7200)); + + // Longer than requested: trust stays 2 h, the router's bound is recorded. + let l = Lease::new(t0, 86_400); + assert_eq!(l.trusted, secs(7200)); + assert_eq!(l.expires_at, t0 + secs(7200)); + assert_eq!(l.router_expiry, t0 + secs(86_400)); + + // Shorter: renew at half, at least 10 s. + let l = Lease::new(t0, 60); + assert_eq!(l.renew_after, t0 + secs(30)); + assert_eq!(l.expires_at, t0 + secs(60)); + let l = Lease::new(t0, 12); + assert_eq!(l.renew_after, t0 + secs(10)); + // Shorter than the 10 s floor: renewal starts at expiry at the latest. + let l = Lease::new(t0, 4); + assert_eq!(l.renew_after, t0 + secs(4)); + assert!(l.renew_after <= l.expires_at); + } + + /// A sleep ages renewal, trust and the router's grant, never the commit instant; an age past + /// what the clock can represent lands on `now` (expired). + #[test] + fn age_moves_every_deadline_but_the_commit() { + let t0 = Instant::now(); + let mut l = Lease::new(t0, 7200); + l.age(secs(1800), t0); + assert_eq!(l.committed_at, t0); + assert_eq!(l.renew_after, t0 + secs(1800)); + assert_eq!(l.expires_at, t0 + secs(5400)); + assert_eq!(l.router_expiry, t0 + secs(5400)); + let now = t0 + secs(1); + let mut l = Lease::new(t0, 7200); + l.age(Duration::MAX, now); + assert!(l.expired(now) && l.renew_due(now)); + } + + /// Lifetimes 0, 1 and `u32::MAX` never panic. + #[test] + fn extreme_lifetimes() { + let t0 = Instant::now(); + for g in [0, 1, u32::MAX] { + let l = Lease::new(t0, g); + assert!(l.renew_after <= l.expires_at); + assert!(l.expires_at <= l.router_expiry); + } + assert!(Lease::new(t0, 0).expired(t0)); + } + + #[test] + fn backoff_steps_and_cap() { + let got: Vec = (1..=8).map(|n| backoff(n).as_secs()).collect(); + assert_eq!(got, [10, 30, 60, 120, 300, 600, 600, 600]); + assert_eq!(backoff(0).as_secs(), 10); + assert_eq!(backoff(u32::MAX).as_secs(), 600); + assert_eq!(jittered(secs(100), 0.0), secs(80)); + assert_eq!(jittered(secs(100), 9.0), secs(120)); + } + + #[test] + fn floor_applies_across_all_triggers() { + let t0 = Instant::now(); + let mut s = Schedule::default(); + assert!(s.may_start(t0)); + s.note_start(t0); + s.note_success(); + assert!(!s.may_start(t0 + secs(1))); + assert!(s.may_start(t0 + ATTEMPT_FLOOR)); + // A network change resets the backoff but not the floor. + s.note_failure(t0, Failure::Transient, 1.0); + s.note_network_change(); + assert!(!s.may_start(t0 + secs(1))); + assert!(s.may_start(t0 + secs(2))); + } + + #[test] + fn failures_back_off_and_refusals_hold_longer() { + let t0 = Instant::now(); + let mut s = Schedule::default(); + s.note_failure(t0, Failure::Transient, 1.0); + assert_eq!(s.next_attempt_at, Some(t0 + secs(10))); + s.note_failure(t0, Failure::Transient, 1.0); + assert_eq!(s.next_attempt_at, Some(t0 + secs(30))); + s.note_failure(t0, Failure::NotAuthorized, 1.0); + assert_eq!(s.next_attempt_at, Some(t0 + NOT_AUTHORIZED_HOLD)); + + let mut s = Schedule::default(); + // A PCP error lifetime above the backoff holds, capped at 30 min. + s.note_failure( + t0, + Failure::Refused { + hold: Some(secs(120)), + }, + 1.0, + ); + assert_eq!(s.next_attempt_at, Some(t0 + secs(120))); + s.note_failure( + t0, + Failure::Refused { + hold: Some(secs(86_400)), + }, + 1.0, + ); + assert_eq!(s.next_attempt_at, Some(t0 + MAX_ERROR_HOLD)); + // A short error lifetime still respects the backoff floor. + let mut s = Schedule::default(); + s.note_failure( + t0, + Failure::Refused { + hold: Some(secs(1)), + }, + 1.0, + ); + assert_eq!(s.next_attempt_at, Some(t0 + secs(10))); + s.note_success(); + assert_eq!((s.failures, s.next_attempt_at), (0, None)); + } + + /// At the schedule level, a failed renewal with a past `renew_after` is spaced by backoff, + /// however often the driver asks. + #[test] + fn past_renew_after_never_spins() { + let t0 = Instant::now(); + let lease = Lease::new(t0, 60); + let mut s = Schedule::default(); + let mut now = t0 + secs(30); + let mut starts = 0; + // Ask every 100 ms for 10 simulated minutes. + while now < t0 + secs(630) { + assert!(s.renewal_due(&lease) >= lease.renew_after); + if s.may_start(now) { + starts += 1; + s.note_start(now); + s.note_failure(now, Failure::Transient, 1.0); + assert!(s.renewal_due(&lease) > now); + } + now += Duration::from_millis(100); + } + // 10 + 30 + 60 + 120 + 300 s fit in 600 s: six starts, not thousands. + assert_eq!(starts, 6); + } +} diff --git a/ts_portmapper/src/test_time.rs b/ts_portmapper/src/test_time.rs new file mode 100644 index 00000000..5a8ef1f2 --- /dev/null +++ b/ts_portmapper/src/test_time.rs @@ -0,0 +1,46 @@ +//! Hold exchange/worker deadlines still while lifecycle tests arrange an interleaving. +//! +//! A router reply barrier does not stop the client's exchange timeout. Plain `time::pause` +//! is insufficient too: Tokio may auto-advance while loopback I/O or our detached OS worker +//! is pending. A blocking task inhibits auto-advance (see Tokio's `time::pause` docs). Keep +//! one alive until the test explicitly releases time; `advance` still works for deadline cuts. + +use std::time::Duration; + +pub(crate) struct FrozenTime(std::sync::mpsc::Sender<()>); + +impl FrozenTime { + pub(crate) fn new() -> Self { + let (tx, rx) = std::sync::mpsc::channel(); + tokio::task::spawn_blocking(move || { + let _ = rx.recv(); + }); + tokio::time::pause(); + Self(tx) + } +} + +impl Drop for FrozenTime { + fn drop(&mut self) { + tokio::time::resume(); + // Also releases the blocking task on panic; runtime teardown must never wait for us. + let _ = self.0.send(()); + } +} + +/// Exercise a deadline with time frozen, after the future has installed its timers/claims. +pub(crate) async fn finish_at_deadline( + future: F, + budget: Duration, +) -> F::Output { + use std::{future::poll_fn, task::Poll}; + let mut future = std::pin::pin!(future); + poll_fn(|cx| { + assert!(future.as_mut().poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + // Tokio rounds timer-wheel deadlines up to the next millisecond. + tokio::time::advance(budget + Duration::from_millis(1)).await; + future.await +} diff --git a/ts_portmapper/src/testing/cap.rs b/ts_portmapper/src/testing/cap.rs index f693b13e..e6a16f27 100644 --- a/ts_portmapper/src/testing/cap.rs +++ b/ts_portmapper/src/testing/cap.rs @@ -18,7 +18,7 @@ use crate::{ Gateway, PXP_PORT, cap::{ BindMode, CapError, ExchangeStats, Flow, LanDest, PolicyHandle, ProbeListener, - PxpTransport, SendGate, exchange_on, sealed, + PxpTransport, SendEvidence, SendGate, exchange_on, sealed, }, }; @@ -33,6 +33,7 @@ pub struct TestCap { aliases: Vec<(SocketAddr, SocketAddrV4)>, gate: SendGate, inject_send_error: Mutex>, + inject_bind_error: Mutex>, panic_next: std::sync::atomic::AtomicBool, } @@ -90,6 +91,7 @@ impl TestCap { gateway, gate: SendGate::new(policy, None), inject_send_error: Mutex::new(None), + inject_bind_error: Mutex::new(None), panic_next: std::sync::atomic::AtomicBool::new(false), }) } @@ -108,6 +110,14 @@ impl TestCap { .unwrap_or_else(|p| p.into_inner()) = raw_os_error; } + /// Fail socket setup before any send attempt, without touching a real interface. + pub fn inject_bind_error(&self, raw_os_error: Option) { + *self + .inject_bind_error + .lock() + .unwrap_or_else(|p| p.into_inner()) = raw_os_error; + } + /// Refuse new work, like [`crate::GatewayCap::set_closing`]. pub fn set_closing(&self) { self.gate.set_closing(); @@ -143,16 +153,23 @@ impl PxpTransport for TestCap { BindMode::LoopbackTest } - async fn exchange( + async fn exchange_with_evidence( &self, dest: LanDest, reqs: &[&[u8]], deadline: Instant, on_reply: &mut (dyn FnMut(&[u8]) -> Flow + Send), + evidence: Option<&SendEvidence>, ) -> Result { let LanDest::Pxp = dest; self.gate.check()?; - self.injected()?; + if let Some(code) = *self + .inject_bind_error + .lock() + .unwrap_or_else(|p| p.into_inner()) + { + return Err(CapError::io("bind", &io::Error::from_raw_os_error(code))); + } assert!( !self .panic_next @@ -162,10 +179,20 @@ impl PxpTransport for TestCap { let sock = tokio::net::UdpSocket::bind((self.bind_ip, 0)) .await .map_err(|e| CapError::io("bind", &e))?; + if let Err(err) = self.injected() { + // Simulate an OS failure of the first send, after setup and the final permit. + self.gate.send(|| { + if let Some(evidence) = evidence { + evidence.enter(0); + } + Ok(0) + })?; + return Err(err); + } let expected = SocketAddrV4::new(self.gateway.gw, PXP_PORT); let present = |src| self.present(src); exchange_on( - &sock, &self.gate, self.to, expected, &present, reqs, deadline, on_reply, + &sock, &self.gate, self.to, expected, &present, reqs, deadline, on_reply, evidence, ) .await } @@ -173,6 +200,13 @@ impl PxpTransport for TestCap { fn send_nowait(&self, dest: LanDest, req: &[u8]) -> Result<(), CapError> { let LanDest::Pxp = dest; self.gate.check()?; + if let Some(code) = *self + .inject_bind_error + .lock() + .unwrap_or_else(|p| p.into_inner()) + { + return Err(CapError::io("bind", &io::Error::from_raw_os_error(code))); + } self.injected()?; let s = std::net::UdpSocket::bind((self.bind_ip, 0)).map_err(|e| CapError::io("bind", &e))?; diff --git a/ts_portmapper/src/testing/lifecycle.rs b/ts_portmapper/src/testing/lifecycle.rs new file mode 100644 index 00000000..25dd2a13 --- /dev/null +++ b/ts_portmapper/src/testing/lifecycle.rs @@ -0,0 +1,201 @@ +//! Harness pieces for the lifecycle client: a [`TestCaps`] factory that builds [`TestCap`]s for +//! logical gateways backed by fake routers, a [`ScriptedLookup`] whose answer a test changes, a +//! [`PanicLookup`] for configurations that must never look a gateway up, and a [`ManualClock`]. + +use std::{ + collections::{HashMap, HashSet}, + net::Ipv4Addr, + sync::{ + Arc, Mutex, MutexGuard, + atomic::{AtomicU64, AtomicUsize, Ordering}, + }, + time::{Duration, SystemTime}, +}; + +use tokio::time::Instant; + +use super::{RouterEndpoint, TestCap}; +use crate::{ + CapError, Gateway, GatewayLookup, LookupError, PolicyHandle, PxpTransport, cap::NetGen, + client::CapFactory, clock::Clock, +}; + +fn lock(m: &Mutex) -> MutexGuard<'_, S> { + m.lock().unwrap_or_else(std::sync::PoisonError::into_inner) +} + +/// Builds [`TestCap`]s: each logical gateway address maps to a fake router's endpoint. A gateway +/// that is not [present](TestCaps::set_present) fails to build, which is how a test says "the old +/// network no longer validates". +#[derive(Debug, Default)] +pub struct TestCaps { + routers: Mutex>, + absent: Mutex>, + built: Mutex>>, + builds: AtomicUsize, +} + +impl TestCaps { + /// An empty factory. + pub fn new() -> Arc { + Arc::new(Self::default()) + } + + /// Route the logical gateway `gw` to `router`. + pub fn add(&self, gw: Ipv4Addr, router: RouterEndpoint) { + lock(&self.routers).insert(gw, router); + } + + /// Whether `gw`'s network still validates. + pub fn set_present(&self, gw: Ipv4Addr, present: bool) { + let mut absent = lock(&self.absent); + if present { + absent.remove(&gw); + } else { + absent.insert(gw); + } + } + + /// How many caps were built. + pub fn builds(&self) -> usize { + self.builds.load(Ordering::SeqCst) + } + + /// Every cap built so far (to inject errors into them). + pub fn built(&self) -> Vec> { + lock(&self.built).clone() + } +} + +impl CapFactory for TestCaps { + fn build( + &self, + gateway: &Gateway, + _net_gen: NetGen, + policy: &PolicyHandle, + ) -> Result, CapError> { + if lock(&self.absent).contains(&gateway.gw) { + return Err(CapError::InvalidGateway("test: network absent")); + } + let Some(router) = lock(&self.routers).get(&gateway.gw).copied() else { + return Err(CapError::InvalidGateway("test: no router for this gateway")); + }; + self.builds.fetch_add(1, Ordering::SeqCst); + let cap = TestCap::with_endpoint(gateway.clone(), &router, policy.clone()); + lock(&self.built).push(cap.clone()); + debug_assert!(cap.is_valid()); + Ok(cap) + } +} + +/// What a [`ScriptedLookup`] answers. +#[derive(Debug, Clone, PartialEq, Eq)] +enum Answer { + Gateway(Gateway), + NoRoute, + Error, +} + +/// A gateway lookup whose answer the test sets. Counts its calls. +#[derive(Debug)] +pub struct ScriptedLookup { + answer: Mutex, + calls: AtomicUsize, +} + +impl ScriptedLookup { + /// A lookup answering `gw`. + pub fn new(gw: Gateway) -> Arc { + Arc::new(Self { + answer: Mutex::new(Answer::Gateway(gw)), + calls: AtomicUsize::new(0), + }) + } + + /// Answer `gw` from now on. + pub fn set(&self, gw: Gateway) { + *lock(&self.answer) = Answer::Gateway(gw); + } + + /// Answer "no default route" (`Ok(None)`) from now on. + pub fn set_no_route(&self) { + *lock(&self.answer) = Answer::NoRoute; + } + + /// Fail from now on. + pub fn set_error(&self) { + *lock(&self.answer) = Answer::Error; + } + + /// How many lookups happened. + pub fn calls(&self) -> usize { + self.calls.load(Ordering::SeqCst) + } +} + +impl GatewayLookup for ScriptedLookup { + fn lookup(&self) -> Result, LookupError> { + self.calls.fetch_add(1, Ordering::SeqCst); + match &*lock(&self.answer) { + Answer::Gateway(g) => Ok(Some(g.clone())), + Answer::NoRoute => Ok(None), + Answer::Error => Err(LookupError::Ambiguous), + } + } +} + +/// A lookup that must never run: test configurations that disable the mapper install it, and a +/// call panics the test. +#[derive(Debug, Default, Clone, Copy)] +pub struct PanicLookup; + +impl GatewayLookup for PanicLookup { + fn lookup(&self) -> Result, LookupError> { + panic!("the gateway lookup ran in a test configuration that must never touch the network"); + } +} + +/// A manual clock: the real monotonic clock plus an offset the test advances, and a wall clock +/// that can also move on its own (a suspend, or a wall-clock jump). +#[derive(Debug, Default)] +pub struct ManualClock { + offset_ms: AtomicU64, + wall_extra_ms: AtomicU64, +} + +impl ManualClock { + /// A clock with no offset. + pub fn new() -> Arc { + Arc::new(Self::default()) + } + + /// Move both clocks forward by `d`. + pub fn advance(&self, d: Duration) { + self.offset_ms.fetch_add(ms(d), Ordering::SeqCst); + } + + /// The host slept for `d`: the wall clock moved, the monotonic clock did not. + pub fn suspend(&self, d: Duration) { + self.wall_extra_ms.fetch_add(ms(d), Ordering::SeqCst); + } + + fn offset(&self) -> Duration { + Duration::from_millis(self.offset_ms.load(Ordering::SeqCst)) + } +} + +fn ms(d: Duration) -> u64 { + u64::try_from(d.as_millis()).unwrap_or(u64::MAX) +} + +impl Clock for ManualClock { + fn now(&self) -> Instant { + Instant::now() + self.offset() + } + + fn wall(&self) -> SystemTime { + SystemTime::now() + + self.offset() + + Duration::from_millis(self.wall_extra_ms.load(Ordering::SeqCst)) + } +} diff --git a/ts_portmapper/src/testing/mod.rs b/ts_portmapper/src/testing/mod.rs index 3384db0c..ecf31c8c 100644 --- a/ts_portmapper/src/testing/mod.rs +++ b/ts_portmapper/src/testing/mod.rs @@ -12,6 +12,7 @@ mod cap; mod clock; +mod lifecycle; mod router; #[cfg(test)] mod tests; @@ -20,6 +21,7 @@ use std::net::{Ipv4Addr, SocketAddr}; pub use cap::TestCap; pub use clock::LeaseClock; +pub use lifecycle::{ManualClock, PanicLookup, ScriptedLookup, TestCaps}; pub use router::{ AssignPort, Barrier, Corruption, Counters, DecodedRequest, EpochMode, FakeRouter, MappingEntry, RequestKind, ResultOverride, RouterEndpoint, RouterOptions, @@ -46,3 +48,12 @@ pub async fn send_datagrams( } s.local_addr() } + +/// Read-only cleanup diagnostics for integration tests; no ledger mutation is exposed. +#[derive(Debug, Clone, Copy)] +pub struct CleanupSnapshot { + /// Unacknowledged delete attempts. + pub delete_tries: u32, + /// Whether an operation currently reserves this entry for deletion. + pub deleting: bool, +} From 5d45d5884c1d692f411717fd7c2276ac658cf594 Mon Sep 17 00:00:00 2001 From: Kasper Nielsen Date: Sat, 17 Oct 2026 23:06:46 +0200 Subject: [PATCH 03/12] test(portmapper): add fuzz targets for every parser A cargo-fuzz crate with the same layout as ts_tka/fuzz, kept out of the workspace: NAT-PMP and PCP replies (including the MAP validators) and both routing-table parsers (the macOS routing-socket dump and Linux /proc/net/route). Signed-off-by: Kasper Nielsen Co-Authored-By: GPT-6.1 (codex) Co-Authored-By: Claude Opus 5.5 --- .gitignore | 3 + checks/src/lints_enabled.rs | 2 + ts_portmapper/fuzz/Cargo.lock | 280 ++++++++++++++++++ ts_portmapper/fuzz/Cargo.toml | 49 +++ ts_portmapper/fuzz/fuzz_targets/parse_pcp.rs | 18 ++ .../fuzz/fuzz_targets/parse_pcp_map.rs | 51 ++++ ts_portmapper/fuzz/fuzz_targets/parse_pmp.rs | 29 ++ .../fuzz/fuzz_targets/parse_proc_net_route.rs | 18 ++ .../fuzz/fuzz_targets/parse_rt_dump.rs | 12 + 9 files changed, 462 insertions(+) create mode 100644 ts_portmapper/fuzz/Cargo.lock create mode 100644 ts_portmapper/fuzz/Cargo.toml create mode 100644 ts_portmapper/fuzz/fuzz_targets/parse_pcp.rs create mode 100644 ts_portmapper/fuzz/fuzz_targets/parse_pcp_map.rs create mode 100644 ts_portmapper/fuzz/fuzz_targets/parse_pmp.rs create mode 100644 ts_portmapper/fuzz/fuzz_targets/parse_proc_net_route.rs create mode 100644 ts_portmapper/fuzz/fuzz_targets/parse_rt_dump.rs diff --git a/.gitignore b/.gitignore index c3cc5a70..af8cfc6b 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,9 @@ site/ ts_tka/fuzz/target/ ts_tka/fuzz/corpus/ ts_tka/fuzz/artifacts/ +ts_portmapper/fuzz/target/ +ts_portmapper/fuzz/corpus/ +ts_portmapper/fuzz/artifacts/ # produced by cargo-bisect-rustc target-bisector-*/ .idea/ diff --git a/checks/src/lints_enabled.rs b/checks/src/lints_enabled.rs index d831a8e5..01c3171a 100644 --- a/checks/src/lints_enabled.rs +++ b/checks/src/lints_enabled.rs @@ -35,6 +35,8 @@ pub fn run(_args: &Args) -> BoxResult<()> { // nightly-only dev tooling excluded from the normal build/test graph. Skip it here // rather than force a redundant `[workspace.lints]` into the fuzz manifest. "!ts_tka/fuzz", + // Same for the port mapper's parser fuzz crate. + "!ts_portmapper/fuzz", ], ) .build()?; diff --git a/ts_portmapper/fuzz/Cargo.lock b/ts_portmapper/fuzz/Cargo.lock new file mode 100644 index 00000000..1092c36e --- /dev/null +++ b/ts_portmapper/fuzz/Cargo.lock @@ -0,0 +1,280 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "arbitrary" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bc62ac97cc33321f50863d514c3bc38a453947a8f9e781137e47c7401020aed" + +[[package]] +name = "cc" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6651c9ed80effdc7db0ff72512157f901af5e3549e341e24b1dd4887d836d838" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "find-msvc-tools" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" + +[[package]] +name = "geiserx_ts_portmapper" +version = "0.57.8" +dependencies = [ + "getrandom", + "if-addrs", + "libc", + "socket2", + "thiserror", + "tokio", + "tracing", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "if-addrs" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0a05c691e1fae256cf7013d99dad472dc52d5543322761f83ec8d47eab40d2b" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom", + "libc", +] + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "libfuzzer-sys" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9fd2f41a1cba099f79a0b6b6c35656cf7c03351a7bae8ff0f28f25270f929d2" +dependencies = [ + "arbitrary", + "cc", +] + +[[package]] +name = "mio" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" +dependencies = [ + "libc", + "wasi", + "windows-sys", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "tokio" +version = "1.53.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce3335fa71841cda333a58d7615b03901380ecf09d59b3296d21f8bbac0dde4e" +dependencies = [ + "libc", + "mio", + "pin-project-lite", + "socket2", + "tokio-macros", + "windows-sys", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" + +[[package]] +name = "ts_portmapper-fuzz" +version = "0.0.0" +dependencies = [ + "geiserx_ts_portmapper", + "libfuzzer-sys", +] + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] diff --git a/ts_portmapper/fuzz/Cargo.toml b/ts_portmapper/fuzz/Cargo.toml new file mode 100644 index 00000000..4134297a --- /dev/null +++ b/ts_portmapper/fuzz/Cargo.toml @@ -0,0 +1,49 @@ +# cargo-fuzz crate for the port mapper's parsers (same layout as ts_tka/fuzz). +# +# Its OWN workspace (the empty `[workspace]` table below), by cargo-fuzz convention: it is not a +# member of the parent workspace, so the nightly-only libfuzzer machinery never enters the normal +# build. Run from this directory: `cargo +nightly fuzz run parse_pcp_map`. +[package] +name = "ts_portmapper-fuzz" +version = "0.0.0" +publish = false +edition = "2021" + +[package.metadata] +cargo-fuzz = true + +[dependencies] +libfuzzer-sys = "0.4" +geiserx_ts_portmapper = { path = ".." } + +[workspace] + +[[bin]] +name = "parse_pmp" +path = "fuzz_targets/parse_pmp.rs" +test = false +doc = false + +[[bin]] +name = "parse_pcp" +path = "fuzz_targets/parse_pcp.rs" +test = false +doc = false + +[[bin]] +name = "parse_pcp_map" +path = "fuzz_targets/parse_pcp_map.rs" +test = false +doc = false + +[[bin]] +name = "parse_rt_dump" +path = "fuzz_targets/parse_rt_dump.rs" +test = false +doc = false + +[[bin]] +name = "parse_proc_net_route" +path = "fuzz_targets/parse_proc_net_route.rs" +test = false +doc = false diff --git a/ts_portmapper/fuzz/fuzz_targets/parse_pcp.rs b/ts_portmapper/fuzz/fuzz_targets/parse_pcp.rs new file mode 100644 index 00000000..09678ef6 --- /dev/null +++ b/ts_portmapper/fuzz/fuzz_targets/parse_pcp.rs @@ -0,0 +1,18 @@ +//! PCP header and ANNOUNCE parsers (port of Go's `FuzzParsePCPResponse`, with invariants). +#![no_main] + +use libfuzzer_sys::fuzz_target; +use ts_portmapper::pcp; + +fuzz_target!(|data: &[u8]| { + if let Ok(h) = pcp::parse_header(data) { + assert!((pcp::HEADER_LEN..=pcp::MAX_LEN).contains(&data.len())); + assert_eq!(data.len() % 4, 0); + assert_eq!(data[0], pcp::VERSION); + assert_ne!(data[1] & pcp::R_BIT, 0); + assert_eq!(h.opcode, data[1] & !pcp::R_BIT); + } + if pcp::parse_announce_reply(data).is_ok() { + assert_eq!(data[1] & !pcp::R_BIT, pcp::OP_ANNOUNCE); + } +}); diff --git a/ts_portmapper/fuzz/fuzz_targets/parse_pcp_map.rs b/ts_portmapper/fuzz/fuzz_targets/parse_pcp_map.rs new file mode 100644 index 00000000..4d3e7ad7 --- /dev/null +++ b/ts_portmapper/fuzz/fuzz_targets/parse_pcp_map.rs @@ -0,0 +1,51 @@ +//! PCP MAP reply parser (port of Go's `FuzzParsePCPMapResponse`): exactly one of a mapping or an +//! error (by construction), a MAP parser never accepts another opcode, a success always satisfies +//! the documented invariants, and only a full-length response that echoes our nonce is ever a +//! success or an authoritative router error. +#![no_main] + +use std::num::NonZeroU16; + +use libfuzzer_sys::fuzz_target; +use ts_portmapper::pcp::{self, MapKind, Nonce, ReplyError}; + +/// The nonce "we sent". Fixed and independent of the input, so a reply has to match it the way a +/// spoofer would have to (libFuzzer's comparison tracing still finds it). +const NONCE: [u8; 12] = *b"ts-pm-nonce!"; + +fuzz_target!(|data: &[u8]| { + let nonce = Nonce::from_bytes(NONCE); + // Use the internal port the reply carries, so the success path is reachable. + let port = data + .get(40..42) + .map(|b| u16::from_be_bytes([b[0], b[1]])) + .and_then(NonZeroU16::new) + .unwrap_or(NonZeroU16::MIN); + for kind in [MapKind::Map, MapKind::Delete] { + let r = pcp::parse_map_reply(data, &nonce, port, kind); + if data + .get(1) + .is_some_and(|op| op & !pcp::R_BIT != pcp::OP_MAP) + { + assert!(r.is_err(), "a MAP parser accepted another opcode"); + } + if matches!(r, Ok(_) | Err(ReplyError::Router { .. })) { + assert!(data.len() >= pcp::MAP_LEN, "authoritative without a nonce"); + assert_eq!(&data[24..36], &NONCE, "authoritative with another nonce"); + } + if let Err(ReplyError::Unmatched { .. }) = r { + assert!(data.len() < pcp::MAP_LEN); + } + if let Ok(m) = r { + assert_eq!(m.internal, port.get()); + assert_eq!(data[36], pcp::PROTO_UDP); + match kind { + MapKind::Map => { + assert!(m.lifetime != 0 && m.external.port() != 0); + assert!(ts_portmapper::is_plausible_external(*m.external.ip())); + } + MapKind::Delete => assert_eq!(m.lifetime, 0), + } + } + } +}); diff --git a/ts_portmapper/fuzz/fuzz_targets/parse_pmp.rs b/ts_portmapper/fuzz/fuzz_targets/parse_pmp.rs new file mode 100644 index 00000000..ebb5e93d --- /dev/null +++ b/ts_portmapper/fuzz/fuzz_targets/parse_pmp.rs @@ -0,0 +1,29 @@ +//! NAT-PMP reply parsers: never panic, and a success always satisfies the documented invariants. +#![no_main] + +use std::num::NonZeroU16; + +use libfuzzer_sys::fuzz_target; +use ts_portmapper::pmp::{self, MapKind}; + +fuzz_target!(|data: &[u8]| { + let _ = pmp::reply_result(data); + if let Ok(r) = pmp::parse_public_addr_reply(data) { + assert_eq!(data.len(), pmp::PUBLIC_ADDR_REPLY_LEN); + assert!(ts_portmapper::is_plausible_external(r.public_ip)); + } + // Ask for the internal port the reply carries (when there is one) to reach the success path. + let port = data + .get(8..10) + .map(|b| u16::from_be_bytes([b[0], b[1]])) + .and_then(NonZeroU16::new) + .unwrap_or(NonZeroU16::MIN); + if let Ok(r) = pmp::parse_map_reply(data, port, MapKind::Map) { + assert_eq!(data.len(), pmp::MAP_REPLY_LEN); + assert_eq!(r.internal, port.get()); + assert!(r.external != 0 && r.lifetime != 0); + } + if let Ok(r) = pmp::parse_map_reply(data, port, MapKind::Delete) { + assert_eq!(r.lifetime, 0); + } +}); diff --git a/ts_portmapper/fuzz/fuzz_targets/parse_proc_net_route.rs b/ts_portmapper/fuzz/fuzz_targets/parse_proc_net_route.rs new file mode 100644 index 00000000..dac3e1bd --- /dev/null +++ b/ts_portmapper/fuzz/fuzz_targets/parse_proc_net_route.rs @@ -0,0 +1,18 @@ +//! Linux `/proc/net/route` and `/proc/net/ipv6_route` parsers: capped, never panic, and every +//! default route has a nonzero gateway and an interface name. +#![no_main] + +use libfuzzer_sys::fuzz_target; +use ts_portmapper::gateway::linux::{ + parse_proc_net_ipv6_route_has_default, parse_proc_net_route, MAX_BYTES, +}; + +fuzz_target!(|data: &[u8]| { + if let Ok(routes) = parse_proc_net_route(data) { + assert!(data.len() <= MAX_BYTES); + assert!(routes + .iter() + .all(|r| !r.gw.is_unspecified() && r.if_name.is_some())); + } + let _ = parse_proc_net_ipv6_route_has_default(data); +}); diff --git a/ts_portmapper/fuzz/fuzz_targets/parse_rt_dump.rs b/ts_portmapper/fuzz/fuzz_targets/parse_rt_dump.rs new file mode 100644 index 00000000..37bf676b --- /dev/null +++ b/ts_portmapper/fuzz/fuzz_targets/parse_rt_dump.rs @@ -0,0 +1,12 @@ +//! macOS routing-socket dump parser: bounds-checked, never panics, and every default route it +//! reports has a nonzero gateway. +#![no_main] + +use libfuzzer_sys::fuzz_target; +use ts_portmapper::gateway::darwin::parse_rt_dump; + +fuzz_target!(|data: &[u8]| { + if let Ok(r) = parse_rt_dump(data) { + assert!(r.v4.iter().all(|d| !d.gw.is_unspecified() && d.if_index.is_some())); + } +}); From 11dc686398018633dc8f909126752458cb49787d Mon Sep 17 00:00:00 2001 From: Kasper Nielsen Date: Sat, 17 Oct 2026 23:06:47 +0200 Subject: [PATCH 04/12] feat(runtime): run the port mapper for the underlay socket A driver (ts_runtime/src/portmap.rs), spawned by the direct manager unless the mapper is off, runs the ts_portmapper lifecycle client for the magicsock's underlay port and installs the mapping as the PortMapped endpoint. - magicsock: SelfEndpointType::PortMapped (Go EndpointPortmapped). self_endpoints() emits Go's priority order (port-mapped, STUN, STUN4LocalPort, local) and lists each address once. A rebind clears the mapping at once. CallMeMaybe carries it. - control: StunFacts and PortMapFacets travel with the endpoints, and NetInfo carries PMP, PCP and HavePortMap on every map request. UPnP is never set. ts_control still does not depend on the port mapper. - driver: wakes on a 10 s network poll (resume and clock-jump aware), the mapping's expiry, the periodic ReSTUN and kicks (start, first peer, rebind, only-tcp-443 lifted). A rebind invalidates synchronously. only-tcp-443 stops every LAN send at once, deletes included. A graceful shutdown closes the client first, within min(1 s, timeout), and deletes with cached capabilities; dropping the runtime sends one best-effort delete per rule without a Tokio runtime. In Linux source-address mode (no SO_BINDTODEVICE permission) both fail closed and leave the rules to expire. - config: disable_port_mapper (Go DisablePortMapper) and PortMapperOptions on tailscale::Config and ts_control::Config, with Go's defaults. TS_DISABLE_PORTMAPPER is read once at startup. Device::port_map_status() reports the facets and the mapping. - feature portmapper, on by default as in Go, on ts_runtime and the facade. Without it the driver is not compiled and the facets stay unset. Tests never reach a real gateway. Every shared runtime test configuration sets port_mapper: None. In ts_runtime's test build the production gateway lookup is a stub that panics. The integration tests that build devices set TS_DISABLE_PORTMAPPER. Signed-off-by: Kasper Nielsen Co-Authored-By: GPT-6.1 (codex) Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 1 + Cargo.lock | 1 + Cargo.toml | 7 +- docs/PORTMAPPER.md | 29 +- src/config.rs | 45 + src/lib.rs | 12 +- tests/basic.rs | 11 +- tests/tailnet_e2e_campaign.rs | 10 +- tests/tailnet_live.rs | 8 +- tests/tsnet_facade_e2e.rs | 3 + ts_control/src/config.rs | 43 + ts_control/src/lib.rs | 4 +- ts_control/src/map_request_builder.rs | 25 + ts_control/src/net_facets.rs | 117 +++ ts_control/src/tokio/client.rs | 194 ++--- ts_magicsock/src/endpoint.rs | 8 +- ts_magicsock/src/sock.rs | 193 ++++- ts_runtime/Cargo.toml | 12 + ts_runtime/src/control_runner.rs | 32 +- ts_runtime/src/derp_latency.rs | 1 + ts_runtime/src/direct.rs | 300 ++++++- ts_runtime/src/env.rs | 64 ++ ts_runtime/src/forwarder_actor.rs | 1 + ts_runtime/src/lib.rs | 101 ++- ts_runtime/src/netmon.rs | 1 + ts_runtime/src/packetfilter.rs | 1 + ts_runtime/src/peer_tracker/mod.rs | 1 + ts_runtime/src/peerapi_doh.rs | 1 + ts_runtime/src/portmap.rs | 1090 +++++++++++++++++++++++++ ts_runtime/src/tun_actor.rs | 1 + 30 files changed, 2124 insertions(+), 193 deletions(-) create mode 100644 ts_control/src/net_facets.rs create mode 100644 ts_runtime/src/portmap.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 039abaa2..00f11f67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ Record breaking or significant changes here. All dates are UTC. ### Features * **portmapper:** add `ts_portmapper`, a NAT-PMP and PCP client modelled on Go's `net/portmapper`, which only ever sends to the LAN gateway's port 5351. +* **runtime:** map the underlay UDP port on the LAN gateway with NAT-PMP or PCP and advertise the mapping to peers, as Go does; `Config::disable_port_mapper` or `TS_DISABLE_PORTMAPPER` turns it off. ### Performance diff --git a/Cargo.lock b/Cargo.lock index 638d3657..aa5f3d3a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2400,6 +2400,7 @@ dependencies = [ "geiserx_ts_packet", "geiserx_ts_packetfilter", "geiserx_ts_packetfilter_state", + "geiserx_ts_portmapper", "geiserx_ts_tka", "geiserx_ts_transport", "geiserx_ts_transport_tun", diff --git a/Cargo.toml b/Cargo.toml index 6ae2030c..c69d8124 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -189,7 +189,7 @@ ts_http_util = { path = "ts_http_util", package = "geiserx_ts_http_util", versio ts_portmapper = { path = "ts_portmapper", package = "geiserx_ts_portmapper", version = "0.57.8" } # x-release-please-version ts_tka = { path = "ts_tka", package = "geiserx_ts_tka", version = "0.57.8" } # x-release-please-version ts_tls_util = { path = "ts_tls_util", package = "geiserx_ts_tls_util", version = "0.57.8" } # x-release-please-version -ts_runtime = { path = "ts_runtime", package = "geiserx_ts_runtime", version = "0.57.8" } # x-release-please-version +ts_runtime = { path = "ts_runtime", package = "geiserx_ts_runtime", version = "0.57.8", default-features = false } # x-release-please-version ts_test_util = { path = "ts_test_util" } ts_time = { path = "ts_time", package = "geiserx_ts_time", version = "0.57.8" } # x-release-please-version ts_transport = { path = "ts_transport", package = "geiserx_ts_transport", version = "0.57.8" } # x-release-please-version @@ -350,7 +350,12 @@ hyper-util = { workspace = true, features = ["tokio"] } workspace = true [features] +default = ["portmapper"] ring-aead = ["ts_runtime/ring-aead"] +# On by default (Go parity): the NAT port mapper (NAT-PMP/PCP) for direct paths through hard NATs. +# `Config::disable_port_mapper` and `TS_DISABLE_PORTMAPPER` turn it off at runtime; without this +# feature it is not compiled at all. +portmapper = ["ts_runtime/portmapper"] # Enable the `tsnet` module: a Go-idiomatic `tsnet.Server`-shaped facade over `Device`/`Config` # (settable fields → `Config`, lazy start, `Listen`/`Dial`/`Up`/`Close`/`ListenFunnel`/`ListenService`). # A thin ergonomics layer only — no new engine, no new crate. Reuses the crate's existing diff --git a/docs/PORTMAPPER.md b/docs/PORTMAPPER.md index 99a2429e..283ef783 100644 --- a/docs/PORTMAPPER.md +++ b/docs/PORTMAPPER.md @@ -1,13 +1,26 @@ # NAT port mapper (NAT-PMP and PCP) -[`ts_portmapper`](../ts_portmapper/src/lib.rs) asks the LAN gateway for an inbound UDP mapping to -the node's underlay port, as Go Tailscale's `net/portmapper` does, so that a peer behind a hard -NAT can reach the node directly instead of over DERP. NAT-PMP (RFC 6886) and PCP (RFC 6887) are -implemented. UPnP-IGD is not (bead `tsr-x4yg`, phase 2). +The node asks its LAN gateway for an inbound UDP mapping to its underlay port, as Go Tailscale's +`net/portmapper` does. It then advertises the mapping as endpoint type 3 (`PortMapped`), so a peer +behind a hard NAT can reach it directly instead of over DERP. NAT-PMP (RFC 6886) and PCP +(RFC 6887) are implemented. UPnP-IGD is not (bead `tsr-x4yg`, phase 2). -The crate holds the lifecycle client, the codecs, the gateway lookup and `GatewayCap`, the only -code that sends packets. (The gateway lookup's kernel source-address check connects a UDP socket -but never sends on it.) Nothing in the node runs it yet. +- Crate: [`ts_portmapper`](../ts_portmapper/src/lib.rs). It holds the lifecycle client, the + codecs, the gateway lookup and `GatewayCap`, the only code that sends packets. (The gateway + lookup's kernel source-address check connects a UDP socket but never sends on it.) +- Driver: [`ts_runtime::portmap`](../ts_runtime/src/portmap.rs), spawned by the direct manager. + +## Turning it off + +| Source | Effect | +|---|---| +| `Config::disable_port_mapper` (Go `DisablePortMapper`) | No client, no driver, no socket; NetInfo facets unset | +| `TS_DISABLE_PORTMAPPER` (Go name), read once at startup | Same | +| Control `only-tcp-443` | Live: every LAN send stops at once, deletes included; facets `PMP`/`PCP` are false | +| Cargo feature `portmapper` off | Not compiled; facets unset | + +The feature is on by default, as in Go. Tests keep it off: every shared runtime test configuration +sets `port_mapper: None`, and the `portmapper_test_isolation` check enforces the rest. ## Deviations from Go @@ -17,6 +30,8 @@ but never sends on it.) Nothing in the node runs it yet. - **Renewal while idle (`renew_while_idle`).** The default is Go's: renewal rides on activity (the periodic ReSTUN and other kicks). With the option on, a due mapping is renewed even while idle, as long as there are peers. With no peers a mapping is never renewed and expires. +- **`HavePortMap` means an advertised mapping**, so control never sees it without a type-3 + endpoint. - **Notifications for every protocol.** Success, loss, expiry and disable publish for NAT-PMP and PCP alike. In Go only PCP successes publish. - **Strict replies.** Replies are accepted only from exactly `gateway:5351`, and every field is diff --git a/src/config.rs b/src/config.rs index 493cbd4f..71c72386 100644 --- a/src/config.rs +++ b/src/config.rs @@ -8,6 +8,9 @@ use ts_keys::PersistState; use crate::keys::NodeState; +/// The port mapper's options (see [`Config::port_mapper`]). +pub use ts_control::PortMapperOptions; + const CONTROL_URL_VAR: &str = "TS_CONTROL_URL"; const HOSTNAME_VAR: &str = "TS_HOSTNAME"; const AUTHKEY_VAR: &str = "TS_AUTH_KEY"; @@ -272,6 +275,20 @@ pub struct Config { /// fail-closed underlay posture (`enable_ipv6` alone widens the family) is unchanged. pub wireguard_listen_port: Option, + /// Turn the NAT port mapper off (Go `magicsock.Options.DisablePortMapper`). Defaults to + /// `false`: like Go, the node asks the LAN gateway for an inbound UDP mapping with NAT-PMP or + /// PCP, so peers behind a hard NAT can reach it directly. With `true` (or + /// `TS_DISABLE_PORTMAPPER` set in the environment) no client, driver or socket exists and no + /// port-mapping NetInfo is sent. Builds without the `portmapper` feature never map. + /// + /// A port-mapped endpoint exposes the host's public address to tailnet peers, as STUN + /// endpoints do; a proxy-egress (`exit_proxy`) deployment may want it off. + pub disable_port_mapper: bool, + + /// The port mapper's options: a validated gateway override, whether to advertise a non-public + /// external address (default yes, Go), and whether to renew while idle (default no, Go). + pub port_mapper: PortMapperOptions, + /// How this node's **application** overlay data path is realized. /// /// Defaults to [`TransportMode::Netstack`](ts_control::TransportMode::Netstack), the userspace @@ -690,6 +707,8 @@ impl From<&Config> for ts_control::Config { enable_ipv6: value.enable_ipv6, network_monitor: value.network_monitor, wireguard_listen_port: value.wireguard_listen_port, + disable_port_mapper: value.disable_port_mapper, + port_mapper: value.port_mapper.clone(), transport_mode: value.transport_mode.clone(), wire_ingress: value.wire_ingress, // A fresh runtime-local flag (default `false`): the runtime flips it when @@ -740,6 +759,8 @@ impl Default for Config { enable_ipv6: false, network_monitor: false, wireguard_listen_port: None, + disable_port_mapper: false, + port_mapper: ts_control::PortMapperOptions::default(), transport_mode: ts_control::TransportMode::default(), wire_ingress: false, advertise_services: vec![], @@ -788,6 +809,12 @@ mod tests { enable_ipv6: true, network_monitor: true, wireguard_listen_port: Some(41641), + disable_port_mapper: true, + port_mapper: ts_control::PortMapperOptions { + gateway_override: Some(("192.168.1.1".parse().unwrap(), None)), + advertise_nonpublic_external: false, + renew_while_idle: true, + }, wire_ingress: true, transport_mode: ts_control::TransportMode::Tun(ts_control::TunConfig { name: Some("tailscale0".to_owned()), @@ -849,6 +876,10 @@ mod tests { Some(41641), "wireguard_listen_port crosses the boundary" ); + assert!(control.disable_port_mapper, "disable_port_mapper crosses"); + assert!(control.port_mapper.renew_while_idle); + assert!(!control.port_mapper.advertise_nonpublic_external); + assert!(control.port_mapper.gateway_override.is_some()); assert!(control.wire_ingress); assert_eq!(control.advertise_services, vec!["svc:samba".to_owned()]); assert_eq!( @@ -955,6 +986,20 @@ mod tests { /// The WireGuard listen port defaults to `None` (OS-chosen ephemeral, today's behavior) and /// crosses the control boundary unchanged. A daemon that wants Go's `--port 41641` sets it /// explicitly; the engine never pins a port by default. + /// The library default is Go's: the port mapper is on, advertises everything and renews + /// lazily. + #[test] + fn port_mapper_defaults_are_go_like() { + let cfg = Config::default(); + assert!(!cfg.disable_port_mapper); + assert!(cfg.port_mapper.advertise_nonpublic_external); + assert!(!cfg.port_mapper.renew_while_idle); + assert!(cfg.port_mapper.gateway_override.is_none()); + let control: ts_control::Config = (&cfg).into(); + assert!(!control.disable_port_mapper); + assert_eq!(control.port_mapper, cfg.port_mapper); + } + #[test] fn from_config_default_wireguard_listen_port_is_none() { let cfg = Config::default(); diff --git a/src/lib.rs b/src/lib.rs index dc719ff4..488c495e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -195,8 +195,8 @@ pub use ts_runtime::taildrop::WaitingFile; #[doc(inline)] pub use ts_runtime::{ DeviceState, DnsQueryResult, ExitNodeSuggestion, FileTarget, IpnBusWatcher, NetcheckReport, - Notify, NotifyWatchOpt, RegionLatency, RegistrationError, Status, StatusNode, TkaLogEntry, - WhoIs, + Notify, NotifyWatchOpt, PortMapFacets, PortMapStatus, RegionLatency, RegistrationError, Status, + StatusNode, TkaLogEntry, WhoIs, }; /// The interactive-login URL type returned by [`Device::pop_browser_url`]. #[doc(inline)] @@ -1299,6 +1299,14 @@ impl Device { self.runtime.status().await.map_err(Into::into) } + /// The NAT port mapper's state: the NetInfo facets sent to control (with Go's netcheck + /// `portmap=` summary) and the current mapping, if any. `None` when the port mapper does not + /// run (`Config::disable_port_mapper`, `TS_DISABLE_PORTMAPPER`, or a build without the + /// `portmapper` feature). + pub fn port_map_status(&self) -> Option { + self.runtime.port_map_status() + } + /// Fetch the current Tailnet Lock (TKA) status pushed by control, if any. /// /// Returns `Ok(None)` when control has sent no `TKAInfo` (tailnet lock not in use, or no change diff --git a/tests/basic.rs b/tests/basic.rs index 2f3b0140..7db1c513 100644 --- a/tests/basic.rs +++ b/tests/basic.rs @@ -241,6 +241,13 @@ async fn make_ts_device() -> Result { // runtime (read in `Device::new`); a global `.cargo/config.toml [env]` was rejected because // it would defeat the deliberate `TS_RS_EXPERIMENT` opt-in for ordinary builds. unsafe { std::env::set_var("TS_RS_EXPERIMENT", "this_is_unstable_software") }; - - Device::new(&Config::default(), Some(ts_test_util::auth_key().unwrap())).await + // SAFETY: as above. Tests never touch the LAN gateway: the port mapper stays off (also set in + // the config below; the env knob covers every device this process builds). + unsafe { std::env::set_var("TS_DISABLE_PORTMAPPER", "1") }; + + let config = Config { + disable_port_mapper: true, + ..Config::default() + }; + Device::new(&config, Some(ts_test_util::auth_key().unwrap())).await } diff --git a/tests/tailnet_e2e_campaign.rs b/tests/tailnet_e2e_campaign.rs index 32364971..7841b6fa 100644 --- a/tests/tailnet_e2e_campaign.rs +++ b/tests/tailnet_e2e_campaign.rs @@ -33,6 +33,9 @@ fn gated() -> Option { } // SAFETY: set before any worker threads read the env; gates the experimental build at runtime. unsafe { std::env::set_var("TS_RS_EXPERIMENT", "this_is_unstable_software") }; + // SAFETY: as above. Tests never touch the LAN gateway: the port mapper stays off for every + // device this process builds. + unsafe { std::env::set_var("TS_DISABLE_PORTMAPPER", "1") }; match ts_test_util::auth_key() { Some(k) => Some(k), None => { @@ -49,7 +52,10 @@ fn is_cgnat(ip: Ipv4Addr) -> bool { async fn join(label: &str) -> Device { let auth = gated().expect("gated() checked by caller"); - let mut config = Config::default(); + let mut config = Config { + disable_port_mapper: true, + ..Config::default() + }; let suffix: u32 = rand::random(); config.requested_hostname = Some(format!("tsrs-e2e-{label}-{suffix:08x}")); let dev = timeout(JOIN_TIMEOUT, Device::new(&config, Some(auth))) @@ -249,6 +255,7 @@ async fn s6_ipv4_only_config() { let Some(auth) = gated() else { return }; let suffix: u32 = rand::random(); let config = Config { + disable_port_mapper: true, enable_ipv6: false, requested_hostname: Some(format!("tsrs-e2e-v4only-{suffix:08x}")), ..Config::default() @@ -393,6 +400,7 @@ async fn s9_wait_until_running_reports_running() { let Some(auth) = gated() else { return }; let suffix: u32 = rand::random(); let config = Config { + disable_port_mapper: true, requested_hostname: Some(format!("tsrs-e2e-waitrun-{suffix:08x}")), ..Config::default() }; diff --git a/tests/tailnet_live.rs b/tests/tailnet_live.rs index 4f2665c1..28ae27ae 100644 --- a/tests/tailnet_live.rs +++ b/tests/tailnet_live.rs @@ -64,11 +64,17 @@ async fn live_tailnet_join() { // runtime (read in `Device::new`); a global `.cargo/config.toml [env]` was rejected because // it would defeat the deliberate `TS_RS_EXPERIMENT` opt-in for ordinary builds. unsafe { std::env::set_var("TS_RS_EXPERIMENT", "this_is_unstable_software") }; + // SAFETY: as above. Tests never touch the LAN gateway: the port mapper stays off. + unsafe { std::env::set_var("TS_DISABLE_PORTMAPPER", "1") }; // Build a default config: control defaults to `https://controlplane.tailscale.com/` — real // Tailscale, exactly what we want to validate against. Use an identifiable, unique hostname so // the node is easy to spot and deregister out-of-band. - let mut config = Config::default(); + let mut config = Config { + // Tests never touch the LAN gateway. + disable_port_mapper: true, + ..Config::default() + }; let rand_suffix: u32 = rand::random(); let hostname = format!("tsrs-livetest-{rand_suffix:08x}"); config.requested_hostname = Some(hostname.clone()); diff --git a/tests/tsnet_facade_e2e.rs b/tests/tsnet_facade_e2e.rs index 7141cc61..336b5ba5 100644 --- a/tests/tsnet_facade_e2e.rs +++ b/tests/tsnet_facade_e2e.rs @@ -41,6 +41,9 @@ fn gated() -> Option { // SAFETY: set before this test spawns any worker thread that reads the env; the var is the // runtime gate for the experimental build (read in `Device::new`). unsafe { std::env::set_var("TS_RS_EXPERIMENT", "this_is_unstable_software") }; + // SAFETY: as above. Tests never touch the LAN gateway: the facade has no config field for it, + // so the port mapper is turned off with Go's env knob for every device this process builds. + unsafe { std::env::set_var("TS_DISABLE_PORTMAPPER", "1") }; match ts_test_util::auth_key() { Some(k) => Some(k), None => { diff --git a/ts_control/src/config.rs b/ts_control/src/config.rs index af26dfa6..7896b532 100644 --- a/ts_control/src/config.rs +++ b/ts_control/src/config.rs @@ -8,6 +8,36 @@ lazy_static::lazy_static! { pub static ref DEFAULT_CONTROL_SERVER: Url = Url::parse("https://controlplane.tailscale.com/").unwrap(); } +/// The port mapper's options (NAT-PMP/PCP; Go `net/portmapper`). Transport-only, like the other +/// dataplane fields: `ts_control` never reads it and does not depend on the port mapper. +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(default)] +pub struct PortMapperOptions { + /// Use this gateway (and optionally this host address on its subnet) instead of discovering + /// it from the routing table. It replaces discovery only: it is validated (private, on-link, + /// a usable interface) exactly like a discovered one. + pub gateway_override: Option<(core::net::Ipv4Addr, Option)>, + /// Advertise an external address that is private, shared (CGNAT) or otherwise not publicly + /// reachable. `true` (the default) is Go's behaviour; with `false` such a mapping is still kept + /// and renewed but never offered to peers, and `HavePortMap` stays false. + pub advertise_nonpublic_external: bool, + /// Renew the mapping when it is due even while the datapath is idle, as long as the node has + /// peers. `false` (the default) is Go's behaviour: renewal rides on activity (the periodic + /// ReSTUN and other kicks), so an idle node's mapping may expire. With no peers a mapping is + /// never renewed either way. + pub renew_while_idle: bool, +} + +impl Default for PortMapperOptions { + fn default() -> Self { + Self { + gateway_override: None, + advertise_nonpublic_external: true, + renew_while_idle: false, + } + } +} + /// Upstream-proxy wire protocol for [`ExitProxyConfig`]. Mirrors `ts_forwarder::ProxyScheme`; /// kept as a separate type here because `ts_control` must not depend on `ts_forwarder` (the /// runtime converts between them at the boundary). @@ -398,6 +428,17 @@ pub struct Config { #[serde(default)] pub wireguard_listen_port: Option, + /// Turn the NAT port mapper (NAT-PMP/PCP) off: no client, no driver, no socket, and no + /// port-mapping NetInfo facets (Go `magicsock.Options.DisablePortMapper`). Defaults to `false` + /// (on, as in Go). `TS_DISABLE_PORTMAPPER` in the environment has the same effect. Like the + /// other dataplane fields it is not read inside `ts_control`. + #[serde(default)] + pub disable_port_mapper: bool, + + /// The port mapper's options. See [`PortMapperOptions`]. + #[serde(default)] + pub port_mapper: PortMapperOptions, + /// WireGuard persistent-keepalive interval applied to every peer, or `None` to disable persistent /// keepalives (`PersistentKeepalive`; Tailscale uses 25s). /// @@ -844,6 +885,8 @@ impl Default for Config { enable_ipv6: false, network_monitor: false, wireguard_listen_port: None, + disable_port_mapper: false, + port_mapper: PortMapperOptions::default(), persistent_keepalive_interval: default_persistent_keepalive(), transport_mode: TransportMode::default(), wire_ingress: false, diff --git a/ts_control/src/lib.rs b/ts_control/src/lib.rs index b88d2dbc..9af2ef2d 100644 --- a/ts_control/src/lib.rs +++ b/ts_control/src/lib.rs @@ -36,6 +36,7 @@ mod expiry; mod hostinfo; #[cfg_attr(not(feature = "async_tokio"), expect(dead_code))] mod map_request_builder; +mod net_facets; mod node; #[cfg(feature = "async_tokio")] mod serve; @@ -60,7 +61,7 @@ pub use cert::{ #[doc(inline)] pub use config::{ Config, DEFAULT_CONTROL_SERVER, DEFAULT_PERSISTENT_KEEPALIVE, ExitProxyConfig, ExitProxyScheme, - LocalApi, TransportMode, TunConfig, exit_node_routes, services_hash, + LocalApi, PortMapperOptions, TransportMode, TunConfig, exit_node_routes, services_hash, }; pub use control_dialer::{ControlDialer, TcpDialer, complete_connection}; pub use derp::{Map as DerpMap, Region as DerpRegion, convert_derp_map}; @@ -71,6 +72,7 @@ pub use expiry::{ FLAG_EXPIRED_PEERS_EPOCH_UNIX, FlaggedPeer, MIN_CLOCK_DELTA_SECS, PEER_KEY_EXPIRED, flag_expired_peers_epoch, }; +pub use net_facets::{PortMapFacets, StunFacts}; pub use node::{ ExitNodeSelector, Id as NodeId, Node, NodeCapMap, PeerChange, StableId as StableNodeId, TailnetAddress, UserProfile, is_tailscale_ip, validate_service_name, diff --git a/ts_control/src/map_request_builder.rs b/ts_control/src/map_request_builder.rs index 69ff873a..4ad6098a 100644 --- a/ts_control/src/map_request_builder.rs +++ b/ts_control/src/map_request_builder.rs @@ -96,6 +96,31 @@ impl<'a> MapRequestBuilder<'a> { self } + /// Set the [`NetInfo::upnp`] field (Go `NetInfo.UPnP`): whether UPnP-IGD is present on the LAN. + pub fn upnp(mut self, value: bool) -> Self { + self.net_info_mut().upnp = Some(value); + self + } + + /// Set the [`NetInfo::pmp`] field (Go `NetInfo.PMP`): whether NAT-PMP is present on the LAN. + pub fn pmp(mut self, value: bool) -> Self { + self.net_info_mut().pmp = Some(value); + self + } + + /// Set the [`NetInfo::pcp`] field (Go `NetInfo.PCP`): whether PCP is present on the LAN. + pub fn pcp(mut self, value: bool) -> Self { + self.net_info_mut().pcp = Some(value); + self + } + + /// Set the [`NetInfo::have_port_map`] field (Go `NetInfo.HavePortMap`): whether the node holds + /// an open (and here: advertised) port mapping. + pub fn have_port_map(mut self, value: bool) -> Self { + self.net_info_mut().have_port_map = Some(value); + self + } + /// Set the [`NetInfo::derp_latency`] field (inside [`MapRequest::host_info`] -> /// [`HostInfo::net_info`]). pub fn derp_latencies(mut self, value: impl IntoIterator) -> Self { diff --git a/ts_control/src/net_facets.rs b/ts_control/src/net_facets.rs new file mode 100644 index 00000000..659c606c --- /dev/null +++ b/ts_control/src/net_facets.rs @@ -0,0 +1,117 @@ +//! Transport-neutral NetInfo facts the runtime hands to control alongside its endpoints. +//! +//! Control never sees how they were learned: [`StunFacts`] come from the magicsock's reflexive +//! (STUN) set, [`PortMapFacets`] from the port mapper. Both are defined here so `ts_control` needs +//! no dependency on either crate. + +use alloc::{collections::BTreeSet, string::String}; +use core::net::SocketAddr; + +/// What the reflexive (STUN) addresses say about UDP, computed from the reflexive set **before** +/// endpoint de-duplication, so a reflexive address that equals the port-mapped one still counts. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct StunFacts { + /// Go `NetInfo.WorkingUDP`: a reflexive address was learned, so UDP reaches the internet. + pub working_udp: bool, + /// Go `NetInfo.MappingVariesByDestIP`: at least two distinct IPv4 reflexive addresses (a + /// symmetric NAT). IPv4-only, matching magicsock's `is_symmetric_nat`. + pub mapping_varies: bool, +} + +impl StunFacts { + /// The facts for this reflexive set. + pub fn from_reflexive(reflexive: &[SocketAddr]) -> Self { + let distinct_v4: BTreeSet<&SocketAddr> = reflexive.iter().filter(|a| a.is_ipv4()).collect(); + Self { + working_udp: !reflexive.is_empty(), + mapping_varies: distinct_v4.len() >= 2, + } + } +} + +/// Port-mapping facets for NetInfo. `None` means "not checked" and is never sent. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct PortMapFacets { + /// Go `NetInfo.UPnP` (never probed yet, so `None`). + pub upnp: Option, + /// Go `NetInfo.PMP`. + pub pmp: Option, + /// Go `NetInfo.PCP`. + pub pcp: Option, + /// Go `NetInfo.HavePortMap`. Here it means an **advertised** mapping, so control never sees + /// `HavePortMap` without a port-mapped endpoint. + pub have_port_map: bool, +} + +impl PortMapFacets { + /// Go's netcheck `portmap=` summary: `?` when nothing was checked, otherwise one letter per + /// protocol (`U`, `M`, `C`) that is present, nothing for one that is absent, and `_` for one + /// not checked (`conciseOptBool`). An advertised mapping prefixes `active-`. + pub fn summary(&self) -> String { + let mut s = String::new(); + if self.have_port_map { + s.push_str("active-"); + } + if self.upnp.is_none() && self.pmp.is_none() && self.pcp.is_none() { + s.push('?'); + return s; + } + for (v, c) in [(self.upnp, 'U'), (self.pmp, 'M'), (self.pcp, 'C')] { + match v { + None => s.push('_'), + Some(true) => s.push(c), + Some(false) => {} + } + } + s + } +} + +#[cfg(test)] +mod tests { + use alloc::vec::Vec; + + use super::*; + + fn addrs(list: &[&str]) -> Vec { + list.iter().map(|a| a.parse().unwrap()).collect() + } + + #[test] + fn stun_facts() { + assert_eq!(StunFacts::from_reflexive(&[]), StunFacts::default()); + let one = StunFacts::from_reflexive(&addrs(&["203.0.113.7:41641"])); + assert!(one.working_udp && !one.mapping_varies); + let two = StunFacts::from_reflexive(&addrs(&["203.0.113.7:41641", "198.51.100.9:51000"])); + assert!(two.working_udp && two.mapping_varies); + let dup = StunFacts::from_reflexive(&addrs(&["203.0.113.7:41641", "203.0.113.7:41641"])); + assert!(!dup.mapping_varies); + let v6 = StunFacts::from_reflexive(&addrs(&["[2001:db8::1]:1", "[2001:db8::2]:1"])); + assert!(v6.working_udp && !v6.mapping_varies); + } + + /// Port of netcheck `TestReportString`'s portmap cases, plus the `active-` prefix. + #[test] + fn summary_matches_go() { + let f = |upnp, pmp, pcp, have| PortMapFacets { + upnp, + pmp, + pcp, + have_port_map: have, + }; + assert_eq!(f(None, None, None, false).summary(), "?"); + assert_eq!( + f(Some(true), Some(true), Some(true), false).summary(), + "UMC" + ); + assert_eq!( + f(Some(true), Some(false), Some(true), false).summary(), + "UC" + ); + assert_eq!( + f(None, Some(true), Some(true), true).summary(), + "active-_MC" + ); + assert_eq!(f(None, Some(false), Some(false), false).summary(), "_"); + } +} diff --git a/ts_control/src/tokio/client.rs b/ts_control/src/tokio/client.rs index b70be561..b9d3af4d 100644 --- a/ts_control/src/tokio/client.rs +++ b/ts_control/src/tokio/client.rs @@ -1,7 +1,4 @@ -use alloc::{ - collections::{BTreeMap, BTreeSet}, - sync::Arc, -}; +use alloc::{collections::BTreeMap, sync::Arc}; use futures_util::{Stream, StreamExt}; use tokio::{ @@ -50,6 +47,10 @@ struct CarriedNetInfo { /// Whether the NAT maps the socket to different reflexive addr:ports per destination (symmetric /// NAT) — Go `NetInfo.MappingVariesByDestIP`. Set by `SetEndpoints`. mapping_varies_by_dest_ip: Option, + /// The port-mapping facets (Go `NetInfo.UPnP`/`PMP`/`PCP`/`HavePortMap`), set by + /// `SetEndpoints` when the runtime runs a port mapper. `None` (no port mapper) sends none of + /// them, like Go without a portmapper. + port_map: Option, } impl CarriedNetInfo { @@ -69,6 +70,21 @@ impl CarriedNetInfo { if let Some(varies) = self.mapping_varies_by_dest_ip { builder = builder.mapping_varies_by_dest_ip(varies); } + if let Some(pm) = self.port_map { + if let Some(v) = pm.upnp { + builder = builder.upnp(v); + } + if let Some(v) = pm.pmp { + builder = builder.pmp(v); + } + if let Some(v) = pm.pcp { + builder = builder.pcp(v); + } + // Go's `HavePortMap` is `omitzero`: false is never on the wire. + if pm.have_port_map { + builder = builder.have_port_map(true); + } + } builder } } @@ -224,15 +240,26 @@ impl AsyncControlClient { } /// Advertise this node's magicsock UDP endpoints (ip:port candidates) to the control server - /// so peers can learn where to attempt direct connections. + /// so peers can learn where to attempt direct connections, with the NetInfo facts that go with + /// them: `stun` (computed from the reflexive set before de-duplication) and `port_map` (`None` + /// when no port mapper runs). #[tracing::instrument(skip_all, fields(map_url = %self.map_url(), n_endpoints), level = "trace")] - pub async fn set_endpoints(&mut self, endpoints: Vec) { + pub async fn set_endpoints( + &mut self, + endpoints: Vec, + stun: crate::StunFacts, + port_map: Option, + ) { tracing::Span::current().record("n_endpoints", endpoints.len()); tracing::trace!("reporting magicsock endpoints to control server"); if let Err(e) = self .command_tx - .send(Command::SetEndpoints { endpoints }) + .send(Command::SetEndpoints { + endpoints, + stun, + port_map, + }) .await { tracing::error!(error = %e, "setting endpoints"); @@ -318,6 +345,10 @@ pub enum Command { }, SetEndpoints { endpoints: Vec, + /// `WorkingUDP` / `MappingVariesByDestIP`, from the reflexive set. + stun: crate::StunFacts, + /// `UPnP` / `PMP` / `PCP` / `HavePortMap`, or `None` without a port mapper. + port_map: Option, }, /// Re-advertise this node's routable IP prefixes (`Hostinfo.RoutableIPs`) mid-session — the wire /// half of a runtime `set_advertise_routes`. The routes travel IN the command (not read from the @@ -529,37 +560,6 @@ fn clear_reauth_url(auth_url_tx: &watch::Sender>) { }); } -/// `NetInfo.WorkingUDP`: UDP works iff the node has learned a STUN reflexive endpoint. Mirrors Go, -/// where a learned STUN endpoint is the evidence UDP reaches the internet. Pure, for unit testing. -fn net_info_working_udp(endpoints: &[ts_control_serde::Endpoint]) -> bool { - endpoints - .iter() - .any(|e| e.ty == ts_control_serde::EndpointType::Stun) -} - -/// `NetInfo.MappingVariesByDestIP`: the NAT maps the one bound socket to different reflexive -/// addr:ports per destination (symmetric NAT) iff at least two DISTINCT **IPv4** STUN reflexive -/// addresses were observed — the wire-side mirror of magicsock `MagicSock::is_symmetric_nat` -/// (`v4_reflexive.len() >= 2`, `sock.rs`). -/// -/// IPv4-only on purpose: magicsock's symmetric-NAT determinant is v4-only (its test -/// `stun4localport_ignores_ipv6_reflexive` asserts two IPv6 reflexives must NOT trip it), so counting -/// v6 reflexives here would make the wire signal disagree with the node's own NAT model — a -/// cross-component incoherence. The fork is IPv4-only anyway, so v6 reflexives are not expected, but -/// filtering keeps the two in lockstep regardless. Pure, for unit testing. -fn net_info_mapping_varies(endpoints: &[ts_control_serde::Endpoint]) -> bool { - let mut seen = BTreeSet::new(); - for e in endpoints { - if e.ty == ts_control_serde::EndpointType::Stun && e.endpoint.is_ipv4() { - seen.insert(e.endpoint); - if seen.len() >= 2 { - return true; - } - } - } - false -} - pub async fn run( state_tx: broadcast::Sender>, mut command_rx: mpsc::Receiver, @@ -898,13 +898,14 @@ async fn run_once( watchdog.set_phase(PollPhase::SetDerpHomeRegion); drop(send_map_request(req, &map_url, &h2_client).await?); }, - Command::SetEndpoints { endpoints } => { - // Derive the NAT facets from the advertised endpoint set, mirroring magicsock: - // working_udp = a STUN reflexive was learned (UDP works); mapping_varies = - // >= 2 DISTINCT reflexive addrs (symmetric NAT, `is_symmetric_nat`). Computed - // before `endpoints` is moved into the builder. - net_info.working_udp = Some(net_info_working_udp(&endpoints)); - net_info.mapping_varies_by_dest_ip = Some(net_info_mapping_varies(&endpoints)); + Command::SetEndpoints { endpoints, stun, port_map } => { + // The NAT facets come with the endpoints, computed by the runtime from the + // reflexive set before de-duplication (a reflexive address equal to the + // port-mapped one still counts): working_udp = a STUN reflexive was + // learned; mapping_varies = >= 2 DISTINCT IPv4 reflexive addrs. + net_info.working_udp = Some(stun.working_udp); + net_info.mapping_varies_by_dest_ip = Some(stun.mapping_varies); + net_info.port_map = port_map; let mut builder = with_hostinfo(MapRequestBuilder::new(node_keys)) .keep_alive(false) .omit_peers(true) @@ -1376,53 +1377,6 @@ mod tests { } } - #[test] - fn working_udp_true_iff_a_stun_endpoint_is_present() { - use ts_control_serde::EndpointType::{Local, Stun}; - // A learned STUN reflexive ⇒ UDP works. - assert!(net_info_working_udp(&[ - ep("192.168.1.2:41641", Local), - ep("203.0.113.7:41641", Stun), - ])); - // Only local endpoints ⇒ no evidence UDP reaches the internet. - assert!(!net_info_working_udp(&[ep("192.168.1.2:41641", Local)])); - // No endpoints at all. - assert!(!net_info_working_udp(&[])); - } - - #[test] - fn mapping_varies_iff_two_distinct_stun_reflexives() { - use ts_control_serde::EndpointType::{Local, Stun, Stun4LocalPort}; - // Two DISTINCT STUN reflexive addrs ⇒ symmetric NAT (mirrors is_symmetric_nat len>=2). - assert!(net_info_mapping_varies(&[ - ep("203.0.113.7:41641", Stun), - ep("198.51.100.9:51000", Stun), - ])); - // Same reflexive addr twice ⇒ NOT varying (distinct count is 1). - assert!(!net_info_mapping_varies(&[ - ep("203.0.113.7:41641", Stun), - ep("203.0.113.7:41641", Stun), - ])); - // One STUN + a non-Stun reflexive guess + a local ⇒ only 1 distinct Stun ⇒ not varying. - assert!(!net_info_mapping_varies(&[ - ep("203.0.113.7:41641", Stun), - ep("203.0.113.7:50000", Stun4LocalPort), - ep("192.168.1.2:41641", Local), - ])); - // IPv4-only, matching magicsock: two DISTINCT IPv6 STUN reflexives must NOT trip it (the - // fork's is_symmetric_nat is v4-only; counting v6 would disagree with the node's NAT model). - assert!(!net_info_mapping_varies(&[ - ep("[2001:db8::1]:41641", Stun), - ep("[2001:db8::2]:41641", Stun), - ])); - // A v4 pair still trips it even alongside v6 reflexives (the v6 ones are simply ignored). - assert!(net_info_mapping_varies(&[ - ep("[2001:db8::1]:41641", Stun), - ep("203.0.113.7:41641", Stun), - ep("198.51.100.9:51000", Stun), - ])); - } - /// THE load-bearing invariant: carried NetInfo is sent WHOLE on every request, never partial. /// After a `SetDerpHomeRegion` sets the home region, a subsequent `SetEndpoints` (which sets the /// UDP/NAT facets) must STILL carry the home region — otherwise control would see `PreferredDERP` @@ -1447,8 +1401,10 @@ mod tests { ep("203.0.113.7:41641", Stun), ep("198.51.100.9:51000", Stun), ]; - carried.working_udp = Some(net_info_working_udp(&endpoints)); - carried.mapping_varies_by_dest_ip = Some(net_info_mapping_varies(&endpoints)); + let stun = + crate::StunFacts::from_reflexive(&[endpoints[0].endpoint, endpoints[1].endpoint]); + carried.working_udp = Some(stun.working_udp); + carried.mapping_varies_by_dest_ip = Some(stun.mapping_varies); // ...and the request it builds must carry the WHOLE NetInfo (home region NOT dropped). let builder = carried.apply(MapRequestBuilder::new(&node_keys)); @@ -1471,6 +1427,56 @@ mod tests { assert_eq!(ni.mapping_varies_by_dest_ip, Some(true)); } + /// The port-mapping facets ride on every request built from the carried state (the + /// streaming re-register after a reconnect included, since the state lives in `run`); UPnP stays + /// unset; `HavePortMap` false is omitted like Go's `omitzero`; a port-map-only endpoint list does + /// not make `WorkingUDP` true. + #[allow(clippy::field_reassign_with_default)] + #[test] + fn carried_port_map_facets_ride_on_every_request() { + use ts_control_serde::EndpointType::{Local, PortMapped}; + let node_keys = ts_keys::NodeState::generate(); + let mut carried = CarriedNetInfo::default(); + // A port-map-only advertisement: no reflexive address at all. + let _endpoints = [ + ep("203.0.113.7:41641", PortMapped), + ep("192.168.1.2:41641", Local), + ]; + let stun = crate::StunFacts::from_reflexive(&[]); + carried.working_udp = Some(stun.working_udp); + carried.mapping_varies_by_dest_ip = Some(stun.mapping_varies); + carried.port_map = Some(crate::PortMapFacets { + upnp: None, + pmp: Some(true), + pcp: Some(false), + have_port_map: true, + }); + for _request in 0..3 { + let req = carried.apply(MapRequestBuilder::new(&node_keys)).build(); + let ni = req.host_info.as_ref().unwrap().net_info.as_ref().unwrap(); + assert_eq!(ni.pmp, Some(true)); + assert_eq!(ni.pcp, Some(false)); + assert_eq!(ni.have_port_map, Some(true)); + assert_eq!(ni.upnp, None, "never probed, never claimed"); + assert_eq!(ni.working_udp, Some(false)); + let json = serde_json::to_string(ni).unwrap(); + assert!(json.contains(r#""PMP":true"#) && json.contains(r#""HavePortMap":true"#)); + assert!(!json.contains("UPnP"), "{json}"); + } + carried.port_map = Some(crate::PortMapFacets { + have_port_map: false, + ..carried.port_map.unwrap() + }); + let req = carried.apply(MapRequestBuilder::new(&node_keys)).build(); + let ni = req.host_info.as_ref().unwrap().net_info.as_ref().unwrap(); + assert_eq!(ni.have_port_map, None, "false is omitted"); + // No port mapper: none of the facets. + carried.port_map = None; + let req = carried.apply(MapRequestBuilder::new(&node_keys)).build(); + let ni = req.host_info.as_ref().unwrap().net_info.as_ref().unwrap(); + assert_eq!((ni.pmp, ni.pcp, ni.have_port_map), (None, None, None)); + } + /// An all-`None` carried state applies nothing: the builder emits no NetInfo at all (wire- /// identical to a node that has not yet learned anything — preserves the pre-change behavior). #[test] diff --git a/ts_magicsock/src/endpoint.rs b/ts_magicsock/src/endpoint.rs index b7840ed8..cfca1d8d 100644 --- a/ts_magicsock/src/endpoint.rs +++ b/ts_magicsock/src/endpoint.rs @@ -5,7 +5,9 @@ //! anti-leak note): //! - the locally bound address ([`SelfEndpointType::Local`]), known the instant we bind, and //! - reflexive addresses learned from disco pong `src` echoes ([`SelfEndpointType::Stun`]), -//! which are the STUN-equivalent mappings a peer observed our traffic arriving from. +//! which are the STUN-equivalent mappings a peer observed our traffic arriving from, and +//! - a NAT port mapping the LAN gateway granted for the bound port +//! ([`SelfEndpointType::PortMapped`]), installed by the runtime's port mapper. use core::net::SocketAddr; @@ -23,6 +25,10 @@ pub enum SelfEndpointType { /// reachable. Mirrors Go's `EndpointSTUN4LocalPort`. A best-effort candidate, only emitted when /// symmetric NAT is detected. Stun4LocalPort, + /// An inbound mapping the LAN gateway granted through NAT-PMP or PCP (Go + /// `EndpointPortmapped`): the router's external address and the port it maps to our bound + /// port. The highest-priority candidate. + PortMapped, } /// One candidate address we believe we are reachable on, tagged with how we learned it. diff --git a/ts_magicsock/src/sock.rs b/ts_magicsock/src/sock.rs index f11ebc1e..7119e164 100644 --- a/ts_magicsock/src/sock.rs +++ b/ts_magicsock/src/sock.rs @@ -341,6 +341,11 @@ pub struct MagicSock { /// in `CallMeMaybe` so peers behind NAT can reach us. Learned only on this one socket — never /// a second egress. reflexive: Arc>>, + /// The NAT port mapping for this socket's port (Go `EndpointPortmapped`), with the instant its + /// local trust ends. Set by the runtime's port-mapper driver; dropped from + /// [`self_endpoints`](Self::self_endpoints) once expired, and cleared by + /// [`rebind`](Self::rebind) because the mapping targets the old socket's port. + portmapped: Mutex>, /// Outstanding STUN Binding Requests, keyed by the transaction id we sent. The value is the /// [`Instant`] we sent at, so stale transactions can be pruned (see [`STUN_TX_TTL`]). /// @@ -542,6 +547,7 @@ impl MagicSock { paths: Default::default(), addr_to_disco: Default::default(), reflexive: Default::default(), + portmapped: Mutex::new(None), stun_in_flight: Default::default(), binding_verifier: None, peer_cap_version: None, @@ -750,6 +756,9 @@ impl MagicSock { // The local NAT mapping changed: drop stale reflexive addresses and every confirmed best // path (keep candidates — see `PeerPaths::invalidate_best`). Locked disjointly, never nested. lock(&self.reflexive).clear(); + // The port mapping targeted the old socket's port. The port mapper re-installs it when the + // network and port are unchanged. + *lock(&self.portmapped) = None; { let mut paths = lock(&self.paths); for peer in paths.values_mut() { @@ -897,6 +906,13 @@ impl MagicSock { Ok(*lock(&self.bound_addr)) } + /// Whether an underlay socket is bound right now. `false` while a rebind is between closing + /// the old socket and binding the new one, and after a rebind that failed to bind (then + /// [`local_addr`](Self::local_addr) still reports the old address, which nothing listens on). + pub fn is_bound(&self) -> bool { + self.sock.borrow().is_some() + } + /// Our candidate self-endpoints: the bound local address plus every reflexive address peers /// have observed our traffic arriving from. /// @@ -912,14 +928,23 @@ impl MagicSock { /// unspecified `[::]:port`. With the default `enable_ipv6 == false` no IPv6 local candidate is /// emitted and the result is byte-for-byte the prior IPv4-only set. pub fn self_endpoints(&self) -> Vec { - let mut eps = Vec::new(); + self.self_endpoints_with_stun().0 + } + + /// [`self_endpoints`](Self::self_endpoints), plus the reflexive (STUN) addresses it was built + /// from **before** de-duplication. A reflexive address equal to the port-mapped one is listed + /// once, as [`SelfEndpointType::PortMapped`], but it is still evidence that UDP works and still + /// counts towards "mapping varies by destination"; callers that derive those facts use the + /// second element. + pub fn self_endpoints_with_stun(&self) -> (Vec, Vec) { + // Candidates are gathered per kind, then emitted in Go's priority order (portmapped, + // STUN, STUN4LocalPort, local; `magicsock.determineEndpoints`) and de-duplicated by + // address, the first (highest-priority) kind winning. + let mut locals = Vec::new(); let local = self.local_addr().ok(); if let Some(local) = local { - eps.push(SelfEndpoint { - addr: local, - ty: SelfEndpointType::Local, - }); + locals.push(local); } // Local IPv6 candidate enumeration (gated on `enable_ipv6`; default `false` ⇒ this whole @@ -955,13 +980,7 @@ impl MagicSock { if !self.is_pingable_candidate(&cand) { continue; } - if eps.iter().any(|e| e.addr == cand) { - continue; - } - eps.push(SelfEndpoint { - addr: cand, - ty: SelfEndpointType::Local, - }); + locals.push(cand); } } Err(e) => { @@ -982,40 +1001,65 @@ impl MagicSock { (all, v4) }; - for addr in &all_reflexive { - eps.push(SelfEndpoint { - addr: *addr, - ty: SelfEndpointType::Stun, - }); - } - // Hard-NAT guess (Go `EndpointSTUN4LocalPort`): behind a symmetric NAT the reflexive port a // peer learns varies per-destination and is useless to a third peer, but the router may have // a static mapping to our fixed *local* port. So advertise `(reflexive_ipv4, local_port)` as // an extra best-effort candidate. Only when symmetric NAT is detected, only for IPv4 // reflexive addresses, and only when it differs from an already-listed reflexive endpoint. + let mut guesses = Vec::new(); if self.is_symmetric_nat(&v4_reflexive) && let Some(local) = local { let local_port = local.port(); - for refl in v4_reflexive { + for refl in &v4_reflexive { let guess = SocketAddr::new(refl.ip(), local_port); - // Skip if the reflexive endpoint already uses the local port (not a symmetric remap) - // or if we'd duplicate an existing candidate. + // Skip if the reflexive endpoint already uses the local port (not a symmetric remap). if guess.port() == refl.port() { continue; } - if eps.iter().any(|e| e.addr == guess) { - continue; - } - eps.push(SelfEndpoint { - addr: guess, - ty: SelfEndpointType::Stun4LocalPort, - }); + guesses.push(guess); } } - eps + let portmapped = self.portmapped(); + + let mut eps: Vec = Vec::new(); + let ordered = portmapped + .into_iter() + .map(|a| (a, SelfEndpointType::PortMapped)) + .chain(all_reflexive.iter().map(|a| (*a, SelfEndpointType::Stun))) + .chain( + guesses + .into_iter() + .map(|a| (a, SelfEndpointType::Stun4LocalPort)), + ) + .chain(locals.into_iter().map(|a| (a, SelfEndpointType::Local))); + for (addr, ty) in ordered { + if !eps.iter().any(|e| e.addr == addr) { + eps.push(SelfEndpoint { addr, ty }); + } + } + + (eps, all_reflexive) + } + + /// Install (or with `None`, clear) the port-mapped endpoint and the instant its local trust + /// ends. The runtime's port-mapper driver calls this on every mapping change. + pub fn set_portmapped(&self, mapping: Option<(SocketAddr, Instant)>) { + *lock(&self.portmapped) = mapping; + } + + /// The port-mapped endpoint, if one is installed and its trust has not ended. + pub fn portmapped(&self) -> Option { + let mut pm = lock(&self.portmapped); + match *pm { + Some((addr, expires)) if Instant::now() < expires => Some(addr), + Some(_) => { + *pm = None; + None + } + None => None, + } } /// Whether to treat this node as behind a symmetric (endpoint-dependent-mapping) NAT, which is @@ -6976,10 +7020,14 @@ mod tests { assert_eq!(stun.len(), 1, "exactly one reflexive endpoint: {eps:?}"); // On loopback the reflexive address B observed is A's own bound address. assert_eq!(stun[0].addr, a_addr, "reflexive addr is A's loopback src"); - assert!( - eps.iter().any(|e| e.ty == SelfEndpointType::Local), - "local endpoint still present" + // The local address equals the reflexive one here, so it is listed once, as the + // higher-priority STUN kind (Go's `determineEndpoints` de-duplicates the same way). + assert_eq!( + eps.iter().filter(|e| e.addr == a_addr).count(), + 1, + "one entry per address: {eps:?}" ); + assert!(!eps.iter().any(|e| e.ty == SelfEndpointType::Local)); a_pump.abort(); b_pump.abort(); @@ -7144,6 +7192,85 @@ mod tests { } } + /// `self_endpoints` lists the port mapping first, then STUN, STUN4LocalPort and + /// local, de-duplicated by address with the higher-priority kind winning; the STUN facts keep + /// a reflexive address that equals the port mapping. + #[tokio::test] + async fn self_endpoints_priority_and_dedup() { + let sock = MagicSock::bind( + localhost(), + DiscoPrivateKey::random(), + ts_keys::NodePrivateKey::random().public_key(), + ) + .await + .unwrap(); + let local = sock.local_addr().unwrap(); + let pm: SocketAddr = "203.0.113.7:41641".parse().unwrap(); + let other: SocketAddr = "203.0.113.7:50000".parse().unwrap(); + lock(&sock.reflexive).insert(pm); + lock(&sock.reflexive).insert(other); + sock.set_portmapped(Some((pm, Instant::now() + Duration::from_secs(60)))); + let (eps, stun) = sock.self_endpoints_with_stun(); + let types: Vec = eps.iter().map(|e| e.ty).collect(); + assert_eq!(eps[0].addr, pm); + assert_eq!(eps[0].ty, SelfEndpointType::PortMapped); + assert_eq!(eps.iter().filter(|e| e.addr == pm).count(), 1, "{eps:?}"); + assert!( + eps.iter() + .any(|e| e.addr == other && e.ty == SelfEndpointType::Stun) + ); + assert_eq!(*types.last().unwrap(), SelfEndpointType::Local); + assert_eq!(eps.last().unwrap().addr, local); + // Priority: no lower kind before a higher one. + let rank = |t: &SelfEndpointType| match t { + SelfEndpointType::PortMapped => 0, + SelfEndpointType::Stun => 1, + SelfEndpointType::Stun4LocalPort => 2, + SelfEndpointType::Local => 3, + }; + assert!( + types.windows(2).all(|w| rank(&w[0]) <= rank(&w[1])), + "{types:?}" + ); + // Both reflexive addresses are still STUN evidence. + assert_eq!(stun.len(), 2); + assert!(stun.contains(&pm)); + // The CallMeMaybe gate's candidate list carries the port mapping too. + let b = DiscoPrivateKey::random(); + let mut frame = sock.seal_call_me_maybe(&b.public_key()).unwrap(); + match disco::open(&b, &mut frame).unwrap() { + Inbound::CallMeMaybe { endpoints, .. } => assert_eq!(endpoints[0], pm), + other => panic!("expected CallMeMaybe, got {other:?}"), + } + } + + /// The port mapping drops out once its trust ends, and a rebind clears it at once. + #[tokio::test] + async fn portmapped_endpoint_expires_and_rebind_clears_it() { + let sock = MagicSock::bind( + localhost(), + DiscoPrivateKey::random(), + ts_keys::NodePrivateKey::random().public_key(), + ) + .await + .unwrap(); + let pm: SocketAddr = "203.0.113.7:41641".parse().unwrap(); + sock.set_portmapped(Some((pm, Instant::now()))); + assert!(sock.portmapped().is_none(), "expired"); + assert!(sock.self_endpoints().iter().all(|e| e.addr != pm)); + sock.set_portmapped(Some((pm, Instant::now() + Duration::from_secs(60)))); + assert_eq!(sock.portmapped(), Some(pm)); + sock.rebind().await.unwrap(); + assert!(sock.portmapped().is_none(), "rebind clears it"); + sock.set_portmapped(Some((pm, Instant::now() + Duration::from_secs(60)))); + sock.set_portmapped(None); + assert!( + sock.self_endpoints() + .iter() + .all(|e| e.ty != SelfEndpointType::PortMapped) + ); + } + #[tokio::test] async fn send_without_path_is_no_path_error() { let a = MagicSock::bind( diff --git a/ts_runtime/Cargo.toml b/ts_runtime/Cargo.toml index 968d8b8b..3b691748 100644 --- a/ts_runtime/Cargo.toml +++ b/ts_runtime/Cargo.toml @@ -39,6 +39,9 @@ ts_netcheck.workspace = true # libc, never a route crate). See the `network-monitor` feature below. ts_netmon = { workspace = true, optional = true } ts_netstack_smoltcp = { workspace = true, features = ["tokio"] } +# The NAT port mapper (NAT-PMP/PCP), behind the default-on `portmapper` feature. LAN control +# sockets to the gateway's port 5351 only; never a forwarding path. +ts_portmapper = { workspace = true, optional = true } ts_overlay_router.workspace = true ts_packet.workspace = true ts_packetfilter = { workspace = true, features = ["checking-filter"] } @@ -80,7 +83,13 @@ rand.workspace = true libc = "0.2" [features] +default = ["portmapper"] ring-aead = ["ts_tunnel/ring-aead"] +# On by default (Go parity): the NAT port mapper asks the LAN gateway for an inbound UDP mapping +# with NAT-PMP or PCP so peers behind hard NATs can reach this node directly. With it off the +# driver is not compiled and the port-mapping NetInfo facets stay unset. Config +# `disable_port_mapper` and `TS_DISABLE_PORTMAPPER` turn it off at runtime. +portmapper = ["dep:ts_portmapper"] # Off by default. Enables the TUN-mode overlay transport (ts_transport_tun) so the runtime can # run the application data path over a real OS TUN interface instead of the userspace netstack. # TUN device creation needs root/CAP_NET_ADMIN; off the ring-only egress path. @@ -100,6 +109,9 @@ acme = ["ts_control/acme"] [dev-dependencies] proptest = "1.11" +# The fake NAT-PMP/PCP router and `TestCap` for the port-mapper driver tests (loopback only; the +# `test-harness` feature is never enabled by a normal dependency). +ts_portmapper = { workspace = true, features = ["test-harness"] } # Test-only: decode a real `MapResponse` JSON body so the WhoIs user-profile tests start from the # bytes control sends rather than from a hand-built domain struct. `ts_control` already depends on # it, so this pulls in no new crate. diff --git a/ts_runtime/src/control_runner.rs b/ts_runtime/src/control_runner.rs index 36f0b589..54a1da58 100644 --- a/ts_runtime/src/control_runner.rs +++ b/ts_runtime/src/control_runner.rs @@ -2144,29 +2144,35 @@ fn select_home_region( } } +/// The wire form of our endpoints, in the advertised (priority) order. +pub(crate) fn wire_endpoints(eps: &[ts_magicsock::SelfEndpoint]) -> Vec { + eps.iter() + .map(|ep| Endpoint { + endpoint: ep.addr, + ty: match ep.ty { + SelfEndpointType::Local => EndpointType::Local, + SelfEndpointType::Stun => EndpointType::Stun, + SelfEndpointType::Stun4LocalPort => EndpointType::Stun4LocalPort, + SelfEndpointType::PortMapped => EndpointType::PortMapped, + }, + }) + .collect() +} + impl Message for ControlRunner { type Reply = (); async fn handle(&mut self, msg: EndpointAdvertisement, _ctx: &mut Context) { - let endpoints: Vec = msg - .endpoints - .iter() - .map(|ep| Endpoint { - endpoint: ep.addr, - ty: match ep.ty { - SelfEndpointType::Local => EndpointType::Local, - SelfEndpointType::Stun => EndpointType::Stun, - SelfEndpointType::Stun4LocalPort => EndpointType::Stun4LocalPort, - }, - }) - .collect(); + let endpoints = wire_endpoints(&msg.endpoints); tracing::debug!( n_endpoints = endpoints.len(), "advertising endpoints to control" ); - self.client.set_endpoints(endpoints).await; + self.client + .set_endpoints(endpoints, msg.stun, msg.port_map) + .await; } } diff --git a/ts_runtime/src/derp_latency.rs b/ts_runtime/src/derp_latency.rs index 3bb2a486..1044097c 100644 --- a/ts_runtime/src/derp_latency.rs +++ b/ts_runtime/src/derp_latency.rs @@ -142,6 +142,7 @@ mod tests { enable_ipv6: false, wireguard_listen_port: None, network_monitor: false, + port_mapper: None, persistent_keepalive_interval: None, ingress_active: Arc::new(std::sync::atomic::AtomicBool::new(false)), } diff --git a/ts_runtime/src/direct.rs b/ts_runtime/src/direct.rs index f2a60423..d4c44d36 100644 --- a/ts_runtime/src/direct.rs +++ b/ts_runtime/src/direct.rs @@ -148,9 +148,28 @@ const ADVERTISE_INTERVAL: Duration = Duration::from_secs(5); /// forward to the control server so peers can learn where to attempt direct connections. /// /// All addresses originate from the single bound underlay socket — there is no second egress. -#[derive(Clone)] +/// The port-mapped endpoint is an address on the LAN gateway that forwards to that socket. +#[derive(Clone, Debug, PartialEq)] pub struct EndpointAdvertisement { + /// The endpoints in priority order (portmapped, STUN, STUN4LocalPort, local), each address + /// once. pub endpoints: Arc>, + /// `WorkingUDP` / `MappingVariesByDestIP`, from the reflexive set before de-duplication. + pub stun: ts_control::StunFacts, + /// The port-mapping facets, or `None` when no port mapper runs. + pub port_map: Option, +} + +impl EndpointAdvertisement { + /// The advertisement for the socket's current endpoints and these port-mapping facets. + pub(crate) fn current(sock: &MagicSock, port_map: Option) -> Self { + let (endpoints, reflexive) = sock.self_endpoints_with_stun(); + Self { + endpoints: Arc::new(endpoints), + stun: ts_control::StunFacts::from_reflexive(&reflexive), + port_map, + } + } } /// The IPv4 bind address for the direct underlay socket (unit tests). @@ -296,6 +315,11 @@ pub struct DirectManager { /// [`should_do_periodic_restun`]). An `AtomicBool` rather than a bus message or an actor `ask` /// because the reader is a plain task and the value is one independent bit. force_background_stun: Arc, + /// The NAT port-mapper driver's handle, when the port mapper runs (see [`crate::portmap`]). + /// Rebinds invalidate through it synchronously; `re_stun`, the first peer and a lifted + /// `only-tcp-443` kick it. + #[cfg(feature = "portmapper")] + portmap: Option, #[allow(dead_code)] tasks: JoinSet<()>, } @@ -383,7 +407,11 @@ impl DirectManager { #[message] pub async fn rebind(&self) -> Result<(), ts_magicsock::Error> { match self.sock.as_ref() { - Some(sock) => sock.rebind().await, + Some(sock) => { + let r = sock.rebind().await; + self.portmap_note_rebind(sock); + r + } None => Ok(()), } } @@ -426,7 +454,11 @@ impl DirectManager { }; // 1. Swap the socket + reset stale local mapping (clears best paths, keeps candidates). - sock.rebind().await?; + // The port mapping is invalidated (or, same network and port, re-installed) before + // anything else runs. + let rebound = sock.rebind().await; + self.portmap_note_rebind(sock); + rebound?; // 2. Re-ping all candidates now on the new socket (every best is None post-rebind, so this // re-pings everything under the normal cadence gates). A send error is non-fatal: the @@ -464,9 +496,30 @@ impl DirectManager { return Ok(()); }; self.stun_sweep_once(sock).await; + // Go's `ReSTUN` re-runs endpoint determination, which asks the port mapper too. + #[cfg(feature = "portmapper")] + if let Some(pm) = &self.portmap { + pm.kick(); + } Ok(()) } + /// Tell the port mapper about a rebind, synchronously: the port of whatever socket is bound + /// now, or that nothing is. A failed rebind does not mean "no socket": with connected sockets + /// on (`SO_REUSEPORT`) the old socket keeps serving, and its mapping stays valid. + fn portmap_note_rebind(&self, sock: &MagicSock) { + #[cfg(feature = "portmapper")] + if let Some(pm) = &self.portmap { + let port = sock + .is_bound() + .then(|| sock.local_addr().ok().map(|a| a.port())) + .flatten(); + pm.note_rebind(port); + } + #[cfg(not(feature = "portmapper"))] + let _ = sock; + } + /// One immediate STUN sweep from the bound socket, gated like the periodic prober (skip while /// there are no peers — Go's `len(peerSet)==0` stop). Best-effort: a stale/unavailable multiderp /// or an empty v4-STUN-server list just skips this round (pong-harvest still re-learns @@ -485,6 +538,27 @@ impl DirectManager { } } +/// Ask the direct manager for the port mapper's client, when it runs: the runtime releases its +/// router rules on shutdown through it and reads its state for status. +#[cfg(feature = "portmapper")] +#[derive(Debug, Clone, Copy)] +pub struct PortmapClient; + +#[cfg(feature = "portmapper")] +impl Message for DirectManager { + type Reply = Option>; + + async fn handle( + &mut self, + _: PortmapClient, + _ctx: &mut Context, + ) -> Self::Reply { + self.portmap + .as_ref() + .map(crate::portmap::PortmapHandle::mapper) + } +} + /// The disco<->node-key binding verifier installed on the [`MagicSock`] (see /// [`ts_magicsock::BindingVerifier`]). A live read of the peer db (it is replaced as netmaps /// arrive), so revocations take effect immediately. @@ -961,6 +1035,11 @@ fn stun_probe_window( /// change, so control can be told where peers may reach us directly. Only republishes on a real /// change to avoid spamming control with redundant side-band map requests. /// +/// A change is a change in the ordered endpoint list, in the STUN facts or in the port-mapping +/// facets (Go's `BasicallyEqual` for these fields). The list is published in priority order, not +/// sorted. A port-mapper state change (`port_map_rx`) wakes the loop at once instead of waiting for +/// the 5 s tick, which stays as the fallback for reflexive addresses learned from pongs. +/// /// Reflexive (STUN-equivalent) endpoints come solely from the disco pong-harvest path on the one /// bound socket (peers echo our public `src`); we deliberately do **not** run a netcheck-style /// multi-socket prober for self-endpoint discovery. Such a prober binds its own sockets (including @@ -968,39 +1047,49 @@ fn stun_probe_window( /// both a different NAT path and a potential IPv6 leak — which is why the old `ts_netcheck` /// `StunProber` was removed entirely rather than left dormant in the production binary. Pong-harvest /// is the leak-safe, parity-correct source for Tier 1. -async fn run_advertiser( +async fn run_advertiser( sock: Arc, - env: Env, + mut port_map_rx: tokio::sync::watch::Receiver>, mut shutdown: tokio::sync::watch::Receiver, -) { + publish: F, +) where + F: Fn(EndpointAdvertisement) -> Fut, + Fut: core::future::Future, +{ let mut interval = tokio::time::interval(ADVERTISE_INTERVAL); interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); - let mut last: Vec = Vec::new(); + let mut last: Option = None; + let mut port_map_open = true; while !*shutdown.borrow() { tokio::select! { _ = shutdown.changed() => break, - _ = interval.tick() => { - let mut eps = sock.self_endpoints(); - eps.sort_by_key(|e| (e.addr, e.ty as u8)); - if eps == last { - continue; - } - last = eps.clone(); - - if let Err(e) = env - .publish(EndpointAdvertisement { - endpoints: Arc::new(eps), - }) - .await - { - tracing::error!(error = %e, "publishing endpoint advertisement"); - } + _ = interval.tick() => {} + r = port_map_rx.changed(), if port_map_open => { + // The driver is gone (port mapper off or stopped): keep the tick only. + port_map_open = r.is_ok(); } } + let adv = EndpointAdvertisement::current(&sock, *port_map_rx.borrow_and_update()); + if last.as_ref() == Some(&adv) { + continue; + } + last = Some(adv.clone()); + publish(adv).await; } } +/// Whether `eps` holds a candidate a remote peer could reach across the internet: a STUN +/// reflexive address or a port mapping (Go counts `EndpointPortmapped` as one too). +fn has_reflexive_candidate(eps: &[SelfEndpoint]) -> bool { + eps.iter().any(|e| { + matches!( + e.ty, + ts_magicsock::SelfEndpointType::Stun | ts_magicsock::SelfEndpointType::PortMapped + ) + }) +} + /// Periodically send a `CallMeMaybe` over DERP to each peer that has no confirmed direct path /// yet, prompting it to disco-ping our candidate endpoints so a direct path can open. Gated on /// [`MagicSock::best_addr`] being `None`: once a path is confirmed we stop relaying to that peer, @@ -1032,10 +1121,7 @@ async fn run_call_me_maybe( // local LAN address is useless to relay over DERP. Skip the whole cadence until we // have one, so peers that can never go direct don't incur perpetual relay load. // Snapshot self_endpoints once per tick (it locks the reflexive set internally). - let have_reflexive = sock - .self_endpoints() - .iter() - .any(|e| e.ty == ts_magicsock::SelfEndpointType::Stun); + let have_reflexive = has_reflexive_candidate(&sock.self_endpoints()); if !have_reflexive { continue; } @@ -1197,6 +1283,8 @@ impl kameo::Actor for DirectManager { multiderp, stun_cursor, force_background_stun, + #[cfg(feature = "portmapper")] + portmap: None, tasks, }); } @@ -1216,10 +1304,22 @@ impl kameo::Actor for DirectManager { env.shutdown.clone(), )); tasks.spawn(run_pinger(sock.clone(), env.shutdown.clone())); + // The port mapper's facets for the advertiser. Without a port mapper the sender is + // dropped here and the advertiser keeps its tick only (facets stay `None`). + let (port_map_tx, port_map_rx) = tokio::sync::watch::channel(None); + let advertiser_env = env.clone(); tasks.spawn(run_advertiser( sock.clone(), - env.clone(), + port_map_rx, env.shutdown.clone(), + move |adv| { + let env = advertiser_env.clone(); + async move { + if let Err(e) = env.publish(adv).await { + tracing::error!(error = %e, "publishing endpoint advertisement"); + } + } + }, )); // Active STUN probing shares the one bound socket; clone the multiderp ref before it is // moved into run_call_me_maybe below. `peer_db`, the datapath idleness clock and the @@ -1256,6 +1356,35 @@ impl kameo::Actor for DirectManager { env.shutdown.clone(), )); + #[cfg(feature = "portmapper")] + let portmap = env.port_mapper.as_ref().map(|settings| { + let mapper = match &settings.factory { + Some(f) => f(), + None => crate::portmap::system_client(&settings.options), + }; + let handle = crate::portmap::PortmapHandle::new(mapper); + let peers_db = peer_db.clone(); + let gate_db = peer_db.clone(); + let activity = env.datapath_activity.clone(); + let force = force_background_stun.clone(); + tasks.spawn(crate::portmap::run_driver(crate::portmap::DriverArgs { + handle: handle.clone(), + sock: sock.clone(), + has_peers: Arc::new(move || stun_probe_should_run(&peers_db)), + restun_gate: Arc::new(move || { + should_do_periodic_restun(&gate_db, &activity, &force) + }), + port_map_tx, + renew_while_idle: settings.options.renew_while_idle, + restun_delay: stun_probe_delay, + poll: crate::portmap::POLL_INTERVAL, + shutdown: env.shutdown.clone(), + })); + handle + }); + #[cfg(not(feature = "portmapper"))] + drop(port_map_tx); + Ok(Self { sock: Some(sock), transport_id: Some(transport_id), @@ -1263,6 +1392,8 @@ impl kameo::Actor for DirectManager { multiderp: multiderp_for_field, stun_cursor, force_background_stun, + #[cfg(feature = "portmapper")] + portmap, tasks, }) } @@ -1346,6 +1477,12 @@ impl Message> for DirectManager { &self.force_background_stun, self.sock.as_deref(), ); + // `only-tcp-443` stops the port mapper's LAN sends too, synchronously and deletes + // included (Go passes `onlyTCP443` to the portmapper as its disable-all knob). + #[cfg(feature = "portmapper")] + if let (Some(node), Some(pm)) = (msg.node.as_ref(), &self.portmap) { + pm.set_only_tcp_443(node.only_tcp_443()); + } } } @@ -1493,8 +1630,21 @@ impl Message> for DirectManager { } } - let mut db = poisoned_write(&self.peer_db); - *db = Some(msg.peers.clone()); + let had_peers = stun_probe_should_run(&self.peer_db); + { + let mut db = poisoned_write(&self.peer_db); + *db = Some(msg.peers.clone()); + } + // The first peer is a reason to (re)establish the port mapping now. + #[cfg(feature = "portmapper")] + if !had_peers + && stun_probe_should_run(&self.peer_db) + && let Some(pm) = &self.portmap + { + pm.kick(); + } + #[cfg(not(feature = "portmapper"))] + let _ = had_peers; } } @@ -2402,6 +2552,96 @@ mod tests { /// the wire key, the accessor and the push — and both directions of it, because Go's /// `SetOnlyTCP443` is live: a netmap that restates the self node without the attribute is /// control withdrawing it, and UDP must come back with no restart. + /// The advertiser publishes the priority-ordered list, republishes on a change in + /// the port-mapping facets alone, not on an unchanged state, and a port-mapper change wakes it + /// well before the 5 s tick. + #[tokio::test] + async fn advertiser_follows_port_map_changes_promptly() { + let sock = Arc::new( + MagicSock::bind( + "127.0.0.1:0".parse().unwrap(), + DiscoPrivateKey::random(), + NodePrivateKey::random().public_key(), + ) + .await + .unwrap(), + ); + let (pm_tx, pm_rx) = tokio::sync::watch::channel(None); + let (_shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false); + let (out_tx, mut out_rx) = tokio::sync::mpsc::unbounded_channel(); + let task = tokio::spawn(run_advertiser(sock.clone(), pm_rx, shutdown_rx, move |a| { + let out = out_tx.clone(); + async move { + out.send(a).ok(); + } + })); + let first = out_rx.recv().await.unwrap(); + assert_eq!(first.endpoints.len(), 1); + assert_eq!(first.port_map, None); + assert!(!first.stun.working_udp); + + let wait = Duration::from_secs(1); + let pm: SocketAddr = "203.0.113.7:41641".parse().unwrap(); + let facets = ts_control::PortMapFacets { + upnp: None, + pmp: Some(true), + pcp: Some(true), + have_port_map: true, + }; + sock.set_portmapped(Some(( + pm, + std::time::Instant::now() + Duration::from_secs(60), + ))); + pm_tx.send_replace(Some(facets)); + let adv = tokio::time::timeout(wait, out_rx.recv()) + .await + .unwrap() + .unwrap(); + assert_eq!(adv.endpoints[0].addr, pm); + assert_eq!( + adv.endpoints[0].ty, + ts_magicsock::SelfEndpointType::PortMapped + ); + assert_eq!(adv.port_map, Some(facets)); + // A port mapping alone is not evidence that UDP works. + assert!(!adv.stun.working_udp); + + // Unchanged: no republish. + pm_tx.send_replace(Some(facets)); + assert!( + tokio::time::timeout(Duration::from_millis(200), out_rx.recv()) + .await + .is_err() + ); + // Facets only: republished. + pm_tx.send_replace(Some(ts_control::PortMapFacets { + pcp: Some(false), + ..facets + })); + let adv = tokio::time::timeout(wait, out_rx.recv()) + .await + .unwrap() + .unwrap(); + assert_eq!(adv.port_map.unwrap().pcp, Some(false)); + task.abort(); + } + + /// A port mapping alone opens the CallMeMaybe gate, like a STUN address; a local + /// address alone does not. + #[test] + fn call_me_maybe_gate_counts_a_port_mapping() { + let ep = |ty| SelfEndpoint { + addr: "203.0.113.7:41641".parse().unwrap(), + ty, + }; + use ts_magicsock::SelfEndpointType::{Local, PortMapped, Stun, Stun4LocalPort}; + assert!(has_reflexive_candidate(&[ep(PortMapped)])); + assert!(has_reflexive_candidate(&[ep(Local), ep(Stun)])); + assert!(!has_reflexive_candidate(&[ep(Local)])); + assert!(!has_reflexive_candidate(&[ep(Stun4LocalPort)])); + assert!(!has_reflexive_candidate(&[])); + } + #[tokio::test] async fn only_tcp_443_rides_the_self_node_to_the_underlay_socket() { let sock = MagicSock::bind( diff --git a/ts_runtime/src/env.rs b/ts_runtime/src/env.rs index deea2b49..4f94877e 100644 --- a/ts_runtime/src/env.rs +++ b/ts_runtime/src/env.rs @@ -147,6 +147,13 @@ pub struct ForwarderConfig { /// spawned (byte-for-byte today's behavior). Fixed for the life of the runtime. pub network_monitor: bool, + /// The NAT port mapper's settings, or `None` when it is off (config `disable_port_mapper`, + /// `TS_DISABLE_PORTMAPPER`, or a build without the `portmapper` feature): then no client, + /// driver or socket exists and the NetInfo facets stay unset. + /// + /// Shared test configurations set this to `None`, so no test touches the real network. + pub port_mapper: Option, + /// The WireGuard persistent-keepalive interval applied to every peer, or `None` to disable. /// /// See [`Config::persistent_keepalive_interval`](ts_control::Config::persistent_keepalive_interval). @@ -188,10 +195,60 @@ impl ForwarderConfig { enable_ipv6: config.enable_ipv6, wireguard_listen_port: config.wireguard_listen_port, network_monitor: config.network_monitor, + port_mapper: (!config.disable_port_mapper).then(|| PortMapperSettings { + options: config.port_mapper.clone(), + #[cfg(feature = "portmapper")] + factory: None, + }), persistent_keepalive_interval: config.persistent_keepalive_interval, ingress_active: config.ingress_active.clone(), } } + + /// Apply the environment knobs read once at runtime construction (Go caches its `envknob` + /// values the same way): `TS_DISABLE_PORTMAPPER` turns the port mapper off like the config + /// option. `getenv` is injected so tests do not touch the process environment. + /// + /// Values parse like Go's `strconv.ParseBool`. An unparsable value is taken as "disabled" (Go + /// refuses to start on one; the zero-I/O reading is the safe one) and logged. + pub fn apply_env_knobs(&mut self, getenv: impl Fn(&str) -> Option) { + let Some(v) = getenv("TS_DISABLE_PORTMAPPER") else { + return; + }; + let disable = match v.as_str() { + "1" | "t" | "T" | "TRUE" | "true" | "True" => true, + "0" | "f" | "F" | "FALSE" | "false" | "False" | "" => false, + other => { + tracing::warn!( + value = other, + "TS_DISABLE_PORTMAPPER is not a boolean; disabling the port mapper" + ); + true + } + }; + if disable && self.port_mapper.is_some() { + tracing::info!("port mapper disabled by TS_DISABLE_PORTMAPPER"); + self.port_mapper = None; + } + } +} + +/// The port mapper's runtime settings (see [`ForwarderConfig::port_mapper`]). +#[derive(Clone)] +pub struct PortMapperSettings { + /// The options from the config. + pub options: ts_control::PortMapperOptions, + /// Builds the client instead of the production one (tests inject a harness-backed client). + #[cfg(feature = "portmapper")] + pub factory: Option, +} + +impl core::fmt::Debug for PortMapperSettings { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("PortMapperSettings") + .field("options", &self.options) + .finish_non_exhaustive() + } } /// Convert the transport-only [`ts_control::ExitProxyConfig`] into the [`ts_forwarder::ProxyConfig`] @@ -329,6 +386,10 @@ pub struct Env { /// spawn the supervisor; not consulted again at runtime. pub network_monitor: bool, + /// The NAT port mapper's settings, or `None` when it is off. See + /// [`ForwarderConfig::port_mapper`]. Read once by the direct manager at startup. + pub port_mapper: Option, + /// The WireGuard persistent-keepalive interval applied to every peer, or `None` to disable. /// /// See [`ForwarderConfig::persistent_keepalive_interval`]. Read by the dataplane actor when it @@ -454,6 +515,7 @@ impl Env { enable_ipv6, wireguard_listen_port, network_monitor, + port_mapper, persistent_keepalive_interval, ingress_active, } = forwarding; @@ -497,6 +559,7 @@ impl Env { enable_ipv6, wireguard_listen_port, network_monitor, + port_mapper, persistent_keepalive_interval, ingress_active, funnel_ingress: Arc::new(std::sync::Mutex::new(None)), @@ -578,6 +641,7 @@ mod tests { enable_ipv6: false, wireguard_listen_port: None, network_monitor: false, + port_mapper: None, persistent_keepalive_interval: None, ingress_active: Arc::new(std::sync::atomic::AtomicBool::new(false)), } diff --git a/ts_runtime/src/forwarder_actor.rs b/ts_runtime/src/forwarder_actor.rs index a36be6e9..72304ae9 100644 --- a/ts_runtime/src/forwarder_actor.rs +++ b/ts_runtime/src/forwarder_actor.rs @@ -419,6 +419,7 @@ mod tests { enable_ipv6: false, wireguard_listen_port: None, network_monitor: false, + port_mapper: None, persistent_keepalive_interval: None, ingress_active: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), } diff --git a/ts_runtime/src/lib.rs b/ts_runtime/src/lib.rs index 2107328e..81312b9a 100644 --- a/ts_runtime/src/lib.rs +++ b/ts_runtime/src/lib.rs @@ -57,6 +57,8 @@ mod packetfilter; pub mod peer_tracker; mod peerapi; mod peerapi_doh; +#[cfg(feature = "portmapper")] +mod portmap; mod route_updater; mod security; /// Stored Serve config + accept-loop runtime (`tsnet`'s `Get/SetServeConfig` + serving runtime). @@ -80,6 +82,7 @@ pub use exit_node_suggest::{ExitNodeSuggestion, SuggestExitNodeError}; pub use ipn_bus::{IpnBusWatcher, Notify, NotifyWatchOpt}; pub use status::{FileTarget, NetcheckReport, RegionLatency, Status, StatusNode, WhoIs}; pub use tka_sync::TkaLogEntry; +pub use ts_control::{PortMapFacets, StunFacts}; pub use ts_dataplane::{CaptureHook, CapturePath}; use crate::peer_tracker::PeerTracker; @@ -166,6 +169,52 @@ pub struct Runtime { #[cfg(feature = "network-monitor")] #[allow(dead_code)] netmon_supervisor: Option>, + /// The NAT port mapper's client, when it runs, behind its release guard: + /// [`graceful_shutdown`](Self::graceful_shutdown) closes it (bounded, deletes acknowledged), + /// and dropping the guard, which every drop of the runtime does, sends one unacknowledged + /// delete per rule with no runtime needed (a no-op after the close). + #[cfg(feature = "portmapper")] + portmap: portmap::PortmapRelease, +} + +/// The port mapper's state, for status lines (Go's netcheck `portmap=` plus the mapping). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PortMapStatus { + /// The NetInfo facets as sent to control. + pub facets: ts_control::PortMapFacets, + /// The current mapping's external address, if one is held (advertised or not). + pub external: Option, + /// The protocol holding it (`"pmp"` or `"pcp"`). + pub proto: Option<&'static str>, + /// Whether the mapping is offered to peers. + pub advertised: bool, +} + +impl PortMapStatus { + /// Go's `portmap=` summary, e.g. `active-_MC`. + pub fn summary(&self) -> String { + self.facets.summary() + } +} + +/// How long [`Runtime::graceful_shutdown`] gives the port mapper to release its router rules, at +/// most (always within the caller's timeout). +#[cfg(feature = "portmapper")] +const PORTMAP_RELEASE_BUDGET: Duration = Duration::from_secs(1); + +/// The port mapper's share of a shutdown with `timeout`: at most [`PORTMAP_RELEASE_BUDGET`], never +/// more than `timeout`. +#[cfg(feature = "portmapper")] +fn portmap_budget(timeout: Option) -> Duration { + timeout.map_or(PORTMAP_RELEASE_BUDGET, |t| t.min(PORTMAP_RELEASE_BUDGET)) +} + +/// The first step of [`Runtime::graceful_shutdown`]: close the port mapper within its budget. +#[cfg(feature = "portmapper")] +async fn close_portmap(pm: Option<&Arc>, timeout: Option) { + if let Some(pm) = pm { + pm.close(portmap_budget(timeout)).await; + } } impl Runtime { @@ -182,11 +231,10 @@ impl Runtime { // `new_with_runtime_txs` returns each `Sender` (mutation capability) grouped in `pref_cells` // so they are retained privately on the `Runtime`, while only the `Receiver`s (the readers' // contract) live on the cloned `Env`. Initial values come from `ForwarderConfig`. - let (env, pref_cells) = Env::new_with_runtime_txs( - keys, - shutdown_rx, - env::ForwarderConfig::from_control_config(&config), - ); + // Environment knobs are read once, here (Go caches its `envknob`s too). + let mut forwarding = env::ForwarderConfig::from_control_config(&config); + forwarding.apply_env_knobs(|k| std::env::var(k).ok()); + let (env, pref_cells) = Env::new_with_runtime_txs(keys, shutdown_rx, forwarding); let mut security = security::SecurityGuard::new(shutdown_tx.clone()); security.watch(&env.bus); @@ -237,6 +285,8 @@ impl Runtime { let direct = DirectManager::spawn((env.clone(), dataplane.clone(), multiderp.clone())); security.watch(&direct); security.ready(&direct).await?; + #[cfg(feature = "portmapper")] + let portmap = portmap::PortmapRelease::new(direct.ask(direct::PortmapClient).await?); // Spawn the forwarder before the route updater. Its `on_start` builds the forwarder // netstack, enables any-IP acceptance, and starts the per-port accept loops synchronously, @@ -505,9 +555,32 @@ impl Runtime { taildrop_reaper, #[cfg(feature = "network-monitor")] netmon_supervisor, + #[cfg(feature = "portmapper")] + portmap, }) } + /// The port mapper's state, or `None` when it does not run (disabled, or a build without the + /// `portmapper` feature). + pub fn port_map_status(&self) -> Option { + #[cfg(feature = "portmapper")] + { + let pm = self.portmap.get()?; + let state = pm.subscribe().borrow().clone(); + Some(PortMapStatus { + facets: portmap::facets(&state), + external: state.mapping.map(|m| m.external), + proto: state.mapping.map(|m| match m.proto { + ts_portmapper::oneshot::Proto::Pmp => "pmp", + ts_portmapper::oneshot::Proto::Pcp => "pcp", + }), + advertised: state.advertised().is_some(), + }) + } + #[cfg(not(feature = "portmapper"))] + None + } + /// Register a fallback TCP handler consulted for every inbound TCP flow that matches no /// explicit listener (`tsnet.Server.RegisterFallbackTCPHandler` parity). /// @@ -1474,8 +1547,16 @@ impl Runtime { /// more violently and with possible resource leaks. pub async fn graceful_shutdown(self, timeout: Option) -> bool { self.shutdown.send_replace(true); + // The port mapper's release is the first step of the timed shutdown (`close_portmap`): + // `close` stops new mapping work before it awaits anything, then deletes its router rules + // over its own LAN sockets (never the underlay socket). Its budget is at most 1 s and + // never more than `timeout`, so the whole shutdown stays within `timeout`. + async fn _shutdown_all(runtime: Runtime, timeout: Option) { + #[cfg(feature = "portmapper")] + runtime.portmap.close(timeout).await; + #[cfg(not(feature = "portmapper"))] + let _ = timeout; - async fn _shutdown_all(runtime: Runtime) { // See the note in `Drop` for why we only need to stop these actors to bring down the // whole runtime. @@ -1490,7 +1571,7 @@ impl Runtime { ]; } - let fut = _shutdown_all(self); + let fut = _shutdown_all(self, timeout); match timeout { Some(timeout) => tokio::time::timeout(timeout, fut).await.is_ok(), @@ -1504,6 +1585,12 @@ impl Runtime { impl Drop for Runtime { fn drop(&mut self) { + // Best-effort release of the port mapper's router rules first: one unacknowledged delete + // per rule, no runtime and no reply wait. A no-op after `graceful_shutdown` closed it. + // (The guard does the same when it is dropped, so no path out of here skips it.) + #[cfg(feature = "portmapper")] + self.portmap.release_now(); + // Stop the taildrop reaper so it cannot outlive the runtime (the `reauth_bridge` pattern). It // also self-exits when `shutdown` flips below, but aborting is immediate and covers the // already-shutdown early-return path too. diff --git a/ts_runtime/src/netmon.rs b/ts_runtime/src/netmon.rs index afe6ecac..efe6c82a 100644 --- a/ts_runtime/src/netmon.rs +++ b/ts_runtime/src/netmon.rs @@ -264,6 +264,7 @@ mod tests { enable_ipv6: false, wireguard_listen_port: None, network_monitor: true, + port_mapper: None, persistent_keepalive_interval: None, ingress_active: Arc::new(std::sync::atomic::AtomicBool::new(false)), }, diff --git a/ts_runtime/src/packetfilter.rs b/ts_runtime/src/packetfilter.rs index 470cae5d..e4ff1e0c 100644 --- a/ts_runtime/src/packetfilter.rs +++ b/ts_runtime/src/packetfilter.rs @@ -361,6 +361,7 @@ mod unlocked_node_tests { enable_ipv6: false, wireguard_listen_port: None, network_monitor: false, + port_mapper: None, persistent_keepalive_interval: None, ingress_active: Arc::new(std::sync::atomic::AtomicBool::new(false)), } diff --git a/ts_runtime/src/peer_tracker/mod.rs b/ts_runtime/src/peer_tracker/mod.rs index 709bea30..ddc2b767 100644 --- a/ts_runtime/src/peer_tracker/mod.rs +++ b/ts_runtime/src/peer_tracker/mod.rs @@ -2130,6 +2130,7 @@ pub(crate) mod tka_tests { enable_ipv6: false, wireguard_listen_port: None, network_monitor: false, + port_mapper: None, persistent_keepalive_interval: None, ingress_active: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }, diff --git a/ts_runtime/src/peerapi_doh.rs b/ts_runtime/src/peerapi_doh.rs index 43fb1f53..f5e6791c 100644 --- a/ts_runtime/src/peerapi_doh.rs +++ b/ts_runtime/src/peerapi_doh.rs @@ -1474,6 +1474,7 @@ mod tests { enable_ipv6: false, wireguard_listen_port: None, network_monitor: false, + port_mapper: None, persistent_keepalive_interval: None, ingress_active: Arc::new(std::sync::atomic::AtomicBool::new(false)), } diff --git a/ts_runtime/src/portmap.rs b/ts_runtime/src/portmap.rs new file mode 100644 index 00000000..be163643 --- /dev/null +++ b/ts_runtime/src/portmap.rs @@ -0,0 +1,1090 @@ +//! The NAT port-mapper driver: runs the [`ts_portmapper`] lifecycle client for the direct +//! underlay socket and turns its state into a port-mapped endpoint and NetInfo facets. +//! +//! Spawned by [`DirectManager`](crate::direct::DirectManager) on start unless the port mapper is +//! disabled (config `disable_port_mapper`, `TS_DISABLE_PORTMAPPER`); with it disabled no client, +//! driver or socket exists. The client's sockets are LAN control sockets to the gateway's +//! NAT-PMP/PCP port only: they never carry forwarded traffic and are not a `RealDialer` path. +//! +//! What wakes the driver: +//! +//! - a 10 s poll, independent of the STUN idle gate, that re-reads the default route (a routing +//! table read, no packet) and compares the wall clock with the monotonic clock to notice a +//! resume or a clock jump; a network found or changed, or a resume, kicks a round; +//! - orphan cleanup/recovery deadlines, independent of peers/activity, with fresh owning-network +//! validation and per-rule cooldowns; +//! - the mapping's local expiry, whatever the activity, so an expired endpoint is withdrawn even +//! on an idle, peerless node; +//! - with `renew_while_idle` and at least one peer, the renewal point and the retry after a failed +//! or refused attempt (its backoff); +//! - the periodic ReSTUN cadence (20–26 s) while Go's `shouldDoPeriodicReSTUN` holds, and kicks +//! (start, first peer, rebind, `re_stun`, `only-tcp-443` lifted). +//! +//! The release on shutdown is done by the runtime through the client's [`PortMapper`] handle, not +//! here. + +use core::{ + net::SocketAddr, + sync::atomic::{AtomicBool, Ordering}, + time::Duration, +}; +use std::sync::Arc; + +use tokio::{ + sync::{Notify, watch}, + time::Instant, +}; +use ts_control::PortMapFacets; +use ts_magicsock::MagicSock; +use ts_portmapper::{DisableReason, PortMapper, PortmapState}; + +/// Builds the port-mapper client (tests inject a harness-backed one). +pub type MapperFactory = Arc Arc + Send + Sync>; + +/// How often the network identity and the clocks are checked. +pub(crate) const POLL_INTERVAL: Duration = Duration::from_secs(10); +/// The least time between two timer-driven ticks, so a stale instant can never spin the loop. +const MIN_WAKE_SPACING: Duration = Duration::from_secs(1); + +/// The production client for `options`. In this crate's own tests the gateway lookup is a stub +/// that panics: a test that turns the mapper on without injecting a harness fails loudly instead +/// of probing the real network. +pub(crate) fn system_client(options: &ts_control::PortMapperOptions) -> Arc { + let gateway_override = options + .gateway_override + .map(|(gw, self_ip)| ts_portmapper::gateway::GatewayOverride { gw, self_ip }); + #[cfg_attr(test, allow(unused_mut))] + let mut cfg = ts_portmapper::Config::system(gateway_override); + cfg.advertise_nonpublic_external = options.advertise_nonpublic_external; + #[cfg(test)] + { + cfg.lookup = Arc::new(tests::NeverLookup); + } + tracing::debug!("port mapper client built"); + #[cfg(test)] + tests::CLIENTS_BUILT.fetch_add(1, Ordering::SeqCst); + let (client, _rx) = ts_portmapper::Client::new(cfg); + Arc::new(client) +} + +/// The driver's handle, held by the direct manager: synchronous invalidation on rebind, kicks, +/// and the `only-tcp-443` switch. +#[derive(Clone)] +pub(crate) struct PortmapHandle { + mapper: Arc, + kick: Arc, + only_tcp_443: Arc, +} + +impl PortmapHandle { + pub(crate) fn new(mapper: Arc) -> Self { + Self { + mapper, + kick: Arc::new(Notify::new()), + only_tcp_443: Arc::new(AtomicBool::new(false)), + } + } + + /// The client, for the runtime's shutdown release and status. + pub(crate) fn mapper(&self) -> Arc { + self.mapper.clone() + } + + /// Ask the driver for a round (cleanup, probe, mapping) soon. + pub(crate) fn kick(&self) { + self.kick.notify_one(); + } + + /// The underlay socket was rebound (`Some(port)`) or the rebind left none (`None`). + /// Synchronous: the mapping is invalidated (or re-installed) before this returns. + pub(crate) fn note_rebind(&self, port: Option) { + self.mapper.note_rebind(port); + self.kick(); + } + + /// Control's `only-tcp-443`: on, every LAN send stops at once (deletes included); off again, + /// leftovers are deleted and mapping resumes. + pub(crate) fn set_only_tcp_443(&self, on: bool) { + if self.only_tcp_443.swap(on, Ordering::SeqCst) == on { + return; + } + if on { + self.mapper.disable(DisableReason::OnlyTcp443); + } else { + self.mapper.enable(); + self.kick(); + } + } +} + +/// The runtime's hold on the port mapper for shutdown: [`close`](Self::close) within the shutdown +/// budget, and on drop the best-effort release (one unacknowledged delete per rule, no runtime +/// needed; a no-op after the close). Holding the client only through this guard means no way of +/// dropping the runtime can skip the release. +pub(crate) struct PortmapRelease(Option>); + +impl PortmapRelease { + pub(crate) fn new(mapper: Option>) -> Self { + Self(mapper) + } + + /// The client, for status. + pub(crate) fn get(&self) -> Option<&Arc> { + self.0.as_ref() + } + + /// The first step of a graceful shutdown: close within [`crate::portmap_budget`]. + pub(crate) async fn close(&self, timeout: Option) { + crate::close_portmap(self.0.as_ref(), timeout).await; + } + + /// The `Drop`-path release, now. + pub(crate) fn release_now(&mut self) { + if let Some(pm) = self.0.take() { + pm.release_blocking_best_effort(); + } + } +} + +impl Drop for PortmapRelease { + fn drop(&mut self) { + self.release_now(); + } +} + +/// The NetInfo facets for a published state. `HavePortMap` means an advertised mapping. +pub(crate) fn facets(state: &PortmapState) -> PortMapFacets { + PortMapFacets { + upnp: state.facets.upnp, + pmp: state.facets.pmp, + pcp: state.facets.pcp, + have_port_map: state.advertised().is_some(), + } +} + +/// What the driver reads from the rest of the direct manager. +pub(crate) struct DriverArgs { + pub handle: PortmapHandle, + pub sock: Arc, + /// Whether the node has at least one peer. + pub has_peers: Arc bool + Send + Sync>, + /// Go's `shouldDoPeriodicReSTUN`: peers and a datapath not idle (or control's override). + pub restun_gate: Arc bool + Send + Sync>, + /// Where the facets go (the advertiser wakes on every send). + pub port_map_tx: watch::Sender>, + /// Renew at the renewal point while idle, as long as there are peers. + pub renew_while_idle: bool, + /// The ReSTUN cadence (jittered 20–26 s in production). + pub restun_delay: fn() -> Duration, + /// The network and clock poll interval ([`POLL_INTERVAL`] in production). + pub poll: Duration, + pub shutdown: watch::Receiver, +} + +/// Install the published state on the socket and hand the facets to the advertiser. +fn apply_state(a: &DriverArgs, state: &PortmapState) { + let mapper = &a.handle.mapper; + let pm = state.advertised().map(|m| { + ( + SocketAddr::V4(m.external), + mapper.to_real(m.expires_at).into_std(), + ) + }); + a.sock.set_portmapped(pm); + a.port_map_tx.send_replace(Some(facets(state))); +} + +/// One activity-driven round. +async fn tick(a: &DriverArgs, warned_port_zero: &mut bool) { + if a.handle.only_tcp_443.load(Ordering::SeqCst) || a.sock.only_tcp_443() { + return; + } + // Nothing bound (a rebind's gap, or a rebind that failed to bind): the cached address has no + // socket behind it, so it is not mapped. The rebind already told the client. + if !a.sock.is_bound() { + return; + } + let mapper = &a.handle.mapper; + let port = a.sock.local_addr().map(|addr| addr.port()).unwrap_or(0); + match mapper.set_local_port(port) { + Ok(()) => *warned_port_zero = false, + Err(e) => { + if !*warned_port_zero { + tracing::warn!(error = %e, "portmapper: underlay socket has no port; not mapping"); + *warned_port_zero = true; + } + return; + } + } + mapper.tick().await; +} + +fn later(t: Option, floor: Instant) -> Option { + t.map(|t| t.max(floor)) +} + +/// Run the driver until shutdown. See the module docs. +pub(crate) async fn run_driver(mut a: DriverArgs) { + let mapper = a.handle.mapper.clone(); + let mut state_rx = mapper.subscribe(); + apply_state(&a, &state_rx.borrow_and_update().clone()); + let mut poll = tokio::time::interval(a.poll); + poll.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + let mut restun = Box::pin(tokio::time::sleep((a.restun_delay)())); + let mut warned_port_zero = false; + let mut last_wake = Instant::now() - MIN_WAKE_SPACING; + a.handle.kick(); + + while !*a.shutdown.borrow() { + let sched = mapper.schedule(); + let floor = last_wake + MIN_WAKE_SPACING; + let cleanup = later(sched.cleanup_due.map(|t| mapper.to_real(t)), floor); + let expiry = later(sched.expires_at.map(|t| mapper.to_real(t)), floor); + let idle_renewal = a.renew_while_idle && (a.has_peers)(); + let renew = sched + .renewal_due + .filter(|_| idle_renewal) + .map(|t| mapper.to_real(t)); + // A wanted start refused only by spacing is retried when allowed; a failure backoff is + // retried by the timer only when idle renewal is on (otherwise activity retries it). + let retry = sched + .retry_at + .filter(|_| !sched.backing_off || idle_renewal) + .map(|t| mapper.to_real(t)); + let wake = later( + match (renew, retry) { + (Some(x), Some(y)) => Some(x.min(y)), + (x, y) => x.or(y), + }, + floor, + ); + + tokio::select! { + biased; + _ = a.shutdown.changed() => break, + r = state_rx.changed() => { + if r.is_err() { + break; + } + let state = state_rx.borrow_and_update().clone(); + apply_state(&a, &state); + } + _ = poll.tick() => { + // A network found or changed is a link change: Go re-runs endpoint discovery + // (and so the port mapper) for it, idle or not. + let found = mapper.check_network().await == ts_portmapper::NetCheck::Changed; + if mapper.check_clock() || found { + a.handle.kick(); + } + mapper.reap(); + } + _ = tokio::time::sleep_until(cleanup.unwrap_or_else(Instant::now)), if cleanup.is_some() => { + last_wake = Instant::now(); + mapper.cleanup().await; + // Retired rules may have freed admission for a wanted mapping. + a.handle.kick(); + } + _ = tokio::time::sleep_until(expiry.unwrap_or_else(Instant::now)), if expiry.is_some() => { + last_wake = Instant::now(); + mapper.expire_if_due(); + } + _ = tokio::time::sleep_until(wake.unwrap_or_else(Instant::now)), if wake.is_some() => { + last_wake = Instant::now(); + tick(&a, &mut warned_port_zero).await; + } + _ = &mut restun => { + restun.set(tokio::time::sleep((a.restun_delay)())); + if (a.restun_gate)() { + tick(&a, &mut warned_port_zero).await; + } + } + _ = a.handle.kick.notified() => { + tick(&a, &mut warned_port_zero).await; + } + } + } + a.sock.set_portmapped(None); +} + +#[cfg(test)] +pub(crate) mod tests { + use core::sync::atomic::AtomicUsize; + + use ts_keys::{DiscoPrivateKey, NodePrivateKey}; + use ts_portmapper::{ + Client, Gateway, GatewayLookup, LookupError, PortMapper, + oneshot::ProtoSet, + testing::{FakeRouter, ManualClock, RouterOptions, ScriptedLookup, TestCap, TestCaps}, + }; + + use super::*; + + /// Clients built by [`system_client`] in this test binary. + pub(crate) static CLIENTS_BUILT: AtomicUsize = AtomicUsize::new(0); + /// Lookups attempted through [`NeverLookup`]. + pub(crate) static NEVER_LOOKUPS: AtomicUsize = AtomicUsize::new(0); + + /// The lookup `system_client` gets in this crate's tests: counting, then panicking. + pub(crate) struct NeverLookup; + + impl GatewayLookup for NeverLookup { + fn lookup(&self) -> Result, LookupError> { + NEVER_LOOKUPS.fetch_add(1, Ordering::SeqCst); + panic!("a ts_runtime test reached the real gateway lookup"); + } + } + + pub(crate) const GW: core::net::Ipv4Addr = core::net::Ipv4Addr::new(192, 168, 1, 1); + + pub(crate) fn gateway() -> Gateway { + Gateway { + gw: GW, + self_ip: core::net::Ipv4Addr::LOCALHOST, + prefix_len: 24, + if_index: 4, + if_name: "en0".to_owned(), + } + } + + /// A harness-backed client: a fake router on loopback behind a `TestCap` for `GW`. + pub(crate) async fn harness_client( + router: RouterOptions, + advertise_nonpublic: bool, + ) -> ( + FakeRouter, + Client, + Arc, + Arc, + ) { + let router = FakeRouter::start(router).await.unwrap(); + let caps = TestCaps::new(); + caps.add(GW, router.endpoint()); + let lookup = ScriptedLookup::new(gateway()); + let clock = ManualClock::new(); + let (client, _rx) = Client::new(ts_portmapper::Config { + lookup: lookup.clone(), + caps, + clock: clock.clone(), + protos: ProtoSet::BOTH, + advertise_nonpublic_external: advertise_nonpublic, + lifetime: ts_portmapper::client::requested_lifetime(), + }); + (router, client, lookup, clock) + } + + struct Driver { + handle: PortmapHandle, + sock: Arc, + peers: Arc, + restun: Arc, + facets: watch::Receiver>, + shutdown: watch::Sender, + task: tokio::task::JoinHandle<()>, + } + + fn fast_restun() -> Duration { + Duration::from_millis(100) + } + + fn never_restun() -> Duration { + Duration::from_secs(3600) + } + + /// The test poll: fast, so a manual-clock jump is noticed quickly. + const POLL: Duration = Duration::from_millis(100); + + async fn driver( + client: Arc, + renew_while_idle: bool, + restun_delay: fn() -> Duration, + ) -> Driver { + let sock = Arc::new( + MagicSock::bind( + "127.0.0.1:0".parse().unwrap(), + DiscoPrivateKey::random(), + NodePrivateKey::random().public_key(), + ) + .await + .unwrap(), + ); + let handle = PortmapHandle::new(client); + let peers = Arc::new(AtomicBool::new(true)); + let restun = Arc::new(AtomicBool::new(false)); + let (port_map_tx, facets) = watch::channel(None); + let (shutdown, shutdown_rx) = watch::channel(false); + let (p, r) = (peers.clone(), restun.clone()); + let task = tokio::spawn(run_driver(DriverArgs { + handle: handle.clone(), + sock: sock.clone(), + has_peers: Arc::new(move || p.load(Ordering::SeqCst)), + restun_gate: Arc::new(move || r.load(Ordering::SeqCst)), + port_map_tx, + renew_while_idle, + restun_delay, + poll: POLL, + shutdown: shutdown_rx, + })); + Driver { + handle, + sock, + peers, + restun, + facets, + shutdown, + task, + } + } + + async fn until(what: &str, f: impl Fn() -> bool) { + let deadline = std::time::Instant::now() + Duration::from_secs(5); + while !f() { + assert!(std::time::Instant::now() < deadline, "timed out: {what}"); + tokio::time::sleep(Duration::from_millis(10)).await; + } + } + + impl Driver { + async fn stop(self) { + self.shutdown.send_replace(true); + self.task.await.unwrap(); + assert!(self.sock.portmapped().is_none(), "withdrawn on stop"); + } + } + + /// The driver maps on start, installs the endpoint on the socket with the facets, and + /// withdraws it at expiry with no ReSTUN tick and no peer. + #[tokio::test] + async fn driver_maps_then_withdraws_at_expiry_while_idle_and_peerless() { + let (router, client, _lookup, _clock) = + harness_client(RouterOptions::default(), true).await; + router.set_lifetime_override(Some(2)); + let d = driver(Arc::new(client.clone()), false, never_restun).await; + d.peers.store(false, Ordering::SeqCst); + until("mapped", || d.sock.portmapped().is_some()).await; + let pm = d.sock.portmapped().unwrap(); + assert_eq!(pm.ip().to_string(), "203.0.113.7"); + let f = d.facets.borrow().unwrap(); + assert!(f.have_port_map && f.pmp == Some(true)); + assert_eq!( + d.sock.self_endpoints()[0].ty, + ts_magicsock::SelfEndpointType::PortMapped + ); + // No ReSTUN, no peers, lazy renewal: nothing renews; the expiry timer withdraws it. + until("withdrawn", || { + d.sock.portmapped().is_none() && !d.facets.borrow().unwrap().have_port_map + }) + .await; + assert_eq!(router.counters().num_pmp_map_recv, 1, "never renewed"); + d.stop().await; + client.close(Duration::from_secs(1)).await; + router.shutdown().await; + } + + /// Exhausted unknown ownership must recover with no peers, ReSTUN or idle renewal. + #[tokio::test] + async fn cleanup_recovers_unknown_grant_while_idle_and_peerless() { + use ts_portmapper::testing::RequestKind; + let (router, client, _lookup, clock) = harness_client(RouterOptions::default(), true).await; + router.knobs_for(Some(RequestKind::PmpMap)); + router.commit_then_drop_next(1000); + router.set_drop_deletes(true); + let d = driver(Arc::new(client.clone()), false, never_restun).await; + d.peers.store(false, Ordering::SeqCst); + until("unknown grant settled", || { + client.owned_rules() == 1 && !client.attempt_in_flight() && client.reap() == 0 + }) + .await; + for _ in 0..4 { + client.cleanup().await; + } + until("initial deletes exhausted", || { + client + .cleanup_snapshot() + .iter() + .all(|e| e.delete_tries >= 3 && !e.deleting) + }) + .await; + assert!(!client.have_mapping()); + router.commit_then_drop_next(0); + router.set_drop_deletes(false); + clock.advance(Duration::from_secs(600)); + until("cleanup timer recovered mapping without activity", || { + client.have_mapping() && d.sock.portmapped().is_some() + }) + .await; + client.cleanup().await; + let table = router.mappings(); + let advertised = d.sock.portmapped().unwrap(); + assert_eq!(table.len(), 1, "next cleanup must preserve advertisement"); + assert_eq!(table[0].internal, d.sock.local_addr().unwrap().port()); + assert_eq!(table[0].external, advertised.port()); + d.stop().await; + client.close(Duration::from_secs(1)).await; + router.shutdown().await; + } + + /// With idle renewal on, the renewal point is honoured while idle with peers; with no + /// peers there is no renewal (the mapping expires); with idle renewal off, none either. + #[tokio::test] + async fn renew_while_idle_policy() { + for (renew_while_idle, peers, want_renewal) in [ + (true, true, true), + (true, false, false), + (false, true, false), + ] { + let (router, client, _lookup, clock) = + harness_client(RouterOptions::default(), true).await; + let d = driver(Arc::new(client.clone()), renew_while_idle, never_restun).await; + d.peers.store(peers, Ordering::SeqCst); + until("mapped", || client.have_mapping()).await; + // Past the renewal point (1 h for a 2 h grant), well before expiry. + clock.advance(Duration::from_secs(3601)); + tokio::time::sleep(Duration::from_millis(800)).await; + let maps = router.counters().num_pmp_map_recv; + assert_eq!( + maps > 1, + want_renewal, + "renew_while_idle={renew_while_idle} peers={peers}: {maps} MAPs" + ); + d.stop().await; + client.close(Duration::from_secs(1)).await; + router.shutdown().await; + } + } + + /// Lazy mode: the periodic ReSTUN tick renews a due mapping when its gate is open, and + /// nothing happens while it is closed. + #[tokio::test] + async fn restun_ticks_follow_the_gate() { + let (router, client, _lookup, clock) = harness_client(RouterOptions::default(), true).await; + let d = driver(Arc::new(client.clone()), false, fast_restun).await; + until("mapped", || client.have_mapping()).await; + clock.advance(Duration::from_secs(3601)); + tokio::time::sleep(Duration::from_millis(500)).await; + assert_eq!( + router.counters().num_pmp_map_recv, + 1, + "gate closed: no renewal" + ); + d.restun.store(true, Ordering::SeqCst); + until("renewed on a ReSTUN tick", || { + router.counters().num_pmp_map_recv == 2 + }) + .await; + d.stop().await; + client.close(Duration::from_secs(1)).await; + router.shutdown().await; + } + + /// The network identity changes while idle (no ReSTUN, no kick): the poll detects it + /// within one interval and the mapping is withdrawn, with no release on network-down. + #[tokio::test] + async fn poll_detects_a_network_change_while_idle() { + let (router, client, lookup, _clock) = harness_client(RouterOptions::default(), true).await; + let d = driver(Arc::new(client.clone()), false, never_restun).await; + until("mapped", || client.have_mapping()).await; + lookup.set_no_route(); + let t = std::time::Instant::now(); + until("withdrawn", || d.sock.portmapped().is_none()).await; + assert!(t.elapsed() <= POLL + Duration::from_millis(500)); + assert_eq!( + router.counters().num_deletes, + 0, + "no release on network-down" + ); + d.stop().await; + client.close(Duration::from_secs(1)).await; + router.shutdown().await; + } + + /// Through the driver, a resume (wall clock jumps, monotonic does not) is noticed by the + /// poll and renews the mapping once. + #[tokio::test] + async fn poll_notices_a_resume_and_renews_once() { + let (router, client, _lookup, clock) = harness_client(RouterOptions::default(), true).await; + let d = driver(Arc::new(client.clone()), false, never_restun).await; + until("mapped", || client.have_mapping()).await; + tokio::time::sleep(Duration::from_millis(300)).await; + let mappings = || { + router + .requests() + .iter() + .filter(|r| { + r.kind == ts_portmapper::testing::RequestKind::PmpMap + && r.lifetime.is_some_and(|n| n > 0) + }) + .count() + }; + clock.suspend(Duration::from_secs(4 * 3600)); + until("renewed after resume", || mappings() == 2).await; + tokio::time::sleep(Duration::from_millis(500)).await; + assert_eq!( + mappings(), + 2, + "exactly one replacement, excluding cleanup deletes" + ); + d.stop().await; + client.close(Duration::from_secs(1)).await; + router.shutdown().await; + } + + /// `only-tcp-443` through the handle: no LAN packet while on, deletes included; off again, + /// the leftover is deleted and mapping resumes. + #[tokio::test] + async fn only_tcp_443_through_the_handle() { + let (router, client, _lookup, clock) = harness_client(RouterOptions::default(), true).await; + let d = driver(Arc::new(client.clone()), false, never_restun).await; + until("mapped", || client.have_mapping()).await; + d.handle.set_only_tcp_443(true); + assert!(!client.have_mapping(), "withdrawn synchronously"); + until("facets false", || { + d.facets + .borrow() + .is_some_and(|f| f.pmp == Some(false) && !f.have_port_map) + }) + .await; + let total = router.counters().num_total_recv; + d.handle.kick(); + tokio::time::sleep(Duration::from_millis(300)).await; + assert_eq!(router.counters().num_total_recv, total, "silent"); + clock.advance(Duration::from_secs(3)); + d.handle.set_only_tcp_443(false); + until("re-mapped", || client.have_mapping()).await; + assert_eq!(router.counters().num_deletes, 1, "the leftover went first"); + d.stop().await; + client.close(Duration::from_secs(1)).await; + router.shutdown().await; + } + + /// Runner mode (idle renewal, peers): a failed first create is retried at its backoff by the + /// driver's timer while idle, with no activity or kick. + #[tokio::test] + async fn failed_create_is_retried_while_idle_with_peers() { + let (router, client, _lookup, clock) = harness_client( + RouterOptions { + pcp: false, + ..RouterOptions::default() + }, + true, + ) + .await; + router.set_result( + ts_portmapper::testing::RequestKind::PmpMap, + Some(ts_portmapper::testing::ResultOverride { + code: 3, + lifetime: 0, + }), + ); + let d = driver(Arc::new(client.clone()), true, never_restun).await; + until("first create refused", || { + router.counters().num_pmp_map_recv == 1 && client.schedule().backing_off + }) + .await; + router.set_result(ts_portmapper::testing::RequestKind::PmpMap, None); + // Past the longest first backoff (10 s + 20 %). + clock.advance(Duration::from_secs(60)); + until("retried and mapped", || client.have_mapping()).await; + d.stop().await; + client.close(Duration::from_secs(1)).await; + router.shutdown().await; + } + + /// The poll finds a network while idle (none at start, as for a launchd job at boot): it kicks + /// a round, so the mapping is made without any activity. + #[tokio::test] + async fn network_found_by_the_poll_kicks_a_round() { + let (router, client, lookup, _clock) = harness_client(RouterOptions::default(), true).await; + lookup.set_no_route(); + let d = driver(Arc::new(client.clone()), true, never_restun).await; + tokio::time::sleep(Duration::from_millis(300)).await; + assert!(!client.has_network()); + lookup.set(gateway()); + until("mapped after the poll found the network", || { + client.have_mapping() + }) + .await; + d.stop().await; + client.close(Duration::from_secs(1)).await; + router.shutdown().await; + } + + /// A [`PortMapper`] that counts how often the driver reads the schedule (once per loop). + struct Counting { + inner: Arc, + schedules: Arc, + } + + impl PortMapper for Counting { + fn subscribe(&self) -> watch::Receiver { + self.inner.subscribe() + } + fn set_local_port(&self, port: u16) -> Result<(), ts_portmapper::client::Error> { + self.inner.set_local_port(port) + } + fn note_rebind(&self, port: Option) { + self.inner.note_rebind(port); + } + fn note_network_down(&self) { + self.inner.note_network_down(); + } + fn check_network(&self) -> BoxFuture<'_, ts_portmapper::NetCheck> { + self.inner.check_network() + } + fn check_clock(&self) -> bool { + self.inner.check_clock() + } + fn disable(&self, why: DisableReason) { + self.inner.disable(why); + } + fn enable(&self) { + self.inner.enable(); + } + fn expire_if_due(&self) -> bool { + self.inner.expire_if_due() + } + fn schedule(&self) -> ts_portmapper::ScheduleSnapshot { + self.schedules.fetch_add(1, Ordering::SeqCst); + self.inner.schedule() + } + fn to_real(&self, t: Instant) -> Instant { + self.inner.to_real(t) + } + fn tick(&self) -> BoxFuture<'_, Option> { + self.inner.tick() + } + fn cleanup(&self) -> BoxFuture<'_, usize> { + self.inner.cleanup() + } + fn close(&self, budget: Duration) -> BoxFuture<'_, ()> { + self.inner.close(budget) + } + fn release_blocking_best_effort(&self) { + self.inner.release_blocking_best_effort(); + } + fn reap(&self) -> usize { + self.inner.reap() + } + } + + type BoxFuture<'a, T> = core::pin::Pin + Send + 'a>>; + + /// A retry request left over when `only-tcp-443` turns mapping off does not wake the driver + /// every second: no attempt could start, so none is scheduled. + #[tokio::test] + async fn no_wakeups_while_only_tcp_443() { + let (router, client, _lookup, _clock) = + harness_client(RouterOptions::default(), true).await; + let schedules = Arc::new(AtomicUsize::new(0)); + let mapper: Arc = Arc::new(Counting { + inner: Arc::new(client.clone()), + schedules: schedules.clone(), + }); + let sock = Arc::new( + MagicSock::bind( + "127.0.0.1:0".parse().unwrap(), + DiscoPrivateKey::random(), + NodePrivateKey::random().public_key(), + ) + .await + .unwrap(), + ); + let handle = PortmapHandle::new(mapper); + let (port_map_tx, _facets) = watch::channel(None); + let (shutdown, shutdown_rx) = watch::channel(false); + let task = tokio::spawn(run_driver(DriverArgs { + handle: handle.clone(), + sock: sock.clone(), + has_peers: Arc::new(|| false), + restun_gate: Arc::new(|| false), + port_map_tx, + renew_while_idle: false, + restun_delay: never_restun, + poll: Duration::from_secs(3600), + shutdown: shutdown_rx, + })); + until("mapped", || client.have_mapping()).await; + // A port change, then a request refused by the 2 s floor: a retry is wanted. + let port = sock.local_addr().unwrap().port(); + client.set_local_port(port.wrapping_add(1)).unwrap(); + assert!(client.get_cached_mapping_or_start_creating_one().is_none()); + assert!(client.schedule().retry_at.is_some()); + handle.set_only_tcp_443(true); + tokio::time::sleep(Duration::from_millis(500)).await; + let before = schedules.load(Ordering::SeqCst); + tokio::time::sleep(Duration::from_secs(3)).await; + let n = schedules.load(Ordering::SeqCst) - before; + shutdown.send_replace(true); + task.await.unwrap(); + client.close(Duration::from_secs(1)).await; + router.shutdown().await; + assert!( + n <= 1, + "the driver woke {n} times in 3 s with only-tcp-443 on" + ); + } + + /// The runtime's release guard: dropping it (every drop of the runtime does) sends the + /// delete with no reply awaited; after a close it sends nothing more. + #[tokio::test] + async fn release_guard_deletes_on_drop_and_once_after_close() { + for close_first in [false, true] { + let (router, client, _lookup, _clock) = + harness_client(RouterOptions::default(), true).await; + client.set_local_port(41641).unwrap(); + client.tick().await; + until("mapped", || client.have_mapping()).await; + let guard = PortmapRelease::new(Some(Arc::new(client.clone()) as Arc)); + assert!(guard.get().is_some()); + if close_first { + guard.close(Some(Duration::from_millis(500))).await; + assert!(router.mappings().is_empty(), "closed: deleted"); + } + drop(guard); + until("deleted", || router.mappings().is_empty()).await; + tokio::time::sleep(Duration::from_millis(100)).await; + assert_eq!( + router.counters().num_deletes, + 1, + "close_first={close_first}" + ); + router.shutdown().await; + } + } + + // ----------------------------------------------------------------------------------------- + // Through the real DirectManager + + use kameo::{ + actor::{ActorRef, Spawn}, + message::{Context, Message}, + }; + + use crate::{ + Env, Error, + direct::{EndpointAdvertisement, PortmapClient, RebindAndReprobe, SockHandle}, + env::{ForwarderConfig, PortMapperSettings}, + }; + + fn forwarder(port_mapper: Option) -> ForwarderConfig { + ForwarderConfig { + accept_routes: false, + accept_dns: true, + exit_node: None, + forward_routes: vec![], + forward_tcp_ports: vec![], + forward_udp_ports: vec![], + forward_all_ports: false, + forward_exit_egress: false, + block_incoming: false, + exit_proxy: None, + exit_node_dns: None, + peerapi_port: None, + taildrop_dir: None, + enable_ipv6: false, + wireguard_listen_port: None, + network_monitor: false, + port_mapper, + persistent_keepalive_interval: None, + ingress_active: Arc::new(AtomicBool::new(false)), + } + } + + struct Stack { + env: Env, + direct: ActorRef, + _shutdown: watch::Sender, + } + + async fn stack(port_mapper: Option) -> Stack { + let (shutdown, shutdown_rx) = watch::channel(false); + let env = Env::new( + ts_keys::NodeState::generate(), + shutdown_rx, + forwarder(port_mapper), + ); + let dataplane = crate::dataplane::DataplaneActor::spawn(env.clone()); + let (_home_tx, home_rx) = watch::channel(None); + let multiderp = + crate::multiderp::Multiderp::spawn((env.clone(), dataplane.clone(), home_rx)); + let direct = + crate::direct::DirectManager::spawn((env.clone(), dataplane.clone(), multiderp)); + direct.ask(SockHandle).await.unwrap().expect("bound"); + Stack { + env, + direct, + _shutdown: shutdown, + } + } + + /// The shared runtime test configuration builds no client, constructs no socket for + /// it and never runs the gateway lookup; the lookup stub this crate's tests get instead of the + /// real one panics, so turning the mapper on without a harness cannot reach the network. (The + /// feature combinations are covered by building with `portmapper` off, on, and with the + /// harness as a dev-dependency.) + #[tokio::test] + async fn shared_test_config_never_builds_a_client() { + let built = CLIENTS_BUILT.load(Ordering::SeqCst); + let lookups = NEVER_LOOKUPS.load(Ordering::SeqCst); + let s = stack(None).await; + assert!(s.direct.ask(PortmapClient).await.unwrap().is_none()); + tokio::time::sleep(Duration::from_millis(200)).await; + assert_eq!(CLIENTS_BUILT.load(Ordering::SeqCst), built, "no client"); + assert_eq!(NEVER_LOOKUPS.load(Ordering::SeqCst), lookups, "no lookup"); + + // The library default is on: a config that does not disable it builds the (stubbed) + // client, whose first lookup hits the panicking stub, never the routing table. + let s = stack(Some(PortMapperSettings { + options: ts_control::PortMapperOptions::default(), + factory: None, + })) + .await; + assert!(s.direct.ask(PortmapClient).await.unwrap().is_some()); + until("stub lookup", || { + NEVER_LOOKUPS.load(Ordering::SeqCst) > lookups + }) + .await; + assert_eq!(CLIENTS_BUILT.load(Ordering::SeqCst), built + 1); + } + + /// `disable_port_mapper` and `TS_DISABLE_PORTMAPPER` (read through + /// an injected reader, parsed like Go's `ParseBool`) both mean no client at all. + #[test] + fn config_and_env_disable() { + let mut cfg = ts_control::Config::default(); + let on = ForwarderConfig::from_control_config(&cfg); + assert!(on.port_mapper.is_some(), "library default: on"); + cfg.disable_port_mapper = true; + assert!( + ForwarderConfig::from_control_config(&cfg) + .port_mapper + .is_none() + ); + for (v, disabled) in [ + ("1", true), + ("true", true), + ("T", true), + ("0", false), + ("false", false), + ("", false), + ("yes", true), + ] { + let mut f = ForwarderConfig::from_control_config(&ts_control::Config::default()); + f.apply_env_knobs(|k| (k == "TS_DISABLE_PORTMAPPER").then(|| v.to_owned())); + assert_eq!(f.port_mapper.is_none(), disabled, "{v:?}"); + } + let mut f = ForwarderConfig::from_control_config(&ts_control::Config::default()); + f.apply_env_knobs(|_| None); + assert!(f.port_mapper.is_some()); + } + + struct AdvTap(Arc>>); + + impl kameo::Actor for AdvTap { + type Args = Arc>>; + type Error = Error; + async fn on_start(seen: Self::Args, _s: ActorRef) -> Result { + Ok(Self(seen)) + } + } + + impl Message for AdvTap { + type Reply = (); + async fn handle(&mut self, m: EndpointAdvertisement, _c: &mut Context) { + self.0.lock().unwrap().push(m); + } + } + + /// Through the real direct manager with a harness-backed client injected, a mapping is + /// obtained, the type-3 endpoint and the facets are advertised for control (wire type 3), a + /// same-port rebind keeps the mapping with no I/O, and the shutdown release makes the router see + /// the delete within the timeout, with no second added. + #[tokio::test] + async fn end_to_end_through_the_direct_manager() { + let (router, client, _lookup, _clock) = + harness_client(RouterOptions::default(), true).await; + let injected = client.clone(); + let s = stack(Some(PortMapperSettings { + options: ts_control::PortMapperOptions::default(), + factory: Some(Arc::new(move || { + Arc::new(injected.clone()) as Arc + })), + })) + .await; + let seen = Arc::new(std::sync::Mutex::new(Vec::new())); + let tap = AdvTap::spawn(seen.clone()); + s.env + .subscribe::(&tap) + .await + .unwrap(); + + until("advertised", || { + seen.lock().unwrap().last().is_some_and(|a| { + a.endpoints.first().map(|e| e.ty) + == Some(ts_magicsock::SelfEndpointType::PortMapped) + && a.port_map.is_some_and(|f| f.have_port_map) + }) + }) + .await; + let adv = seen.lock().unwrap().last().cloned().unwrap(); + let wire = crate::control_runner::wire_endpoints(&adv.endpoints); + // Go's MapRequest carries `EndpointTypes` as integers: PortMapped is 3. + assert_eq!(wire[0].ty, ts_control_serde::EndpointType::PortMapped); + assert_eq!(serde_json::to_string(&wire[0].ty).unwrap(), "3"); + assert_eq!(wire[0].endpoint, "203.0.113.7:4242".parse().unwrap()); + assert!(!adv.stun.working_udp, "a port mapping is not STUN evidence"); + let f = adv.port_map.unwrap(); + assert_eq!((f.pmp, f.pcp, f.upnp), (Some(true), Some(true), None)); + + // A rebind on the same port and network: the mapping survives with no router traffic. + let before = router.counters().num_total_recv; + s.direct.ask(RebindAndReprobe).await.unwrap(); + assert!(client.have_mapping()); + let sock = s.direct.ask(SockHandle).await.unwrap().unwrap(); + until("re-installed", || sock.portmapped().is_some()).await; + assert_eq!(router.counters().num_total_recv, before, "no I/O"); + + // The runtime's shutdown step: within the timeout, and the router saw the delete. + let mapper = s.direct.ask(PortmapClient).await.unwrap().unwrap(); + let t = std::time::Instant::now(); + let timeout = Duration::from_millis(400); + assert!( + tokio::time::timeout(timeout, crate::close_portmap(Some(&mapper), Some(timeout))) + .await + .is_ok() + ); + assert!(t.elapsed() < timeout); + assert!(router.mappings().is_empty()); + assert_eq!(router.counters().num_deletes, 1); + router.shutdown().await; + } + + /// The shutdown budget is at most 1 s and never more than the caller's timeout (no added + /// second), even with the router gone. + #[tokio::test] + async fn shutdown_release_stays_within_the_timeout() { + assert_eq!(crate::portmap_budget(None), Duration::from_secs(1)); + assert_eq!( + crate::portmap_budget(Some(Duration::from_millis(200))), + Duration::from_millis(200) + ); + assert_eq!( + crate::portmap_budget(Some(Duration::from_secs(5))), + Duration::from_secs(1) + ); + let (router, client, _lookup, _clock) = + harness_client(RouterOptions::default(), true).await; + client.set_local_port(41641).unwrap(); + client.tick().await; + until("mapped", || client.have_mapping()).await; + router.drop_next(1000); + let mapper: Arc = Arc::new(client); + let t = std::time::Instant::now(); + crate::close_portmap(Some(&mapper), Some(Duration::from_millis(300))).await; + assert!( + t.elapsed() < Duration::from_millis(450), + "{:?}", + t.elapsed() + ); + router.shutdown().await; + } +} diff --git a/ts_runtime/src/tun_actor.rs b/ts_runtime/src/tun_actor.rs index 525f2614..d4ca637a 100644 --- a/ts_runtime/src/tun_actor.rs +++ b/ts_runtime/src/tun_actor.rs @@ -1561,6 +1561,7 @@ mod tests { enable_ipv6: false, wireguard_listen_port: None, network_monitor: false, + port_mapper: None, persistent_keepalive_interval: None, ingress_active: Arc::new(std::sync::atomic::AtomicBool::new(false)), }, From 2ccd2f93b099d3d0cfeb29331493200d96a567c8 Mon Sep 17 00:00:00 2001 From: Kasper Nielsen Date: Sat, 17 Oct 2026 23:06:47 +0200 Subject: [PATCH 05/12] feat(exit_node): port-mapper flags and a one-shot --portmap-probe --portmap-probe runs before any identity, state or control connection exists. It maps a dedicated test port on the LAN gateway with each protocol separately, can hold the mapping open to count inbound datagrams, deletes it and exits (0: every protocol mapped and deleted, 2: some, 1: none or no usable gateway). Interrupts end the wait or the hold early, and the delete still runs. For the node, the runner defaults differ from the library's (Go's) defaults in two places. It does not advertise a private or CGNAT mapping (a double NAT). It renews a due mapping while idle as long as it has peers, so an idle exit node stays directly reachable. --portmap-advertise-nonpublic and --portmap-lazy-renew restore Go's behaviour. --portmap-gateway overrides the gateway (validated like a discovered one), and --no-portmapper turns the mapper off. The status line gains Go's netcheck portmap= summary and the mapping. Signed-off-by: Kasper Nielsen Co-Authored-By: GPT-6.1 (codex) Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 1 + Cargo.lock | 1 + Cargo.toml | 2 + docs/PORTMAPPER.md | 15 +- examples/exit_node/main.rs | 309 ++++++++++++++++++++++++++++++++++++- 5 files changed, 318 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 00f11f67..901948f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ Record breaking or significant changes here. All dates are UTC. * **portmapper:** add `ts_portmapper`, a NAT-PMP and PCP client modelled on Go's `net/portmapper`, which only ever sends to the LAN gateway's port 5351. * **runtime:** map the underlay UDP port on the LAN gateway with NAT-PMP or PCP and advertise the mapping to peers, as Go does; `Config::disable_port_mapper` or `TS_DISABLE_PORTMAPPER` turns it off. +* **exit node:** add `--portmap-probe`, a one-shot NAT-PMP/PCP check, and flags for the port mapper. ### Performance diff --git a/Cargo.lock b/Cargo.lock index aa5f3d3a..c6184e60 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1828,6 +1828,7 @@ dependencies = [ "geiserx_ts_metrics", "geiserx_ts_netstack_smoltcp", "geiserx_ts_packet", + "geiserx_ts_portmapper", "geiserx_ts_runtime", "geiserx_ts_tka", "hyper", diff --git a/Cargo.toml b/Cargo.toml index c69d8124..1816fcdf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -338,6 +338,8 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] } tracing-test = "0.2" ts_keys.workspace = true +# `examples/exit_node --portmap-probe`: the one-shot NAT-PMP/PCP check. +ts_portmapper.workspace = true ts_test_util.workspace = true # The in-process fake tsrecorder the SSH session-recording tests run against: `hyper`'s h2c server diff --git a/docs/PORTMAPPER.md b/docs/PORTMAPPER.md index 283ef783..b6a79ac3 100644 --- a/docs/PORTMAPPER.md +++ b/docs/PORTMAPPER.md @@ -9,6 +9,7 @@ behind a hard NAT can reach it directly instead of over DERP. NAT-PMP (RFC 6886) codecs, the gateway lookup and `GatewayCap`, the only code that sends packets. (The gateway lookup's kernel source-address check connects a UDP socket but never sends on it.) - Driver: [`ts_runtime::portmap`](../ts_runtime/src/portmap.rs), spawned by the direct manager. +- One-shot field check: `examples/exit_node --portmap-probe`. ## Turning it off @@ -24,12 +25,14 @@ sets `port_mapper: None`, and the `portmapper_test_isolation` check enforces the ## Deviations from Go -- **Advertising (`advertise_nonpublic_external`).** The default is Go's: advertise every external - address. With the option off, a private or CGNAT external address (a double NAT) is kept and - renewed, but not advertised. -- **Renewal while idle (`renew_while_idle`).** The default is Go's: renewal rides on activity (the - periodic ReSTUN and other kicks). With the option on, a due mapping is renewed even while idle, - as long as there are peers. With no peers a mapping is never renewed and expires. +- **Advertising (`advertise_nonpublic_external`).** The library default is Go's: advertise every + external address. The exit-node runner turns it off. A private or CGNAT external address (a + double NAT) is then kept and renewed, but not advertised. `--portmap-advertise-nonpublic` reverts + the runner to Go's behaviour. +- **Renewal while idle (`renew_while_idle`).** The library default is Go's: renewal rides on + activity (the periodic ReSTUN and other kicks). The runner renews a due mapping even while idle, + as long as it has peers. With no peers a mapping is never renewed and expires. + `--portmap-lazy-renew` reverts the runner to Go's behaviour. - **`HavePortMap` means an advertised mapping**, so control never sees it without a type-3 endpoint. - **Notifications for every protocol.** Success, loss, expiry and disable publish for NAT-PMP and diff --git a/examples/exit_node/main.rs b/examples/exit_node/main.rs index fb8d33ee..78b13b56 100644 --- a/examples/exit_node/main.rs +++ b/examples/exit_node/main.rs @@ -19,6 +19,16 @@ //! //! The advertised route still has to be approved in the admin console (or by an `autoApprovers` //! policy) before peers can select this node. +//! +//! `--portmap-probe` runs a one-shot NAT-PMP/PCP check instead of the node, before any identity, +//! state or control connection exists: it maps a dedicated test port on the LAN gateway, each +//! protocol separately, optionally holds it open to count inbound datagrams, deletes it and exits +//! (0: every protocol mapped and deleted, 2: some, 1: none or no usable gateway). Its output may +//! contain the router's public address. +//! +//! ```sh +//! cargo run --release --example exit_node -- --portmap-probe --portmap-hold 60 +//! ``` use std::{error::Error, path::PathBuf, time::Duration}; @@ -32,8 +42,8 @@ use tracing_subscriber::filter::LevelFilter; #[command(version, about)] struct Args { /// Directory holding this node's persisted identity (`keys.json`). Created if missing. - #[arg(long)] - state_dir: PathBuf, + #[arg(long, required_unless_present = "portmap_probe")] + state_dir: Option, /// File containing the auth key. Only read if the identity is not yet registered. #[arg(long)] @@ -94,10 +104,163 @@ struct Args { /// Seconds between status lines (peers, direct vs DERP path). 0 disables. #[arg(long, default_value_t = 30)] status_interval: u64, + + /// Run the one-shot NAT-PMP/PCP probe on a dedicated test port and exit. Needs no state + /// directory, keys or control connection. + #[arg(long)] + portmap_probe: bool, + + /// Probe: which protocols to map, each separately. + #[arg(long, value_enum, default_value_t = PortmapProto::Both, requires = "portmap_probe")] + portmap_proto: PortmapProto, + + /// Probe: the test socket's UDP port (0 = ephemeral). Never `--port`, 41641 or 41642. + #[arg(long, default_value_t = 0, requires = "portmap_probe")] + portmap_test_port: u16, + + /// Probe: requested lease in seconds. + #[arg( + long, + default_value_t = ts_portmapper::probe_tool::DEFAULT_LIFETIME, + value_parser = clap::value_parser!(u32).range(1..=7200), + requires = "portmap_probe" + )] + portmap_lifetime: u32, + + /// Probe: seconds to hold each mapping open, counting inbound datagrams (no payloads). + #[arg( + long, + default_value_t = 0, + value_parser = clap::value_parser!(u64).range(0..=600), + requires = "portmap_probe" + )] + portmap_hold: u64, + + /// Probe: seconds to keep retrying the gateway lookup (once a second; routing-table reads + /// only) when there is no usable default route yet, e.g. a launchd job at boot. + #[arg( + long, + default_value_t = 0, + value_parser = clap::value_parser!(u64).range(0..=120), + requires = "portmap_probe" + )] + portmap_wait: u64, + + /// Use this gateway (`GW` or `GW,SELF`) instead of the default route's, for the node's port + /// mapper and for the probe. It is validated like a discovered one (private, on-link). + #[arg(long, value_name = "GW[,SELF]")] + portmap_gateway: Option, + + /// Turn the NAT port mapper (NAT-PMP/PCP) off. Equivalent to `TS_DISABLE_PORTMAPPER=1`. + #[arg(long, conflicts_with = "portmap_probe")] + no_portmapper: bool, + + /// Advertise a port-mapped address even when it is private or CGNAT (a double NAT), as Go + /// does. By default this runner keeps and renews such a mapping but does not offer it to + /// peers. + #[arg(long, conflicts_with = "portmap_probe")] + portmap_advertise_nonpublic: bool, + + /// Renew the port mapping only on activity, as Go does. By default this runner renews it when + /// due even while idle, as long as it has peers, so an idle exit node stays directly + /// reachable. + #[arg(long, conflicts_with = "portmap_probe")] + portmap_lazy_renew: bool, +} + +impl Args { + /// The node's port-mapper options: this runner's defaults are suppression of non-public + /// external addresses and idle renewal with peers; the flags revert either to Go's behaviour. + fn port_mapper_options(&self) -> tailscale::config::PortMapperOptions { + tailscale::config::PortMapperOptions { + gateway_override: self.portmap_gateway.map(|g| (g.gw, g.self_ip)), + advertise_nonpublic_external: self.portmap_advertise_nonpublic, + renew_while_idle: !self.portmap_lazy_renew, + } + } +} + +/// `--portmap-proto` values. +#[derive(Clone, Copy, Debug, PartialEq, Eq, clap::ValueEnum)] +enum PortmapProto { + Pmp, + Pcp, + Both, +} + +impl Args { + /// The probe's options from the command line. + fn probe_options(&self) -> ts_portmapper::probe_tool::ProbeOptions { + ts_portmapper::probe_tool::ProbeOptions { + protos: ts_portmapper::oneshot::ProtoSet { + pmp: self.portmap_proto != PortmapProto::Pcp, + pcp: self.portmap_proto != PortmapProto::Pmp, + }, + test_port: self.portmap_test_port, + lifetime: self.portmap_lifetime, + hold: Duration::from_secs(self.portmap_hold), + gateway_override: self.portmap_gateway, + node_port: self.port, + wait: Duration::from_secs(self.portmap_wait), + } + } +} + +/// Run `--portmap-probe`: no `Device`, no identity, no control. Interrupts (Ctrl-C, SIGTERM) end +/// the gateway wait or the hold early and skip the remaining maps; the delete for anything sent +/// still runs. The handlers are installed before the probe starts, so an interrupt at any point +/// is caught rather than killing the process. Returns the process exit code. +fn run_portmap_probe(args: &Args) -> Result> { + // The probe's report goes to stdout; logs, if any, to stderr. + tracing_subscriber::fmt() + .with_writer(std::io::stderr) + .with_env_filter( + tracing_subscriber::EnvFilter::builder() + .with_default_directive(LevelFilter::INFO.into()) + .from_env_lossy(), + ) + .init(); + let opts = args.probe_options(); + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + rt.block_on(async { + // Register the handlers now (not lazily on first poll): from here on SIGINT and SIGTERM + // are recorded instead of killing the process mid-exchange. + #[cfg(unix)] + let interrupt = { + let mut int = signal(SignalKind::interrupt())?; + let mut term = signal(SignalKind::terminate())?; + async move { + tokio::select! { + _ = int.recv() => {} + _ = term.recv() => {} + } + } + }; + #[cfg(not(unix))] + let interrupt = { + let (tx, rx) = tokio::sync::oneshot::channel::<()>(); + tokio::spawn(async move { + if tokio::signal::ctrl_c().await.is_ok() { + tx.send(()).ok(); + } + }); + async move { + rx.await.ok(); + } + }; + let mut stdout = std::io::stdout(); + Ok::<_, Box>(ts_portmapper::probe_tool::run(&opts, interrupt, &mut stdout).await) + }) } fn main() -> Result<(), Box> { let args = Args::parse(); + if args.portmap_probe { + let code = run_portmap_probe(&args)?; + std::process::exit(code); + } tokio::runtime::Builder::new_multi_thread() .worker_threads(args.worker_threads.max(1)) .enable_all() @@ -114,8 +277,12 @@ async fn run(args: Args) -> Result<(), Box> { ) .init(); - tokio::fs::create_dir_all(&args.state_dir).await?; - let key_file = args.state_dir.join("keys.json"); + let state_dir = args + .state_dir + .clone() + .ok_or("--state-dir is required unless --portmap-probe is given")?; + tokio::fs::create_dir_all(&state_dir).await?; + let key_file = state_dir.join("keys.json"); let first_run = !tokio::fs::try_exists(&key_file).await?; let auth_key = match (&args.auth_key_file, first_run) { @@ -153,6 +320,8 @@ async fn run(args: Args) -> Result<(), Box> { tcp_buffer_size: args.tcp_buffer_size, wireguard_listen_port: Some(args.port), netmap_cache_dir: None, + disable_port_mapper: args.no_portmapper, + port_mapper: args.port_mapper_options(), ..Config::default() }; @@ -248,12 +417,31 @@ async fn log_status(dev: &Device) { return; } }; + // Go's netcheck `portmap=` summary (e.g. `active-_MC`), plus the mapping when one is held. + let (portmap, portmap_ext) = match dev.port_map_status() { + Some(pm) => ( + pm.summary(), + pm.external.map(|e| { + format!( + "{e}{}", + if pm.advertised { + "" + } else { + " (not advertised)" + } + ) + }), + ), + None => ("off".to_owned(), None), + }; match &status.self_node { Some(me) => tracing::info!( routes = ?me.allowed_routes, exit_node = me.is_exit_node, peers = status.peers.len(), health = ?status.health, + %portmap, + ext = portmap_ext.as_deref().unwrap_or("-"), "self" ), None => { @@ -319,3 +507,116 @@ async fn serve_sink(mut conn: tailscale::netstack::TcpStream) { _ => {} } } + +#[cfg(test)] +mod tests { + use clap::Parser; + + use super::*; + + fn parse(args: &[&str]) -> Result { + Args::try_parse_from(std::iter::once("exit_node").chain(args.iter().copied())) + } + + /// The probe needs no state directory; the node still does. + #[test] + fn probe_runs_without_state_dir() { + let a = parse(&["--portmap-probe"]).unwrap(); + assert!(a.portmap_probe && a.state_dir.is_none()); + let o = a.probe_options(); + assert!(o.protos.pmp && o.protos.pcp); + assert_eq!((o.test_port, o.lifetime, o.node_port), (0, 120, 41642)); + assert!(o.validate().is_ok()); + assert!(parse(&[]).is_err()); + assert!(parse(&["--state-dir", "/tmp/x"]).is_ok()); + } + + #[test] + fn probe_flags_require_the_probe() { + for flags in [ + &["--state-dir", "s", "--portmap-hold", "5"][..], + &["--state-dir", "s", "--portmap-proto", "pmp"], + &["--state-dir", "s", "--portmap-test-port", "5000"], + &["--state-dir", "s", "--portmap-wait", "5"], + ] { + assert!(parse(flags).is_err(), "{flags:?}"); + } + } + + #[test] + fn probe_flag_ranges_and_reserved_ports() { + assert!(parse(&["--portmap-probe", "--portmap-lifetime", "0"]).is_err()); + assert!(parse(&["--portmap-probe", "--portmap-lifetime", "7201"]).is_err()); + assert!(parse(&["--portmap-probe", "--portmap-hold", "601"]).is_err()); + assert!(parse(&["--portmap-probe", "--portmap-wait", "121"]).is_err()); + let a = parse(&["--portmap-probe", "--portmap-wait", "30"]).unwrap(); + assert_eq!(a.probe_options().wait, Duration::from_secs(30)); + for port in ["41641", "41642"] { + let a = parse(&["--portmap-probe", "--portmap-test-port", port]).unwrap(); + assert!(a.probe_options().validate().is_err(), "{port}"); + } + let a = parse(&[ + "--portmap-probe", + "--port", + "50000", + "--portmap-test-port", + "50000", + ]) + .unwrap(); + assert!(a.probe_options().validate().is_err()); + let a = parse(&[ + "--portmap-probe", + "--portmap-proto", + "pcp", + "--portmap-gateway", + "192.168.1.1,192.168.1.20", + "--portmap-hold", + "60", + ]) + .unwrap(); + let o = a.probe_options(); + assert!(!o.protos.pmp && o.protos.pcp); + assert_eq!(o.hold, Duration::from_secs(60)); + assert_eq!( + o.gateway_override.unwrap().self_ip, + Some("192.168.1.20".parse().unwrap()) + ); + assert!(parse(&["--portmap-probe", "--portmap-gateway", "nope"]).is_err()); + } + + /// The runner's port-mapper defaults (suppress non-public, renew while idle) and the flags + /// that revert each to Go's behaviour; the gateway override serves the node too. + #[test] + fn port_mapper_flags() { + let a = parse(&["--state-dir", "s"]).unwrap(); + assert!(!a.no_portmapper); + let o = a.port_mapper_options(); + assert!(!o.advertise_nonpublic_external); + assert!(o.renew_while_idle); + assert!(o.gateway_override.is_none()); + let a = parse(&[ + "--state-dir", + "s", + "--portmap-advertise-nonpublic", + "--portmap-lazy-renew", + "--portmap-gateway", + "192.168.1.1,192.168.1.20", + ]) + .unwrap(); + let o = a.port_mapper_options(); + assert!(o.advertise_nonpublic_external && !o.renew_while_idle); + assert_eq!( + o.gateway_override, + Some(( + "192.168.1.1".parse().unwrap(), + Some("192.168.1.20".parse().unwrap()) + )) + ); + assert!( + parse(&["--state-dir", "s", "--no-portmapper"]) + .unwrap() + .no_portmapper + ); + assert!(parse(&["--portmap-probe", "--no-portmapper"]).is_err()); + } +} From 624278d6e723bd813f4813f4260a745d7ead61ee Mon Sep 17 00:00:00 2001 From: Kasper Nielsen Date: Sat, 17 Oct 2026 23:06:46 +0200 Subject: [PATCH 06/12] feat(checks): guard the port mapper's LAN-only boundary A tripwire for what the compiler cannot check. ts_portmapper's dependency closure has no TLS or HTTP crate, ts_control does not depend on it, and only dev-dependencies may enable its test-harness feature. Outside cap.rs and the test harness, no line of ts_portmapper/src may name a socket type, a send or connect call, a raw fd, a child process or the IPv6 wildcard. The kernel source-address lookup and a macOS netstat cross-check test are allowlisted by function name. Signed-off-by: Kasper Nielsen Co-Authored-By: GPT-6.1 (codex) Co-Authored-By: Claude Opus 5.5 --- checks/src/main.rs | 2 + checks/src/portmapper_lan_only.rs | 568 ++++++++++++++++++++++++++++++ 2 files changed, 570 insertions(+) create mode 100644 checks/src/portmapper_lan_only.rs diff --git a/checks/src/main.rs b/checks/src/main.rs index 0a9be37b..4e4b84a2 100644 --- a/checks/src/main.rs +++ b/checks/src/main.rs @@ -12,6 +12,7 @@ mod no_std_deps; mod no_std_honest; mod parity_audit; mod porting_ledger; +mod portmapper_lan_only; mod restock_beads; mod ssh_isolation; @@ -38,6 +39,7 @@ pub const CHECK_FNS: &[(&str, CheckFn)] = &[ ("no_std_deps", no_std_deps::run), ("funnel_fail_closed", funnel_fail_closed::run), ("ssh_isolation", ssh_isolation::run), + ("portmapper_lan_only", portmapper_lan_only::run), ("porting_ledger", porting_ledger::run), ("parity_audit", parity_audit::run), ("restock_beads", restock_beads::run), diff --git a/checks/src/portmapper_lan_only.rs b/checks/src/portmapper_lan_only.rs new file mode 100644 index 00000000..2d7ea45e --- /dev/null +++ b/checks/src/portmapper_lan_only.rs @@ -0,0 +1,568 @@ +//! Leak firewall: the NAT port mapper (`ts_portmapper`) stays a LAN-only control path. +//! +//! WHY: the port mapper sends UDP to the LAN gateway's NAT-PMP/PCP port and nothing else. The real +//! boundary is the type system and runtime tests: `GatewayCap` (in `ts_portmapper/src/cap.rs`) is +//! the only code that creates sockets, its API takes a `LanDest` rather than an address, and the +//! test transport exists only behind `cfg(test)` / the `test-harness` feature. This check guards +//! the parts of that boundary a compiler cannot: +//! +//! 1. **Dependency graph** (`cargo metadata`): `ts_portmapper`'s normal and build dependency +//! closure contains no TLS or HTTP-client crate, and `ts_control` does not depend on +//! `ts_portmapper` at all (control only ever sees transport-neutral facets). +//! 2. **Feature graph**: no workspace crate enables `ts_portmapper/test-harness` through a normal +//! or build dependency or one of its own features (dev-dependencies may), and the feature is +//! not on by default. +//! 3. **Tripwire scan**: outside `cap.rs` and `testing/`, no line of `ts_portmapper/src` may +//! mention a socket type, a socket syscall, a send/connect/write call on a raw fd, a raw-fd +//! wrapper, the `nix` crate, a child process, or an IPv6 wildcard. The +//! kernel source-selection lookup in the gateway module (a `connect()`ed UDP socket that never +//! sends) is allow-listed by function name. +//! +//! The scan is a tripwire, not a proof: it matches text, so a determined author can evade it (a +//! macro, an alias built from a re-export elsewhere). It exists to make an accidental second +//! socket path loud, with adversarial fixtures for the obvious evasions (an aliased import, a +//! qualified path) below. + +use std::collections::{BTreeSet, HashMap}; + +use crate::{Args, BoxResult}; + +/// The port mapper's package name. +const PORTMAPPER: &str = "geiserx_ts_portmapper"; +/// The control client's package name. +const CONTROL: &str = "geiserx_ts_control"; +/// The harness feature only dev-dependencies may enable. +const HARNESS_FEATURE: &str = "test-harness"; + +/// Crates (or crate-name prefixes ending in `*`) that mean TLS or an HTTP client. +const FORBIDDEN_DEPS: &[&str] = &[ + "rustls*", + "tokio-rustls", + "webpki*", + "native-tls", + "openssl*", + "aws-lc*", + "ring", + "hyper", + "hyper-util", + "h2", + "http", + "httparse", + "reqwest", + "ureq", + "isahc", + "curl", + "geiserx_ts_http_util", + "geiserx_ts_tls_util", +]; + +/// The tree the tripwire scans. +const SCAN_ROOT: &str = "ts_portmapper/src"; + +/// Paths under [`SCAN_ROOT`] that may create sockets. +const SOCKET_MODULES: &[&str] = &["cap.rs", "testing/"]; + +/// `(file, function)` pairs allowed to match a pattern: the kernel source-selection lookup, and +/// the macOS test that cross-checks the routing-table parser against `netstat -rn` (a local exec, +/// no network traffic). +const ALLOWED_FNS: &[(&str, &str)] = &[ + ("gateway/mod.rs", "kernel_source"), + ("gateway/darwin.rs", "agrees_with_netstat"), +]; + +/// Text that means "a socket or a send" outside the capability. +const PATTERNS: &[&str] = &[ + "UdpSocket", + "TcpStream", + "TcpSocket", + "TcpListener", + "socket2", + "tokio::net", + "mio::", + "libc::socket", + "libc::sendto", + "libc::sendmsg", + "libc::connect", + "send_to(", + "sendto(", + "connect(", + // A raw fd written to directly, or one wrapped from elsewhere. + "libc::send(", + "libc::write(", + "from_raw_fd", + "FromRawFd", + "nix::", + // A child process (natpmpc, route, curl, pfctl) is another egress or a host mutation. + "Command::new", + "process::Command", + "Ipv6Addr::UNSPECIFIED", + // A string literal that starts with the IPv6 wildcard, bracketed or not. + "\"[::]", + "\"::", +]; + +pub fn run(_args: &Args) -> BoxResult<()> { + let cargo = std::env::var("CARGO").unwrap_or_else(|_| "cargo".to_owned()); + let out = std::process::Command::new(cargo) + .args([ + "metadata", + "--format-version", + "1", + "--locked", + "--all-features", + ]) + .output()?; + if !out.status.success() { + return Err(format!( + "cargo metadata failed: {}", + String::from_utf8_lossy(&out.stderr) + ) + .into()); + } + let metadata: serde_json::Value = serde_json::from_slice(&out.stdout)?; + + let mut problems = graph_problems(&metadata)?; + problems.extend(feature_problems(&metadata)); + + let glob = globwalk::GlobWalkerBuilder::from_patterns(SCAN_ROOT, &["**/*.rs"]).build()?; + for entry in glob { + let entry = entry?; + let path = entry.path(); + let rel = path + .strip_prefix(SCAN_ROOT) + .unwrap_or(path) + .to_string_lossy() + .replace('\\', "/"); + let contents = std::fs::read_to_string(path)?; + problems.extend(scan_file(&rel, &contents)); + } + + if problems.is_empty() { + return Ok(()); + } + eprintln!("the port mapper's LAN-only boundary is broken:"); + for p in &problems { + eprintln!(" {p}"); + } + eprintln!( + "Every LAN socket and send must go through GatewayCap (ts_portmapper/src/cap.rs); the \ + port mapper must not pull in TLS/HTTP clients, ts_control must not depend on it, and \ + only dev-dependencies may enable its test harness. STOP." + ); + Err("port mapper LAN-only invariant violated".into()) +} + +fn forbidden_dep(name: &str) -> bool { + FORBIDDEN_DEPS.iter().any(|f| match f.strip_suffix('*') { + Some(prefix) => name.starts_with(prefix), + None => name == *f, + }) +} + +/// Package names by id. +type Names = HashMap; +/// The resolve graph: `id -> [(dep id, dependency kinds)]`. +type Edges = HashMap)>>; + +/// Package names by id, and the resolve graph. +fn graph(metadata: &serde_json::Value) -> BoxResult<(Names, Edges)> { + let mut names = HashMap::new(); + for p in metadata["packages"] + .as_array() + .ok_or("metadata: no packages")? + { + if let (Some(id), Some(name)) = (p["id"].as_str(), p["name"].as_str()) { + names.insert(id.to_owned(), name.to_owned()); + } + } + let mut edges = HashMap::new(); + for n in metadata["resolve"]["nodes"] + .as_array() + .ok_or("metadata: no resolve graph")? + { + let id = n["id"].as_str().unwrap_or_default().to_owned(); + let deps = n["deps"] + .as_array() + .map(|ds| { + ds.iter() + .map(|d| { + let kinds = d["dep_kinds"] + .as_array() + .map(|ks| { + ks.iter() + .map(|k| k["kind"].as_str().unwrap_or("normal").to_owned()) + .collect() + }) + .unwrap_or_default(); + (d["pkg"].as_str().unwrap_or_default().to_owned(), kinds) + }) + .collect() + }) + .unwrap_or_default(); + edges.insert(id, deps); + } + Ok((names, edges)) +} + +/// Every package reachable from `start` through edges of the given kinds. +fn closure(edges: &Edges, start: &str, kinds: &[&str]) -> BTreeSet { + let mut seen = BTreeSet::new(); + let mut stack = vec![start.to_owned()]; + while let Some(id) = stack.pop() { + for (dep, dep_kinds) in edges.get(&id).into_iter().flatten() { + if dep_kinds.iter().any(|k| kinds.contains(&k.as_str())) && seen.insert(dep.clone()) { + stack.push(dep.clone()); + } + } + } + seen +} + +/// Invariant 1: the port mapper's closure has no TLS/HTTP crate, and control does not reach it. +fn graph_problems(metadata: &serde_json::Value) -> BoxResult> { + let (names, edges) = graph(metadata)?; + let id_of = |name: &str| { + names + .iter() + .find(|(_, n)| n.as_str() == name) + .map(|(id, _)| id.clone()) + }; + let mut problems = Vec::new(); + let pm = id_of(PORTMAPPER).ok_or("metadata: no ts_portmapper package")?; + for dep in closure(&edges, &pm, &["normal", "build"]) { + let name = names.get(&dep).map_or(dep.as_str(), String::as_str); + if forbidden_dep(name) { + problems.push(format!("{PORTMAPPER} depends on {name} (TLS/HTTP)")); + } + } + if let Some(control) = id_of(CONTROL) { + // Control's own dependencies of every kind, then what those really pull in (normal and + // build edges; a dependency's dev-dependencies are never built for control). + let mut reach = BTreeSet::new(); + for (dep, _) in edges.get(&control).into_iter().flatten() { + reach.insert(dep.clone()); + reach.extend(closure(&edges, dep, &["normal", "build"])); + } + if reach.contains(&pm) { + problems.push(format!("{CONTROL} depends on {PORTMAPPER}")); + } + } + Ok(problems) +} + +/// Invariant 2: only dev-dependencies enable the harness feature. +fn feature_problems(metadata: &serde_json::Value) -> Vec { + let mut problems = Vec::new(); + let members: BTreeSet<&str> = metadata["workspace_members"] + .as_array() + .into_iter() + .flatten() + .filter_map(|m| m.as_str()) + .collect(); + for p in metadata["packages"].as_array().into_iter().flatten() { + let Some(id) = p["id"].as_str() else { continue }; + if !members.contains(id) { + continue; + } + let name = p["name"].as_str().unwrap_or_default(); + if name == PORTMAPPER { + let default = &p["features"]["default"]; + if default + .as_array() + .is_some_and(|d| d.iter().any(|f| f.as_str() == Some(HARNESS_FEATURE))) + { + problems.push(format!( + "{PORTMAPPER}'s default features enable {HARNESS_FEATURE}" + )); + } + continue; + } + let mut aliases = vec!["ts_portmapper".to_owned(), PORTMAPPER.to_owned()]; + for d in p["dependencies"].as_array().into_iter().flatten() { + if d["name"].as_str() != Some(PORTMAPPER) { + continue; + } + if let Some(r) = d["rename"].as_str() { + aliases.push(r.to_owned()); + } + let dev = d["kind"].as_str() == Some("dev"); + let harness = d["features"] + .as_array() + .is_some_and(|fs| fs.iter().any(|f| f.as_str() == Some(HARNESS_FEATURE))); + if harness && !dev { + problems.push(format!( + "{name} enables {PORTMAPPER}/{HARNESS_FEATURE} on a non-dev dependency" + )); + } + } + if let Some(features) = p["features"].as_object() { + for (feature, enables) in features { + for e in enables + .as_array() + .into_iter() + .flatten() + .filter_map(|e| e.as_str()) + { + let hit = aliases.iter().any(|a| { + e == format!("{a}/{HARNESS_FEATURE}") + || e == format!("{a}?/{HARNESS_FEATURE}") + }); + if hit { + problems.push(format!( + "{name}'s feature `{feature}` enables {PORTMAPPER}/{HARNESS_FEATURE}" + )); + } + } + } + } + } + problems +} + +/// `line` without its `//` comment (a `//` inside a string literal is kept). +fn strip_comment(line: &str) -> &str { + let mut in_str = false; + let mut escaped = false; + let bytes = line.as_bytes(); + for (i, &b) in bytes.iter().enumerate() { + if in_str { + match b { + _ if escaped => escaped = false, + b'\\' => escaped = true, + b'"' => in_str = false, + _ => {} + } + } else if b == b'"' { + in_str = true; + } else if b == b'/' && bytes.get(i + 1) == Some(&b'/') { + return &line[..i]; + } + } + line +} + +/// The function a line belongs to, tracked by the last `fn name(` seen. +fn fn_name(line: &str) -> Option<&str> { + let after = line.split("fn ").nth(1)?; + let end = after.find(|c: char| !(c.is_alphanumeric() || c == '_'))?; + (end > 0).then(|| &after[..end]) +} + +/// Whether `code` contains `pattern` starting at a word boundary (so `nix::` does not match inside +/// `std::os::unix::process`, nor `connect(` inside `disconnect(`). +fn contains_at_boundary(code: &str, pattern: &str) -> bool { + code.match_indices(pattern).any(|(i, _)| { + !code[..i] + .chars() + .next_back() + .is_some_and(|c| c.is_alphanumeric() || c == '_') + }) +} + +/// Invariant 3: the tripwire over one file, `rel` relative to [`SCAN_ROOT`]. +fn scan_file(rel: &str, contents: &str) -> Vec { + if SOCKET_MODULES.iter().any(|m| match m.strip_suffix('/') { + Some(dir) => rel.starts_with(&format!("{dir}/")), + None => rel == *m, + }) { + return Vec::new(); + } + let mut hits = Vec::new(); + let mut current_fn: Option = None; + for (i, raw) in contents.lines().enumerate() { + let code = strip_comment(raw); + if let Some(f) = fn_name(code) { + current_fn = Some(f.to_owned()); + } + let Some(pattern) = PATTERNS.iter().find(|p| contains_at_boundary(code, p)) else { + continue; + }; + let allowed = ALLOWED_FNS + .iter() + .any(|(file, f)| *file == rel && current_fn.as_deref() == Some(*f)); + if !allowed { + hits.push(format!( + "{SCAN_ROOT}/{rel}:{}: `{pattern}` outside GatewayCap: {}", + i + 1, + raw.trim() + )); + } + } + hits +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The adversarial fixtures: an aliased import and a qualified path are both caught. + #[test] + fn evasions_are_flagged() { + for line in [ + "use std::net::UdpSocket as Sock;", + "let s = ::std::net::UdpSocket::bind((Ipv4Addr::UNSPECIFIED, 0))?;", + "use tokio::net::{UdpSocket as U};", + // The aliased import is the line that is caught; later uses of the alias are not. + "use socket2 as s2;", + "unsafe { libc::sendto(fd, p, n, 0, sa, l) };", + "sock.send_to(&pkt, dst).await?;", + "s.connect(dst)?;", + "let a = Ipv6Addr::UNSPECIFIED;", + r#"let any = "[::]:0";"#, + r#"let any: SocketAddr = "[::]".parse()?;"#, + r#"let any = ("::", 0);"#, + "unsafe { libc::send(fd, p, n, 0) };", + "unsafe { libc::write(fd, p, n) };", + "let s = unsafe { std::net::UdpSocket::from_raw_fd(fd) };", + "use std::os::fd::FromRawFd;", + "use nix::sys::socket;", + "std::process::Command::new(\"natpmpc\").arg(\"-a\").status()?;", + "let c = Command::new(\"route\");", + "use std::process::Command as Cmd;", + ] { + let src = format!("fn sneaky() {{\n {line}\n}}\n"); + assert_eq!( + scan_file("oneshot.rs", &src).len(), + 1, + "not flagged: {line}" + ); + } + } + + #[test] + fn comments_strings_and_allowed_places_are_not_flagged() { + let src = "/// Sends through a `UdpSocket` in GatewayCap.\n\ + // connect( is fine in a comment\n\ + let url = \"http://x\"; // UdpSocket\n"; + assert!(scan_file("oneshot.rs", src).is_empty()); + let sock = "fn exchange() {\n let s = UdpSocket::bind(a)?;\n s.send_to(b, c)?;\n}\n"; + assert!(scan_file("cap.rs", sock).is_empty()); + assert!(scan_file("testing/router.rs", sock).is_empty()); + let ks = "fn kernel_source(dst: Ipv4Addr) {\n let s = std::net::UdpSocket::bind(x).ok()?;\n s.connect(y).ok()?;\n}\n"; + assert!(scan_file("gateway/mod.rs", ks).is_empty()); + // The same function name elsewhere is not allowed, nor another function in that file. + assert_eq!(scan_file("gateway/darwin.rs", ks).len(), 2); + let other = ks.replace("kernel_source", "helper"); + assert_eq!(scan_file("gateway/mod.rs", &other).len(), 2); + // A file named like an allowed directory is still scanned. + assert_eq!(scan_file("testing.rs", sock).len(), 2); + } + + /// Patterns match at a word boundary only. + #[test] + fn patterns_match_at_word_boundaries() { + for line in [ + "let ppid = std::os::unix::process::parent_id();", + "peer.disconnect();", + "let w = rewrite(x);", + ] { + let src = format!("fn fine() {{\n {line}\n}}\n"); + assert!( + scan_file("probe_tool.rs", &src).is_empty(), + "flagged: {line}" + ); + } + for line in [ + "use ::nix::sys::socket;", + "let x = (nix::unistd::getpid(),);", + ] { + let src = format!("fn sneaky() {{\n {line}\n}}\n"); + assert_eq!(scan_file("probe_tool.rs", &src).len(), 1, "{line}"); + } + } + + #[test] + fn comment_stripping_respects_strings() { + assert_eq!( + strip_comment(r#"let a = "//"; b // c"#), + r#"let a = "//"; b "# + ); + assert_eq!( + strip_comment(r#"let a = "\"//"; b"#), + r#"let a = "\"//"; b"# + ); + assert_eq!(strip_comment("/// doc"), ""); + } + + fn metadata(extra_pm_dep: Option<&str>, control_dep: bool, harness: &str) -> serde_json::Value { + let mut pm_deps = vec![serde_json::json!({ + "pkg": "tokio-id", "dep_kinds": [{"kind": null}] + })]; + if let Some(d) = extra_pm_dep { + pm_deps + .push(serde_json::json!({"pkg": format!("{d}-id"), "dep_kinds": [{"kind": null}]})); + } + let mut control_deps = vec![]; + if control_dep { + control_deps.push(serde_json::json!({"pkg": "pm-id", "dep_kinds": [{"kind": "dev"}]})); + } + let mut packages = vec![ + serde_json::json!({"id": "pm-id", "name": PORTMAPPER, "features": {"test-harness": []}, "dependencies": []}), + serde_json::json!({"id": "tokio-id", "name": "tokio", "features": {}, "dependencies": []}), + serde_json::json!({"id": "control-id", "name": CONTROL, "features": {}, "dependencies": []}), + serde_json::json!({ + "id": "rt-id", "name": "geiserx_ts_runtime", + "features": {"portmapper": ["dep:ts_portmapper"], "x": [harness]}, + "dependencies": [{"name": PORTMAPPER, "rename": "ts_portmapper", "kind": null, "features": []}] + }), + ]; + if let Some(d) = extra_pm_dep { + packages.push(serde_json::json!({"id": format!("{d}-id"), "name": d, "features": {}, "dependencies": []})); + } + serde_json::json!({ + "packages": packages, + "workspace_members": ["pm-id", "control-id", "rt-id"], + "resolve": {"nodes": [ + {"id": "pm-id", "deps": pm_deps}, + {"id": "control-id", "deps": control_deps}, + {"id": "rt-id", "deps": [{"pkg": "pm-id", "dep_kinds": [{"kind": null}]}]}, + ]} + }) + } + + #[test] + fn graph_rules() { + assert!( + graph_problems(&metadata(None, false, "")) + .unwrap() + .is_empty() + ); + for bad in [ + "rustls", + "hyper", + "reqwest", + "aws-lc-sys", + "geiserx_ts_http_util", + ] { + assert_eq!( + graph_problems(&metadata(Some(bad), false, "")) + .unwrap() + .len(), + 1, + "{bad}" + ); + } + assert_eq!(graph_problems(&metadata(None, true, "")).unwrap().len(), 1); + } + + #[test] + fn feature_rules() { + assert!(feature_problems(&metadata(None, false, "")).is_empty()); + assert_eq!( + feature_problems(&metadata(None, false, "ts_portmapper/test-harness")).len(), + 1 + ); + assert_eq!( + feature_problems(&metadata(None, false, "ts_portmapper?/test-harness")).len(), + 1 + ); + let mut m = metadata(None, false, ""); + m["packages"][3]["dependencies"][0]["features"] = serde_json::json!(["test-harness"]); + assert_eq!(feature_problems(&m).len(), 1); + m["packages"][3]["dependencies"][0]["kind"] = serde_json::json!("dev"); + assert!(feature_problems(&m).is_empty()); + let mut d = metadata(None, false, ""); + d["packages"][0]["features"]["default"] = serde_json::json!(["test-harness"]); + assert_eq!(feature_problems(&d).len(), 1); + } +} From 80ea74a813c968e369f497c7e0c2fcf2f4a631ab Mon Sep 17 00:00:00 2001 From: Kasper Nielsen Date: Sat, 17 Oct 2026 23:06:47 +0200 Subject: [PATCH 07/12] feat(checks): keep tests off the LAN gateway The port mapper is on by default, so a test that builds a real device would ask the host's router for a mapping. A new check, portmapper_test_isolation, makes the guards a gate: - tests/*.rs that build a Device or Server must set TS_DISABLE_PORTMAPPER; - runtime sources outside env.rs and portmap.rs may not enable the mapper; - the cfg(test) gateway-lookup stub in portmap.rs must stay. Signed-off-by: Kasper Nielsen Co-Authored-By: GPT-6.1 (codex) Co-Authored-By: Claude Opus 5.5 --- checks/src/main.rs | 2 + checks/src/portmapper_test_isolation.rs | 145 ++++++++++++++++++++++++ 2 files changed, 147 insertions(+) create mode 100644 checks/src/portmapper_test_isolation.rs diff --git a/checks/src/main.rs b/checks/src/main.rs index 4e4b84a2..75a8e781 100644 --- a/checks/src/main.rs +++ b/checks/src/main.rs @@ -13,6 +13,7 @@ mod no_std_honest; mod parity_audit; mod porting_ledger; mod portmapper_lan_only; +mod portmapper_test_isolation; mod restock_beads; mod ssh_isolation; @@ -40,6 +41,7 @@ pub const CHECK_FNS: &[(&str, CheckFn)] = &[ ("funnel_fail_closed", funnel_fail_closed::run), ("ssh_isolation", ssh_isolation::run), ("portmapper_lan_only", portmapper_lan_only::run), + ("portmapper_test_isolation", portmapper_test_isolation::run), ("porting_ledger", porting_ledger::run), ("parity_audit", parity_audit::run), ("restock_beads", restock_beads::run), diff --git a/checks/src/portmapper_test_isolation.rs b/checks/src/portmapper_test_isolation.rs new file mode 100644 index 00000000..3841b1bc --- /dev/null +++ b/checks/src/portmapper_test_isolation.rs @@ -0,0 +1,145 @@ +//! Test isolation for the NAT port mapper: no test may ask the real LAN gateway for a mapping. +//! +//! WHY: the port mapper is on by default (Go parity), so a test that builds a real device or runtime +//! would look up the host's default route and send NAT-PMP/PCP to the real router, creating real +//! mappings. The runtime's own unit tests are guarded in code: every `ForwarderConfig` names its +//! `port_mapper` field (the shared test configurations set `None`), and in that crate's test build +//! the production client's gateway lookup is a stub that panics (`ts_runtime/src/portmap.rs`, +//! `shared_test_config_never_builds_a_client`). This check guards what code cannot: +//! +//! 1. **Integration tests that build devices** (`tests/*.rs` calling `Device::new` or +//! `Server::new`) must turn the mapper off with `TS_DISABLE_PORTMAPPER` (the env knob covers +//! every device the process builds, the facade server included). +//! 2. **Runtime sources** may enable the mapper (`port_mapper: Some(`) only in the production +//! conversion (`env.rs`) and the driver's own tests (`portmap.rs`), where a harness is injected +//! or the stub lookup is in place. +//! 3. **The stub stays**: `portmap.rs` keeps the `cfg(test)` lookup stub. +//! +//! Like any text scan it is a tripwire, not a proof. + +use crate::{Args, BoxResult}; + +/// What a device-building integration test must contain. +const ENV_KNOB: &str = "TS_DISABLE_PORTMAPPER"; +/// Calls that build a device. +const DEVICE_BUILDERS: &[&str] = &["Device::new(", "Server::new("]; +/// Runtime files allowed to enable the mapper. +const ENABLE_ALLOWED: &[&str] = &["env.rs", "portmap.rs"]; + +pub fn run(_args: &Args) -> BoxResult<()> { + let mut problems = Vec::new(); + + for entry in globwalk::GlobWalkerBuilder::from_patterns("tests", &["*.rs"]).build()? { + let entry = entry?; + let contents = std::fs::read_to_string(entry.path())?; + problems.extend(integration_test_problems( + &entry.path().display().to_string(), + &contents, + )); + } + + for entry in + globwalk::GlobWalkerBuilder::from_patterns("ts_runtime/src", &["**/*.rs"]).build()? + { + let entry = entry?; + let path = entry.path(); + let name = path + .file_name() + .map(|n| n.to_string_lossy().into_owned()) + .unwrap_or_default(); + let contents = std::fs::read_to_string(path)?; + problems.extend(runtime_problems(&name, &contents)); + } + + let portmap = std::fs::read_to_string("ts_runtime/src/portmap.rs")?; + problems.extend(stub_problems(&portmap)); + + if problems.is_empty() { + return Ok(()); + } + eprintln!("a test could reach the real LAN gateway through the port mapper:"); + for p in &problems { + eprintln!(" {p}"); + } + Err("port mapper test isolation violated".into()) +} + +/// An integration test file that builds a device must set the env knob. +fn integration_test_problems(path: &str, contents: &str) -> Vec { + let builds = DEVICE_BUILDERS.iter().any(|b| contents.contains(b)); + if builds && !contents.contains(ENV_KNOB) { + vec![format!( + "{path}: builds a device but never sets {ENV_KNOB}; set it (and `disable_port_mapper`) \ + before the first device" + )] + } else { + Vec::new() + } +} + +/// A runtime source file other than the allowed ones must not enable the mapper. +fn runtime_problems(name: &str, contents: &str) -> Vec { + if ENABLE_ALLOWED.contains(&name) { + return Vec::new(); + } + contents + .lines() + .enumerate() + .filter(|(_, l)| l.replace(' ', "").contains("port_mapper:Some(")) + .map(|(i, _)| { + format!( + "ts_runtime/src/{name}:{}: enables the port mapper; tests must use `port_mapper: \ + None` or inject a harness in portmap.rs", + i + 1 + ) + }) + .collect() +} + +/// The `cfg(test)` lookup stub must stay in place. +fn stub_problems(portmap: &str) -> Vec { + if portmap.contains("cfg.lookup = Arc::new(tests::NeverLookup)") && portmap.contains("panic!(") + { + Vec::new() + } else { + vec![ + "ts_runtime/src/portmap.rs: the cfg(test) gateway-lookup stub (NeverLookup) is gone" + .to_owned(), + ] + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn device_builders_need_the_env_knob() { + let bad = "async fn t() { Device::new(&Config::default(), None).await; }"; + assert_eq!(integration_test_problems("tests/x.rs", bad).len(), 1); + let bad = "let mut srv = Server::new();"; + assert_eq!(integration_test_problems("tests/x.rs", bad).len(), 1); + let ok = "unsafe { std::env::set_var(\"TS_DISABLE_PORTMAPPER\", \"1\") }; Device::new(&c)"; + assert!(integration_test_problems("tests/x.rs", ok).is_empty()); + assert!(integration_test_problems("tests/x.rs", "fn no_device() {}").is_empty()); + } + + #[test] + fn only_allowed_runtime_files_enable_the_mapper() { + let src = "let f = ForwarderConfig {\n port_mapper: Some(settings),\n};"; + assert_eq!(runtime_problems("netmon.rs", src).len(), 1); + assert_eq!( + runtime_problems("netmon.rs", "port_mapper : Some( x )").len(), + 1 + ); + assert!(runtime_problems("portmap.rs", src).is_empty()); + assert!(runtime_problems("netmon.rs", "port_mapper: None,").is_empty()); + } + + #[test] + fn the_stub_must_stay() { + assert_eq!(stub_problems("fn system_client() {}").len(), 1); + let ok = "cfg.lookup = Arc::new(tests::NeverLookup); panic!(\"x\")"; + assert!(stub_problems(ok).is_empty()); + } +} From 7cfbe2472f37c4c2d94d6143ee896ea16a1543ea Mon Sep 17 00:00:00 2001 From: Kasper Nielsen Date: Sat, 17 Oct 2026 23:06:47 +0200 Subject: [PATCH 08/12] docs: record the port mapper in the ledger and roadmap PORTING.md maps net/portmapper's NAT-PMP and PCP halves to ts_portmapper and the runtime driver, leaves UPnP-IGD and feature/debugportmapper as roadmap, and counts the NetInfo facets now reported: seven of Go's twelve, with UPnP modelled but never set. The migration status and parity roadmap mark NAT-PMP and PCP done and keep tsr-x4yg open for UPnP-IGD. Signed-off-by: Kasper Nielsen Co-Authored-By: GPT-6.1 (codex) Co-Authored-By: Claude Opus 5.5 --- PORTING.md | 24 ++++++++++++++---------- docs/MIGRATION_STATUS.md | 4 ++-- docs/PARITY_ROADMAP.md | 9 +++++---- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/PORTING.md b/PORTING.md index be0179ef..529851c8 100644 --- a/PORTING.md +++ b/PORTING.md @@ -86,6 +86,7 @@ where that behaviour lives", not "this is a transliteration of that file". | `wgengine/router`, OS side of `net/dns` | [`ts_host_net`](ts_host_net/src/lib.rs) (Linux `ip`/`resolvectl`, macOS `route`/`scutil`) | | `net/dns/resolver` wire encoding | [`ts_dns_wire`](ts_dns_wire/src/lib.rs) + the MagicDNS server in [`ts_runtime::magic_dns`](ts_runtime/src/magic_dns.rs) | | `net/netmon` | [`ts_netmon`](ts_netmon/src/lib.rs) | +| `net/portmapper`, `feature/portmapper` — NAT-PMP and PCP (UPnP-IGD is roadmap) | [`ts_portmapper`](ts_portmapper/src/lib.rs) (the lifecycle client, codecs, gateway lookup and `GatewayCap`) + the driver in [`ts_runtime::portmap`](ts_runtime/src/portmap.rs); `--portmap-probe` in [`examples/exit_node`](examples/exit_node/main.rs). Deviations: [`docs/PORTMAPPER.md`](docs/PORTMAPPER.md) | | `net/art` + `github.com/gaissmai/bart` | [`ts_bart`](ts_bart/src/lib.rs) (+ [`ts_array256`](ts_array256/src/lib.rs), [`ts_bitset`](ts_bitset/src/lib.rs), [`ts_dynbitset`](ts_dynbitset/src/lib.rs)) | ### Runtime, API and utilities @@ -118,7 +119,8 @@ cut with its eyes open. Items already tracked in the 3-way bind handshake and the Geneve-framed relay data path); this node never serves as a relay itself, and does not send `AllocateUDPRelayEndpointRequest` — a relay-capable peer allocates on our behalf and announces the endpoint with a `CallMeMaybeVia`. -- `net/portmapper`, `feature/portmapper`, `feature/debugportmapper` — UPnP / PCP / NAT-PMP *(roadmap)*. +- `feature/debugportmapper`, and UPnP-IGD (the third protocol of `net/portmapper`, whose NAT-PMP + and PCP halves are mapped above) — the debug CLI and SSDP/SOAP mapping *(roadmap)*. - `appc`, `feature/conn25`, `types/appctype` — app connectors (classic and conn25) *(roadmap)*. - `drive`, `feature/drive` — Taildrive. - `ipn/ipnserver`, `cmd/tailscaled`, `cmd/tailscale` — the daemon and its CLI. This is an embedded @@ -2002,8 +2004,8 @@ rows were ported in the interval. - already §A rows, assessed there and not repeated: `DisableHostsFileUpdates` (132), `ForceRegisterMagicDNSIPv4Only` (133, the one open §A row), `EmitRuntimeMetrics` (139), the four GRO/GSO knobs (140), `NeverGSOEqualTail` (141), `ScopeQuad100OnMacOS` (145). - - no subsystem to switch: `DisableUPnP` and `ProbeUDPLifetime` (no portmapper, no UDP-lifetime - probing — *roadmap*), `PeerMTUEnable` (no path-MTU discovery on this datapath), + - no subsystem to switch: `DisableUPnP` and `ProbeUDPLifetime` (no UPnP in the port mapper, + which speaks NAT-PMP and PCP only, and no UDP-lifetime probing — *roadmap*), `PeerMTUEnable` (no path-MTU discovery on this datapath), `LinuxForceIPTables` / `LinuxForceNfTables` (`ts_host_net` installs no firewall rules), `AppCStoreRoutes` (no app connector — *roadmap*), `DisableCaptivePortalDetection` (no captive-portal detection), `DisableSplitDNSWhenNoCustomResolvers` (an iOS battery optimisation @@ -2030,7 +2032,7 @@ rows were ported in the interval. socket-binding caps, and the debug caps `debug-no-wg-trim` (no lazy WireGuard config here) and `debug-disable-subnets-if-pac` (no WPAD detection). -- **`NetInfo` is reported with four of the twelve facets Go fills** +- **`NetInfo` is reported with seven of the twelve facets Go fills** (`wgengine/magicsock/magicsock.go`: `updateNetInfo`) — **not applicable**, on grounds this fork has already settled elsewhere. (The thirteenth field this tree models is `HairPinning`, and that one is a row of its own above.) Go builds a whole `tailcfg.NetInfo` @@ -2038,12 +2040,14 @@ rows were ported in the interval. `DERPLatency` map keyed `"-v4"`/`"-v6"`, `WorkingIPv6`, `OSHasIPv6`, `WorkingUDP`, `WorkingICMPv4`, `PreferredDERP` and `FirewallMode`, and hands it to a callback that ships it on the next map request — de-duplicated by `BasicallyEqual`, so an unchanged report is not re-uploaded. - `ts_control::tokio::client`'s `CarriedNetInfo` carries four of those — `preferred_derp`, - `derp_latency`, `working_udp`, `mapping_varies_by_dest_ip` — and applies the whole of it to every - map request, which is Go's `hi.NetInfo = c.netinfo.Clone()` invariant and is already tested here. - The eight it does not carry each have an answer already recorded in the *no counterpart here* list - or in this fork's design: `UPnP`, `PMP`, `PCP` and `HavePortMap` need `net/portmapper`, which is - roadmap; `FirewallMode` describes an iptables-vs-nftables choice `ts_host_net` never makes because + `ts_control::tokio::client`'s `CarriedNetInfo` models eight of those — `preferred_derp`, + `derp_latency`, `working_udp`, `mapping_varies_by_dest_ip`, and from the port mapper `UPnP`, + `PMP`, `PCP` and `HavePortMap` — and applies the whole of it to every map request, which is Go's + `hi.NetInfo = c.netinfo.Clone()` invariant and is already tested here. `UPnP` is never set (so + seven are reported): it is never probed, so it is never claimed false. `HavePortMap` here means an *advertised* mapping, so + control never sees it without a port-mapped endpoint (see + [`docs/PORTMAPPER.md`](docs/PORTMAPPER.md)). The four it does not carry each have an answer + already recorded in the *no counterpart here* list or in this fork's design: `FirewallMode` describes an iptables-vs-nftables choice `ts_host_net` never makes because it installs no firewall rules (the same reason capability version 136 is not applicable); `WorkingIPv6` and `OSHasIPv6` come from a netcheck that probes v6, and this fork's `ts_netcheck` is HTTPS-over-IPv4 only by deliberate anti-leak design — the note at the top of diff --git a/docs/MIGRATION_STATUS.md b/docs/MIGRATION_STATUS.md index c053e7fa..6d0551f6 100644 --- a/docs/MIGRATION_STATUS.md +++ b/docs/MIGRATION_STATUS.md @@ -30,7 +30,7 @@ Verdict legend: **PRESENT** = works, no material gap · **PARTIAL** = works with | Subsystem | Verdict | Notes | Open beads | |---|---|---|---| | Control protocol (register/auth, map-poll, Noise/TS2021, zstd, capver) | **PARTIAL** | + auto-reauth on key expiry, NeedsMachineAuth-transient, zstd map responses, Container/Env hostinfo — all shipped this cycle. Complete on an unlocked tailnet. On a locked tailnet the node never sends its own `NodeKeySignature` (2026-10-14 rows C1 to C3), and several MapResponse and PingRequest fields are parsed but not acted on (rows C9 to C14) | parity polish: `tsr-84mh`, `tsr-cbw4`, `tsr-ajyc`. Rows C1 to C3 and C9 to C14 have no bead | -| MagicSock / disco / direct paths | **PARTIAL** | STUN/DERP/CallMeMaybe/ping-pong/best-addr work; peer-relay receiving side present (disco 0x04–0x09 codecs, bind handshake, Geneve data path for a relay a peer allocated); the relay **client** side that discovers relay servers and allocates one is absent (2026-10-14 row M18); **portmapper absent**, relay **serving** absent | `tsr-x4yg` (portmapper), `tsr-a9x4` (peer relays), `tsr-dsjq` (full netcheck), `tsr-7s3` (ping cadence), `tsr-qc3o` (NoPath demand-ping), `tsr-51o6` (flaky test) | +| MagicSock / disco / direct paths | **PARTIAL** | STUN/DERP/CallMeMaybe/ping-pong/best-addr work; peer-relay receiving side present (disco 0x04–0x09 codecs, bind handshake, Geneve data path for a relay a peer allocated); the relay **client** side that discovers relay servers and allocates one is absent (2026-10-14 row M18); **portmapper: NAT-PMP and PCP present** (`ts_portmapper` + `ts_runtime::portmap`; UPnP-IGD absent), relay **serving** absent | `tsr-x4yg` (portmapper; UPnP is phase 2), `tsr-a9x4` (peer relays), `tsr-dsjq` (full netcheck), `tsr-7s3` (ping cadence), `tsr-qc3o` (NoPath demand-ping), `tsr-51o6` (flaky test) | | WireGuard tunnel | **PARTIAL** | initiator path complete (handshake/session/rekey/replay/cookie-consume); responder cookie issuance missing | `tsr-rl3n` (responder cookie + IsUnderLoad), `tsr-qn2` (MTU-aware padding) | | Netstack (userspace TCP/IP + TUN) | **PARTIAL** | TCP/UDP both modes; needs egress backpressure + icmp-echo | `tsr-fqus` (ingress bounded since #587 at 4096 packets and 4096×MTU bytes, `ts_runtime/src/netstack_actor.rs:49-64`; egress still unbounded, `ts_runtime/src/dataplane.rs:115`, `:141`), `tsr-rsu` (auto-icmp-echo), `tsr-4pp` (per-session sharding) | | MagicDNS | **PARTIAL** | responder + plain-UDP upstream + recursive (TUN) work; encrypted upstream dropped | `tsr-3l78` (DoH/DoT upstream), `tsr-i2w` (exit-node DoH authoritative-vs-forward) | @@ -139,7 +139,7 @@ merged or closed. **P2 (next-highest leverage):** - `tsr-doz6` netmon (in-progress: slice a shipped; `tsr-7wrn` Linux + `tsr-cnlh` macOS backends remain) -- `tsr-x4yg` portmapper (UPnP-IGD/NAT-PMP/PCP) — EPIC; direct paths behind mapping-NATs stay on DERP +- `tsr-x4yg` portmapper — EPIC. NAT-PMP and PCP landed (phases 0–1: lifecycle client, runtime driver, `PortMapped` endpoint, NetInfo `PMP`/`PCP`/`HavePortMap`); UPnP-IGD (phase 2) is still open, so a router with UPnP only stays on DERP - `tsr-0xn` Serve Go-wire-compatibility (closed by `50611ab`) - `tsr-fqus` netstack backpressure (ingress half closed by #587; egress still unbounded) - `tsr-7s3` magicsock ping cadence / path-selection parity (in-progress) diff --git a/docs/PARITY_ROADMAP.md b/docs/PARITY_ROADMAP.md index 0318925b..e7ff1b40 100644 --- a/docs/PARITY_ROADMAP.md +++ b/docs/PARITY_ROADMAP.md @@ -140,8 +140,8 @@ refused, `control_runner.rs:1430`). | M19 | No WireGuard persistent keepalive anywhere. A grep for `PersistentKeepalive` over the whole Go tree finds 0 hits, and `wgengine/wgcfg/nmcfg` does not exist | A 25 s keepalive is on by default for every peer (`ts_control/src/config.rs:102-114`, `:837`) | product-decision | The idle-wedge fix. A deliberate divergence that the code comment called Go parity. Bead `tsr-0hd2` (blocked). | | M25 | Static endpoints from config (`magicsock.go:1002`, `:1407`) and cloud-metadata public IPs (`magicsock.go:1373`) | Not found. A grep for `static_endpoints\|GetPublicIPs\|cloud_info` finds nothing | product-decision | A metadata query is an extra egress. | -Still open from earlier docs and beads: the portmapper (`tsr-x4yg`, nothing under -`portmap\|upnp\|nat_pmp\|pcp_`), the netmon OS backends (`tsr-7wrn`, `tsr-cnlh`; only +Still open from earlier docs and beads: the portmapper's UPnP-IGD half (`tsr-x4yg`; NAT-PMP and +PCP are in `ts_portmapper`), the netmon OS backends (`tsr-7wrn`, `tsr-cnlh`; only `ManualLinkMonitor` and `NoopLinkMonitor` exist, `ts_netmon/src/lib.rs:260`, `:297`), a full netcheck report (`tsr-dsjq`; `ts_netcheck/src` holds only `derp_latency.rs` and `https.rs`), the NoPath demand ping (`tsr-qc3o`, `sock.rs:2832-2836`), `disable-relay-client` (`tsr-re0d2b5d`), Mullvad and @@ -296,7 +296,7 @@ except two: #616 replaced #600 by taking the fix from smoltcp 0.14.0, and #604 c | `tsr-7wrn` | P2 | bug | open | tsr-doz6 slice b: Linux netlink LinkMonitor backend | | `tsr-cnlh` | P2 | bug | open | tsr-doz6 slice c: macOS PF_ROUTE LinkMonitor backend | | `tsr-re0d2b5d` | P2 | bug | open | Honour disable-relay-client instead of opening relay paths control switched off | -| `tsr-x4yg` | P2 | bug | open | magicsock: no portmapper (UPnP-IGD/NAT-PMP/PCP), direct paths behind mapping-NATs stay on DERP (EPIC) | +| `tsr-x4yg` | P2 | bug | open | magicsock portmapper (EPIC): NAT-PMP and PCP done (`ts_portmapper`, `ts_runtime::portmap`); UPnP-IGD (phase 2) open, so UPnP-only routers stay on DERP | | `tsr-0h2` | P3 | feature | open | Daemon ask: SSH check-mode (HoldAndDelegate) round-trip | | `tsr-1rs` | P3 | task | open | harden flaky udp_forwarder_splices test (real-UDP round-trip on loaded CI) | | `tsr-3l78` | P3 | bug | open | magic-dns: DoH/DoT/HttpWireguard upstream resolvers parsed-then-dropped, encrypted-upstream split-DNS silently ignored | @@ -460,7 +460,8 @@ Most recent wave: `tka_disable`, `src/lib.rs:1347-1403`). Still missing: add and remove keys, local disable, recovery AUMs, affected-sigs and wrap-preauth-key (2026-10-14 row C8), and the node's own `NodeKeySignature` on register (rows C1 to C3). -- **UPnP / PCP / NAT-PMP portmapper** — gateway port-mapping for better direct connectivity. +- **UPnP-IGD port mapping** — NAT-PMP and PCP are done ([`ts_portmapper`](../ts_portmapper/src/lib.rs)); + UPnP-IGD (SSDP discovery, SOAP mapping) is phase 2. - **App connector** — `4via6`-style application connectors. - **`4via6`** subnet-route encoding. - ~~**Serve get/set_serve_config + accept-loop runtime**~~: shipped. `Device::set_serve_config` From 85886891964d26179d49da4cd41830653f5198fa Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 18 Oct 2026 18:11:17 +0200 Subject: [PATCH 09/12] fix(portmapper): match Go on retry holds, reset order and tombstone lifetime Drop the 30 min NOT_AUTHORIZED hold (Go retries on the next ReSTUN), run the epoch-rollback check before recording the fresh probe answer as Go does, and bound an unanswered MAP's possible grant to the requested 7200 s so four left networks cannot stop port mapping for the process. State the remaining deliberate differences from Go v1.102.4 in docs/PORTMAPPER.md and correct the notification and fuzzer provenance claims. --- docs/PORTMAPPER.md | 30 +++-- ts_portmapper/fuzz/fuzz_targets/parse_pcp.rs | 3 +- .../fuzz/fuzz_targets/parse_pcp_map.rs | 8 +- ts_portmapper/src/cap.rs | 6 +- ts_portmapper/src/client_tests.rs | 108 ++++++++++++++++-- ts_portmapper/src/lifecycle.rs | 10 +- ts_portmapper/src/oneshot.rs | 4 +- ts_portmapper/src/owned.rs | 5 +- ts_portmapper/src/pcp.rs | 5 +- ts_portmapper/src/schedule.rs | 17 +-- 10 files changed, 158 insertions(+), 38 deletions(-) diff --git a/docs/PORTMAPPER.md b/docs/PORTMAPPER.md index 99a2429e..4c6e0245 100644 --- a/docs/PORTMAPPER.md +++ b/docs/PORTMAPPER.md @@ -17,13 +17,18 @@ but never sends on it.) Nothing in the node runs it yet. - **Renewal while idle (`renew_while_idle`).** The default is Go's: renewal rides on activity (the periodic ReSTUN and other kicks). With the option on, a due mapping is renewed even while idle, as long as there are peers. With no peers a mapping is never renewed and expires. -- **Notifications for every protocol.** Success, loss, expiry and disable publish for NAT-PMP and - PCP alike. In Go only PCP successes publish. +- **Notifications.** Go publishes every successful mapping (NAT-PMP, PCP, UPnP, including a + reused one) and never a loss, expiry or disable. Here loss, expiry and disable publish too, for + both protocols. - **Strict replies.** Replies are accepted only from exactly `gateway:5351`, and every field is validated: version, opcode, R bit, length, internal port, protocol and nonce. Go's probe checks only the source port. - **NAT-PMP delete.** The suggested external port is 0, as RFC 6886 §3.4 requires. Go sends the current one. +- **PCP nonce and delete.** The nonce is kept for renewal and delete and every MAP reply must echo + it (RFC 6887 §11.1, §15.1). Go v1.102.4 draws a fresh nonce for every request, including the + delete, and never checks the reply nonce. The PCP delete suggests port 0 and `::ffff:0.0.0.0`; + Go sends the current external address and port. - **Renewal protocol.** A usable mapping keeps its owning protocol, even if another service becomes available. PMP and PCP may share one router rule, so switching and then deleting the old rule could delete its replacement. @@ -40,6 +45,17 @@ but never sends on it.) Nothing in the node runs it yet. external address is rejected; Go accepts it. - **Epoch.** Go's rule applies to probe replies only. An epoch that goes backwards drops the mapping without a release. +- **PCP seen.** A PCP ANNOUNCE reply counts as "PCP seen" and is checked for an epoch rollback + only when its result is SUCCESS. Go records any ANNOUNCE reply, NOT_AUTHORIZED and + ADDRESS_MISMATCH included, and then prefers PCP for the next MAP. +- **Attempt spacing.** Go retries a failed create on the next ReSTUN with no backoff (only a 5 s + skip after a probe that found nothing). Here consecutive failures space attempts + 10/30/60/120/300/600 s (±20 %), no two attempts start within 2 s, and a PCP error lifetime is + honoured up to 30 min; while that backoff holds, a probe after one that found nothing is + skipped too. This bounds LAN traffic to a refusing or absent router. +- **Probe and MAP exchange.** The probe returns as soon as NAT-PMP and PCP are both decided (Go + waits the full 250 ms unless UPnP also answered), and the NAT-PMP MAP always sends the + public-address request alongside (Go sends it only when it has no recent public address). ## Router state, stated honestly @@ -61,11 +77,11 @@ but never sends on it.) Nothing in the node runs it yet. after socket setup, route validation and the final send permit. A fresh never-sent identity retires without DELETE; an unsent adoption/renewal retains its previous grant and nonce/protocol. NAT-PMP's public-address request alone cannot create a mapping. A matched success records the - actual grant. After a MAP attempt without a success reply, the grant could exceed the request, - so the tombstone conservatively uses the full wire maximum (`u32::MAX` seconds, about 136 years) - from the latest syscall attempt. - Four unresolved rules can therefore stop further creates for this process. Wall-clock jumps - withdraw advertisement trust but do not shorten possible ownership. + actual grant. An unanswered MAP keeps its slot for the requested lifetime (7200 s) from the + latest syscall attempt: a NAT-PMP gateway should not grant longer than requested (RFC 6886 + §3.3), and Go trusts no unanswered MAP at all. Four unresolved rules can therefore stop further + creates for up to two hours, never for the rest of the process. Wall-clock jumps withdraw + advertisement trust but do not shorten possible ownership. - **Network down.** Nothing is deleted, since a delete could reach the wrong router. Such rules are left to expire. A create for the same rule on the same network adopts them, because PCP refuses a second nonce for a live rule. diff --git a/ts_portmapper/fuzz/fuzz_targets/parse_pcp.rs b/ts_portmapper/fuzz/fuzz_targets/parse_pcp.rs index 09678ef6..5bdce367 100644 --- a/ts_portmapper/fuzz/fuzz_targets/parse_pcp.rs +++ b/ts_portmapper/fuzz/fuzz_targets/parse_pcp.rs @@ -1,4 +1,5 @@ -//! PCP header and ANNOUNCE parsers (port of Go's `FuzzParsePCPResponse`, with invariants). +//! PCP header and ANNOUNCE parsers (port of upstream Go's `FuzzParsePCPResponse`, on tailscale +//! main after v1.102.4, with invariants). #![no_main] use libfuzzer_sys::fuzz_target; diff --git a/ts_portmapper/fuzz/fuzz_targets/parse_pcp_map.rs b/ts_portmapper/fuzz/fuzz_targets/parse_pcp_map.rs index 4d3e7ad7..4ab22e67 100644 --- a/ts_portmapper/fuzz/fuzz_targets/parse_pcp_map.rs +++ b/ts_portmapper/fuzz/fuzz_targets/parse_pcp_map.rs @@ -1,7 +1,7 @@ -//! PCP MAP reply parser (port of Go's `FuzzParsePCPMapResponse`): exactly one of a mapping or an -//! error (by construction), a MAP parser never accepts another opcode, a success always satisfies -//! the documented invariants, and only a full-length response that echoes our nonce is ever a -//! success or an authoritative router error. +//! PCP MAP reply parser (port of upstream Go's `FuzzParsePCPMapResponse`, on tailscale main after +//! v1.102.4): exactly one of a mapping or an error (by construction), a MAP parser never accepts +//! another opcode, a success always satisfies the documented invariants, and only a full-length +//! response that echoes our nonce is ever a success or an authoritative router error. #![no_main] use std::num::NonZeroU16; diff --git a/ts_portmapper/src/cap.rs b/ts_portmapper/src/cap.rs index 20ddfe07..82becbeb 100644 --- a/ts_portmapper/src/cap.rs +++ b/ts_portmapper/src/cap.rs @@ -306,9 +306,13 @@ impl SendEvidence { .runtime_origin .checked_add(elapsed) .unwrap_or(self.runtime_origin); + // An unanswered MAP's possible grant is bounded by the requested lifetime: a NAT-PMP + // gateway SHOULD NOT grant longer than requested (RFC 6886 §3.3). Go trusts no + // unanswered MAP at all (net/portmapper/portmapper.go:394-411), so this is already + // stricter than Go, and a tombstone for a network we left cannot hold its slot for ever. Some(crate::oneshot::lease_deadline( logical.max(runtime), - u32::MAX, + crate::schedule::REQUESTED_LIFETIME, )) } } diff --git a/ts_portmapper/src/client_tests.rs b/ts_portmapper/src/client_tests.rs index ca14342a..2bf2fb91 100644 --- a/ts_portmapper/src/client_tests.rs +++ b/ts_portmapper/src/client_tests.rs @@ -862,8 +862,9 @@ async fn drop_path_release_needs_no_runtime() { // --------------------------------------------------------------------------------------------- // Router errors, hostile replies, backoff -/// Router refusals map to no mapping and backoff; NOT_AUTHORIZED holds ≥ 30 min; a PCP -/// error lifetime is honoured; ANNOUNCE refusals report `pcp = false`. +/// Router refusals map to no mapping and backoff; NOT_AUTHORIZED takes the ordinary backoff (no +/// long hold, as Go has none); a PCP error lifetime is honoured; ANNOUNCE refusals report +/// `pcp = false`. #[tokio::test] async fn router_errors_back_off() { for code in 1..=5u16 { @@ -875,11 +876,7 @@ async fn router_errors_back_off() { assert!(r.map().await.is_none()); assert_eq!(r.client.owned_rules(), 0, "a refused create left nothing"); let wait = r.client.schedule().next_attempt.unwrap() - r.clock.now(); - if code == 2 { - assert!(wait >= secs(30 * 60) - secs(1), "{code}: {wait:?}"); - } else { - assert!(wait >= secs(7) && wait <= secs(13), "{code}: {wait:?}"); - } + assert!(wait >= secs(7) && wait <= secs(13), "{code}: {wait:?}"); // No hot loop: many ticks inside the backoff send nothing. for _ in 0..50 { r.client.tick().await; @@ -919,7 +916,8 @@ async fn router_errors_back_off() { r.client.tick().await; r.settle().await; let wait = r.client.schedule().next_attempt.unwrap() - r.clock.now(); - assert!(wait >= secs(30 * 60) - secs(1), "NOT_AUTHORIZED: {wait:?}"); + // The second consecutive failure: 30 s ±20 %, no 30 min hold. + assert!(wait >= secs(23) && wait <= secs(37), "NOT_AUTHORIZED: {wait:?}"); r.shutdown().await; // ANNOUNCE refused: PCP present but unusable. @@ -1171,6 +1169,50 @@ async fn epoch_rollback_drops_without_release() { } } +/// The probe that detects a router restart still counts as seeing the service, so the remap goes +/// ahead at once on the same protocol without another probe (Go invalidates before it records the +/// fresh answer, net/portmapper/portmapper.go:1098-1101, 1138-1141). +#[tokio::test] +async fn remap_right_after_the_rollback_probe() { + for pmp in [false, true] { + let r = rig_with(Opts { + router: RouterOptions { + pmp, + ..RouterOptions::default() + }, + ..Opts::default() + }) + .await; + r.router.set_epoch(5000); + let first = r.map().await; + assert!(first.is_some(), "pmp={pmp}"); + let held_pcp = r.router.counters().num_pcp_map_recv; + let held_pmp = r.router.counters().num_pmp_map_recv; + assert_eq!((held_pmp > 0, held_pcp > 0), (pmp, !pmp), "mapped over the expected protocol"); + r.router.reboot(); + r.advance(secs(601)); + r.client.probe().await.unwrap(); + assert!(!r.client.have_mapping(), "pmp={pmp}: rollback detected"); + let probes = { + let c = r.router.counters(); + c.num_pcp_disco_recv + }; + // No further probe: the remap starts from the rollback probe's answer. + assert_eq!(r.client.get_cached_mapping_or_start_creating_one(), None); + r.settle().await; + assert!(r.client.have_mapping(), "pmp={pmp}: remapped at once"); + let c = r.router.counters(); + assert_eq!(c.num_pcp_disco_recv, probes, "pmp={pmp}: no second probe"); + if pmp { + assert_eq!(c.num_pmp_map_recv, held_pmp + 1, "remapped over NAT-PMP"); + assert_eq!(c.num_pcp_map_recv, held_pcp, "not switched to PCP"); + } else { + assert_eq!(c.num_pcp_map_recv, held_pcp + 1, "remapped over PCP"); + } + r.shutdown().await; + } +} + /// A stale probe's epoch cannot drop a newer mapping. #[tokio::test] async fn stale_probe_cannot_drop_a_newer_mapping() { @@ -4782,6 +4824,52 @@ async fn unsent_adoption_preserves_prior_expiry() { } } +/// A roaming host that leaves four networks, each with one MAP whose reply was lost, is not shut +/// out of port mapping for the rest of the process: each tombstone holds its slot only for the +/// requested lifetime from its send attempt. +#[tokio::test] +async fn left_networks_with_lost_replies_free_their_slots() { + let r = rig().await; + // Deletes are never acknowledged, so a tombstone cannot be cleaned up while still present. + r.router.set_drop_deletes(true); + r.router.knobs_for(Some(RequestKind::PmpMap)); + let gws: Vec = (1..=5).map(|n| Ipv4Addr::new(10, 0, n, 1)).collect(); + for gw in &gws { + r.caps.add(*gw, r.router.endpoint()); + } + let mut prev = GW1; + for (i, gw) in gws.iter().enumerate() { + r.caps.set_present(prev, false); + r.lookup.set(gateway(*gw)); + assert_eq!(r.client.check_network().await, NetCheck::Changed, "{gw}"); + r.advance(secs(3)); + if i < 4 { + r.router.commit_then_drop_next(1); + assert!(r.map().await.is_none(), "{gw}: reply lost"); + assert_eq!(r.client.owned_rules(), i + 1, "{gw}: tombstone kept"); + } + prev = *gw; + } + // The fifth network: the ledger is full until the tombstones' possible grants end. + let before = r.maps(); + assert!(r.map().await.is_none()); + assert_eq!(r.maps(), before, "ledger full: no MAP sent"); + assert_eq!(r.client.owned_rules(), 4); + r.advance(secs(7200)); + let mut mapped = false; + for _ in 0..5 { + r.client.tick().await; + r.settle().await; + if r.client.have_mapping() { + mapped = true; + break; + } + r.advance(secs(3)); + } + assert!(mapped, "a fifth network maps once the old grants have run out"); + r.shutdown().await; +} + #[tokio::test] async fn send_evidence_survives_cancellation() { for pcp in [false, true] { @@ -4828,7 +4916,9 @@ async fn send_evidence_survives_cancellation() { if sent { let entry = r.client.owned_snapshot()[0].clone(); assert_eq!(entry.rule, rule); - assert!(entry.router_expiry() >= r.clock.now() + secs(u64::from(u32::MAX) - 1)); + // Bounded by the requested lifetime from the send attempt (RFC 6886 §3.3). + assert!(entry.router_expiry() >= r.clock.now() + secs(7200 - 1)); + assert!(entry.router_expiry() < r.clock.now() + secs(7201)); assert!(!entry.mapping() && !entry.deleting()); } assert_eq!(delete_requests(&r.router), 0); diff --git a/ts_portmapper/src/lifecycle.rs b/ts_portmapper/src/lifecycle.rs index 975b7224..c21723a1 100644 --- a/ts_portmapper/src/lifecycle.rs +++ b/ts_portmapper/src/lifecycle.rs @@ -865,11 +865,14 @@ impl Lifecycle { let before = self.data.facets; if want.pmp { if r.pmp { - self.data.probe.pmp_seen_at = Some(now); - self.data.probe.pmp_public_ip = r.pmp_public_ip; + // The reset check runs before the fresh answer is recorded, since a reset clears + // what it would record (Go: "must be before we write to c.pmp*", + // net/portmapper/portmapper.go:1138-1141). if let Some(e) = r.pmp_epoch { rolled_back |= self.router_reset(Proto::Pmp, e, started, gateway); } + self.data.probe.pmp_seen_at = Some(now); + self.data.probe.pmp_public_ip = r.pmp_public_ip; } self.data.facets.pmp = Some(r.pmp); } else if protos.pmp { @@ -877,10 +880,11 @@ impl Lifecycle { } if want.pcp { if r.pcp { - self.data.probe.pcp_seen_at = Some(now); + // Before the write, as Go does (net/portmapper/portmapper.go:1098-1101). if let Some(e) = r.pcp_epoch { rolled_back |= self.router_reset(Proto::Pcp, e, started, gateway); } + self.data.probe.pcp_seen_at = Some(now); } self.data.facets.pcp = Some(r.pcp); if let Some(code @ (pcp::ResultCode::NotAuthorized | pcp::ResultCode::AddressMismatch)) = diff --git a/ts_portmapper/src/oneshot.rs b/ts_portmapper/src/oneshot.rs index 68cd2369..9262bcc7 100644 --- a/ts_portmapper/src/oneshot.rs +++ b/ts_portmapper/src/oneshot.rs @@ -402,7 +402,9 @@ pub async fn probe(t: &T, protos: ProtoSet) -> Result note(&mut notes, format!("pcp: {m}")), diff --git a/ts_portmapper/src/owned.rs b/ts_portmapper/src/owned.rs index 1be2be4e..59bc86d4 100644 --- a/ts_portmapper/src/owned.rs +++ b/ts_portmapper/src/owned.rs @@ -20,8 +20,9 @@ pub(crate) const MAX_OWNED: usize = 4; /// Initial unacknowledged deletes before switching to infrequent recovery attempts. pub(crate) const MAX_DELETE_TRIES: u32 = 3; /// After the initial attempts, at most one delete per rule per ten minutes, with fresh gateway -/// validation each time. Unknown grants never age out at the requested lifetime: they keep an -/// admission slot until per-rule evidence (a matched reply/delete or accepted reset) resolves it. +/// validation each time. An unanswered MAP keeps its admission slot for the requested lifetime +/// from its last send attempt (RFC 6886 §3.3), unless per-rule evidence (a matched reply/delete +/// or accepted reset) resolves it first. pub(crate) const DELETE_RETRY_INTERVAL: std::time::Duration = std::time::Duration::from_secs(600); /// Where an owned rule stands. diff --git a/ts_portmapper/src/pcp.rs b/ts_portmapper/src/pcp.rs index d4df4f88..ddf6cf76 100644 --- a/ts_portmapper/src/pcp.rs +++ b/ts_portmapper/src/pcp.rs @@ -823,8 +823,9 @@ mod tests { assert_eq!(format!("{a:?}"), "Nonce(..)"); } - /// The seeds of Go's `FuzzParsePCPResponse` and `FuzzParsePCPMapResponse` - /// (`net/portmapper/fuzz_test.go`), with stronger expectations: a MAP parser rejects the + /// The seeds of upstream Go's `FuzzParsePCPResponse` and `FuzzParsePCPMapResponse` + /// (`net/portmapper/fuzz_test.go` on tailscale main, added after v1.102.4), with stronger + /// expectations: a MAP parser rejects the /// ANNOUNCE-opcode seed and the IPv6 external address, and the result is always exactly one of /// a mapping or an error (guaranteed by `Result`). #[test] diff --git a/ts_portmapper/src/schedule.rs b/ts_portmapper/src/schedule.rs index 939803c2..19cdfa1e 100644 --- a/ts_portmapper/src/schedule.rs +++ b/ts_portmapper/src/schedule.rs @@ -5,7 +5,9 @@ //! //! - [`Schedule::next_attempt_at`]: the earliest time **any** create or renew may start, whatever //! the trigger (driver timer, kick, rebind, ReSTUN, `get_cached…`). Failures push it out by a -//! jittered backoff; router refusals push it further. +//! jittered backoff; a PCP error lifetime may push it further. Go has no backoff (a failed create +//! is retried on the next ReSTUN, `net/portmapper/portmapper.go:477-514`); this spacing is a +//! stated deviation in `docs/PORTMAPPER.md`. //! - [`Lease::renew_after`]: when a healthy mapping should be renewed. //! - [`Lease::expires_at`]: the end of local trust. After it the mapping is dropped and its endpoint //! withdrawn, whether or not anything else happens. @@ -38,8 +40,6 @@ pub const BACKOFF_STEPS: [Duration; 6] = [ ]; /// The least time between any two attempt starts, even across a network change. pub const ATTEMPT_FLOOR: Duration = Duration::from_secs(2); -/// How long a "not authorized" refusal holds off the next attempt. -pub const NOT_AUTHORIZED_HOLD: Duration = Duration::from_secs(30 * 60); /// The longest hold taken from a PCP error lifetime (RFC 6887 §7.4, §11.4). pub const MAX_ERROR_HOLD: Duration = Duration::from_secs(30 * 60); @@ -110,7 +110,9 @@ pub(crate) fn earlier(t: Instant, by: Duration, now: Instant) -> Instant { pub enum Failure { /// Timeout, no responder, no services, an OS error or a malformed exchange. Transient, - /// NAT-PMP result 2 or PCP `NOT_AUTHORIZED`: hold off for at least [`NOT_AUTHORIZED_HOLD`]. + /// NAT-PMP result 2 or PCP `NOT_AUTHORIZED`: the ordinary backoff, no longer hold, so a router + /// on which port mapping is switched on is mapped within the normal ladder (Go retries on the + /// next ReSTUN with no hold, `net/portmapper/portmapper.go:477-494, 505-514`). NotAuthorized, /// Another router refusal; `hold` is a PCP error lifetime, if any. Refused { @@ -173,10 +175,8 @@ impl Schedule { self.failures = self.failures.saturating_add(1); let mut hold = jittered(backoff(self.failures), jitter); match why { - Failure::Transient => {} - Failure::NotAuthorized => hold = hold.max(NOT_AUTHORIZED_HOLD), + Failure::Transient | Failure::NotAuthorized | Failure::Refused { hold: None } => {} Failure::Refused { hold: Some(h) } => hold = hold.max(h.min(MAX_ERROR_HOLD)), - Failure::Refused { hold: None } => {} } self.next_attempt_at = Some(now.checked_add(hold).unwrap_or(now)); } @@ -293,8 +293,9 @@ mod tests { assert_eq!(s.next_attempt_at, Some(t0 + secs(10))); s.note_failure(t0, Failure::Transient, 1.0); assert_eq!(s.next_attempt_at, Some(t0 + secs(30))); + // NOT_AUTHORIZED takes the ordinary ladder: no long hold. s.note_failure(t0, Failure::NotAuthorized, 1.0); - assert_eq!(s.next_attempt_at, Some(t0 + NOT_AUTHORIZED_HOLD)); + assert_eq!(s.next_attempt_at, Some(t0 + secs(60))); let mut s = Schedule::default(); // A PCP error lifetime above the backoff holds, capped at 30 min. From aa10ef695e9cc90185cb163c00b9acaf8fc183e6 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 18 Oct 2026 18:13:04 +0200 Subject: [PATCH 10/12] fix(runtime): state the port mapper's differences from Go honestly Print active-___ for a held mapping with nothing checked, as Go's NetInfo.portMapSummary does, sort the STUN endpoint group so the advertiser's ordered comparison is stable (Go compares sets), and stop attributing the only-tcp-443 behaviour and the CallMeMaybe gate to Go. List only-tcp-443, TS_DISABLE_PORTMAPPER, same-port rebind, the missing endpoint tracker, the network poll and resume detection as deviations. --- ts_control/src/net_facets.rs | 20 ++++++++++++-------- ts_magicsock/src/sock.rs | 6 +++++- ts_runtime/src/direct.rs | 12 ++++++++---- 3 files changed, 25 insertions(+), 13 deletions(-) diff --git a/ts_control/src/net_facets.rs b/ts_control/src/net_facets.rs index 659c606c..5173e847 100644 --- a/ts_control/src/net_facets.rs +++ b/ts_control/src/net_facets.rs @@ -44,18 +44,19 @@ pub struct PortMapFacets { } impl PortMapFacets { - /// Go's netcheck `portmap=` summary: `?` when nothing was checked, otherwise one letter per - /// protocol (`U`, `M`, `C`) that is present, nothing for one that is absent, and `_` for one - /// not checked (`conciseOptBool`). An advertised mapping prefixes `active-`. + /// Go's NetInfo `portmap=` summary (`tailcfg.NetInfo.portMapSummary`): `?` when there is no + /// mapping and nothing was checked, otherwise one letter per protocol (`U`, `M`, `C`) that is + /// present, nothing for one that is absent, and `_` for one not checked (`conciseOptBool`). A + /// mapping prefixes `active-`; here that means an advertised mapping, where Go's means any + /// held mapping. pub fn summary(&self) -> String { + if !self.have_port_map && self.upnp.is_none() && self.pmp.is_none() && self.pcp.is_none() { + return "?".into(); + } let mut s = String::new(); if self.have_port_map { s.push_str("active-"); } - if self.upnp.is_none() && self.pmp.is_none() && self.pcp.is_none() { - s.push('?'); - return s; - } for (v, c) in [(self.upnp, 'U'), (self.pmp, 'M'), (self.pcp, 'C')] { match v { None => s.push('_'), @@ -90,7 +91,8 @@ mod tests { assert!(v6.working_udp && !v6.mapping_varies); } - /// Port of netcheck `TestReportString`'s portmap cases, plus the `active-` prefix. + /// Port of netcheck `TestReportString`'s portmap cases, plus the `active-` prefix of Go's + /// `NetInfo.portMapSummary`. #[test] fn summary_matches_go() { let f = |upnp, pmp, pcp, have| PortMapFacets { @@ -113,5 +115,7 @@ mod tests { "active-_MC" ); assert_eq!(f(None, Some(false), Some(false), false).summary(), "_"); + // A mapping with nothing checked: Go prints `active-___`, not `active-?`. + assert_eq!(f(None, None, None, true).summary(), "active-___"); } } diff --git a/ts_magicsock/src/sock.rs b/ts_magicsock/src/sock.rs index 7119e164..5dc18d4a 100644 --- a/ts_magicsock/src/sock.rs +++ b/ts_magicsock/src/sock.rs @@ -996,7 +996,11 @@ impl MagicSock { // guess loop below: the critical section is the cheap copy, not the candidate assembly. let (all_reflexive, v4_reflexive): (Vec, Vec) = { let reflexive = lock(&self.reflexive); - let all: Vec = reflexive.iter().copied().collect(); + let mut all: Vec = reflexive.iter().copied().collect(); + // A deterministic order for the STUN group, so the advertiser's ordered comparison + // does not see a change when a re-filled set iterates differently (Go compares + // endpoint sets order-insensitively, `endpointSetsEqual`). + all.sort(); let v4: Vec = all.iter().filter(|a| a.is_ipv4()).copied().collect(); (all, v4) }; diff --git a/ts_runtime/src/direct.rs b/ts_runtime/src/direct.rs index d4c44d36..b26b1921 100644 --- a/ts_runtime/src/direct.rs +++ b/ts_runtime/src/direct.rs @@ -1036,8 +1036,9 @@ fn stun_probe_window( /// change to avoid spamming control with redundant side-band map requests. /// /// A change is a change in the ordered endpoint list, in the STUN facts or in the port-mapping -/// facets (Go's `BasicallyEqual` for these fields). The list is published in priority order, not -/// sorted. A port-mapper state change (`port_map_rx`) wakes the loop at once instead of waiting for +/// facets. Go compares endpoints with the order-insensitive `endpointSetsEqual` and the NetInfo +/// facts with `BasicallyEqual`; the STUN group here is sorted so the ordered comparison is stable. +/// The list is published in priority order. A port-mapper state change (`port_map_rx`) wakes the loop at once instead of waiting for /// the 5 s tick, which stays as the fallback for reflexive addresses learned from pongs. /// /// Reflexive (STUN-equivalent) endpoints come solely from the disco pong-harvest path on the one @@ -1080,7 +1081,8 @@ async fn run_advertiser( } /// Whether `eps` holds a candidate a remote peer could reach across the internet: a STUN -/// reflexive address or a port mapping (Go counts `EndpointPortmapped` as one too). +/// reflexive address or a port mapping. (This gate is a Rust addition; Go's `enqueueCallMeMaybe` +/// sends whatever endpoints it last determined.) fn has_reflexive_candidate(eps: &[SelfEndpoint]) -> bool { eps.iter().any(|e| { matches!( @@ -1478,7 +1480,9 @@ impl Message> for DirectManager { self.sock.as_deref(), ); // `only-tcp-443` stops the port mapper's LAN sends too, synchronously and deletes - // included (Go passes `onlyTCP443` to the portmapper as its disable-all knob). + // included. Go passes `onlyTCP443` as the portmapper's disable-all knob, which blocks + // only new creates and probes: Go keeps advertising the cached mapping and still releases + // it on Close. #[cfg(feature = "portmapper")] if let (Some(node), Some(pm)) = (msg.node.as_ref(), &self.portmap) { pm.set_only_tcp_443(node.only_tcp_443()); From 9c04e494b8aede01243cc55d68035132c67a626d Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 18 Oct 2026 18:16:44 +0200 Subject: [PATCH 11/12] test(portmapper): pin the bounded tombstone lifetime Two tests pinned the old rule that an unanswered MAP holds its slot for ever. Now its slot frees after the requested lifetime from the last send; network loss alone still retires nothing. --- docs/PORTMAPPER.md | 6 +++-- ts_portmapper/src/client_tests.rs | 39 +++++++++++++++++++++---------- 2 files changed, 31 insertions(+), 14 deletions(-) diff --git a/docs/PORTMAPPER.md b/docs/PORTMAPPER.md index 4c6e0245..0fe6cb31 100644 --- a/docs/PORTMAPPER.md +++ b/docs/PORTMAPPER.md @@ -79,8 +79,10 @@ but never sends on it.) Nothing in the node runs it yet. NAT-PMP's public-address request alone cannot create a mapping. A matched success records the actual grant. An unanswered MAP keeps its slot for the requested lifetime (7200 s) from the latest syscall attempt: a NAT-PMP gateway should not grant longer than requested (RFC 6886 - §3.3), and Go trusts no unanswered MAP at all. Four unresolved rules can therefore stop further - creates for up to two hours, never for the rest of the process. Wall-clock jumps withdraw + §3.3), and Go trusts no unanswered MAP at all. A PCP server may grant longer; such an unobserved + longer grant is forgotten after 7200 s and left to the router's own expiry, as after a crash. + Four unresolved rules can therefore stop further creates for up to two hours, never for the + rest of the process. Wall-clock jumps withdraw advertisement trust but do not shorten possible ownership. - **Network down.** Nothing is deleted, since a delete could reach the wrong router. Such rules are left to expire. A create for the same rule on the same network adopts them, because PCP refuses a diff --git a/ts_portmapper/src/client_tests.rs b/ts_portmapper/src/client_tests.rs index 2bf2fb91..44b42549 100644 --- a/ts_portmapper/src/client_tests.rs +++ b/ts_portmapper/src/client_tests.rs @@ -917,7 +917,10 @@ async fn router_errors_back_off() { r.settle().await; let wait = r.client.schedule().next_attempt.unwrap() - r.clock.now(); // The second consecutive failure: 30 s ±20 %, no 30 min hold. - assert!(wait >= secs(23) && wait <= secs(37), "NOT_AUTHORIZED: {wait:?}"); + assert!( + wait >= secs(23) && wait <= secs(37), + "NOT_AUTHORIZED: {wait:?}" + ); r.shutdown().await; // ANNOUNCE refused: PCP present but unusable. @@ -1188,7 +1191,11 @@ async fn remap_right_after_the_rollback_probe() { assert!(first.is_some(), "pmp={pmp}"); let held_pcp = r.router.counters().num_pcp_map_recv; let held_pmp = r.router.counters().num_pmp_map_recv; - assert_eq!((held_pmp > 0, held_pcp > 0), (pmp, !pmp), "mapped over the expected protocol"); + assert_eq!( + (held_pmp > 0, held_pcp > 0), + (pmp, !pmp), + "mapped over the expected protocol" + ); r.router.reboot(); r.advance(secs(601)); r.client.probe().await.unwrap(); @@ -2159,9 +2166,11 @@ async fn completed_stale_attempt_cannot_publish_after_invalidation() { } } -/// An unobserved grant may exceed the request: tombstones must not expire at the request alone. +/// An unobserved grant may exceed the request (PCP may grant more), but a tombstone keeps its +/// slot only for the requested lifetime from its send: after that the client maps again, and the +/// router's longer rule is left to its own expiry, as after a crash. #[tokio::test] -async fn lost_long_grants_still_count_after_requested_lifetime() { +async fn lost_long_grants_free_their_slots_after_requested_lifetime() { let r = rig_with(Opts { router: RouterOptions { pmp: false, @@ -2190,10 +2199,10 @@ async fn lost_long_grants_still_count_after_requested_lifetime() { r.settle().await; assert_eq!( r.router.mappings().len(), - 4, - "unknown long grants still occupy their slots" + 5, + "the fifth create went ahead; the long grants stay on the router until they expire" ); - assert_eq!(r.client.owned_rules(), 4); + assert!(r.client.owned_rules() <= 1, "the old tombstones retired"); r.shutdown().await; } @@ -3176,16 +3185,19 @@ async fn network_down_fences_a_late_recovery_validation() { assert_eq!(cleanup.await.unwrap(), 0); drop(frozen); entered.expect("exhausted entry must reach fresh validation before retry"); - for _ in 0..5 { - r.advance(secs(7200)); - client.cleanup().await; - } + client.cleanup().await; assert!(client.owned_snapshot().iter().all(|e| !e.releasable())); assert_eq!( client.owned_rules(), 1, "network loss is not retirement evidence" ); + // Only the end of the possible grant (the requested lifetime from the last send) retires it. + for _ in 0..5 { + r.advance(secs(7200)); + client.cleanup().await; + } + assert_eq!(client.owned_rules(), 0, "the possible grant ran out"); assert!(!client.have_mapping()); client.close(secs(1)).await; assert_eq!( @@ -4866,7 +4878,10 @@ async fn left_networks_with_lost_replies_free_their_slots() { } r.advance(secs(3)); } - assert!(mapped, "a fifth network maps once the old grants have run out"); + assert!( + mapped, + "a fifth network maps once the old grants have run out" + ); r.shutdown().await; } From 2f8a0c8c303e0bfa42dbcf35c59932b990ef4f82 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 18 Oct 2026 18:18:10 +0200 Subject: [PATCH 12/12] fix(exit_node): run the probe's tests in CI and refuse it under TS_DISABLE_PORTMAPPER The exit_node tests never ran: examples default to test = false. The probe now refuses to run when TS_DISABLE_PORTMAPPER is set, as Go's debug portmap does, --portmap-hold says it must be shorter than the lifetime, and the runner's non-public suppression and the portmap= summary are explained and attributed correctly. --- CHANGELOG.md | 2 +- Cargo.toml | 5 ++++ docs/PORTMAPPER.md | 8 +++--- examples/exit_node/main.rs | 52 +++++++++++++++++++++++++++++++++++--- 4 files changed, 59 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 901948f0..a1d8c692 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ Record breaking or significant changes here. All dates are UTC. * **portmapper:** add `ts_portmapper`, a NAT-PMP and PCP client modelled on Go's `net/portmapper`, which only ever sends to the LAN gateway's port 5351. * **runtime:** map the underlay UDP port on the LAN gateway with NAT-PMP or PCP and advertise the mapping to peers, as Go does; `Config::disable_port_mapper` or `TS_DISABLE_PORTMAPPER` turns it off. -* **exit node:** add `--portmap-probe`, a one-shot NAT-PMP/PCP check, and flags for the port mapper. +* **exit node:** add `--portmap-probe`, a one-shot NAT-PMP/PCP check, and flags for the port mapper, with two runner defaults that differ from Go (`--portmap-advertise-nonpublic` and `--portmap-lazy-renew` restore Go's). ### Performance diff --git a/Cargo.toml b/Cargo.toml index 1816fcdf..942b691d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -416,3 +416,8 @@ required-features = ["tsnet"] [[example]] name = "ssh_peer_lookup" required-features = ["ssh"] + +# Run the argument-parsing tests in `cargo test` (examples default to `test = false`). +[[example]] +name = "exit_node" +test = true diff --git a/docs/PORTMAPPER.md b/docs/PORTMAPPER.md index 7d3e6d72..467e8a4b 100644 --- a/docs/PORTMAPPER.md +++ b/docs/PORTMAPPER.md @@ -9,7 +9,8 @@ behind a hard NAT can reach it directly instead of over DERP. NAT-PMP (RFC 6886) codecs, the gateway lookup and `GatewayCap`, the only code that sends packets. (The gateway lookup's kernel source-address check connects a UDP socket but never sends on it.) - Driver: [`ts_runtime::portmap`](../ts_runtime/src/portmap.rs), spawned by the direct manager. -- One-shot field check: `examples/exit_node --portmap-probe`. +- One-shot field check: `examples/exit_node --portmap-probe`. Like Go's `tailscale debug + portmap`, it is refused when `TS_DISABLE_PORTMAPPER` is set. ## Turning it off @@ -27,8 +28,9 @@ sets `port_mapper: None`, and the `portmapper_test_isolation` check enforces the - **Advertising (`advertise_nonpublic_external`).** The library default is Go's: advertise every external address. The exit-node runner turns it off. A private or CGNAT external address (a - double NAT) is then kept and renewed, but not advertised. `--portmap-advertise-nonpublic` reverts - the runner to Go's behaviour. + double NAT) is then kept and renewed, but not advertised: it is not reachable from outside that + NAT, so advertising it only costs peers failed disco probes. `--portmap-advertise-nonpublic` + reverts the runner to Go's behaviour. - **Renewal while idle (`renew_while_idle`).** The library default is Go's: renewal rides on activity (the periodic ReSTUN and other kicks). The runner renews a due mapping even while idle, as long as it has peers. With no peers a mapping is never renewed and expires. diff --git a/examples/exit_node/main.rs b/examples/exit_node/main.rs index 78b13b56..3f8e447e 100644 --- a/examples/exit_node/main.rs +++ b/examples/exit_node/main.rs @@ -106,7 +106,8 @@ struct Args { status_interval: u64, /// Run the one-shot NAT-PMP/PCP probe on a dedicated test port and exit. Needs no state - /// directory, keys or control connection. + /// directory, keys or control connection. Refused when TS_DISABLE_PORTMAPPER is set, as Go's + /// debug portmap is. #[arg(long)] portmap_probe: bool, @@ -127,7 +128,8 @@ struct Args { )] portmap_lifetime: u32, - /// Probe: seconds to hold each mapping open, counting inbound datagrams (no payloads). + /// Probe: seconds to hold each mapping open, counting inbound datagrams (no payloads). Must be + /// shorter than --portmap-lifetime (default 120). #[arg( long, default_value_t = 0, @@ -157,7 +159,8 @@ struct Args { /// Advertise a port-mapped address even when it is private or CGNAT (a double NAT), as Go /// does. By default this runner keeps and renews such a mapping but does not offer it to - /// peers. + /// peers. A private or CGNAT external address is not reachable from outside that NAT, so + /// advertising it only costs peers failed disco probes. #[arg(long, conflicts_with = "portmap_probe")] portmap_advertise_nonpublic: bool, @@ -255,9 +258,24 @@ fn run_portmap_probe(args: &Args) -> Result> { }) } +/// Whether `TS_DISABLE_PORTMAPPER` turns the port mapper off, parsed with the same table as +/// `ts_runtime`'s env knob (Go's `strconv.ParseBool`; an unparsable value counts as disabled). +fn portmap_disabled_by_env(getenv: impl Fn(&str) -> Option) -> bool { + match getenv("TS_DISABLE_PORTMAPPER").as_deref() { + None | Some("0" | "f" | "F" | "FALSE" | "false" | "False" | "") => false, + Some(_) => true, + } +} + fn main() -> Result<(), Box> { let args = Args::parse(); if args.portmap_probe { + // Go's `tailscale debug portmap` fails with ErrPortMappingDisabled under this knob + // (net/portmapper/portmapper.go:85-88, 876-878): refuse before any gateway lookup. + if portmap_disabled_by_env(|k| std::env::var(k).ok()) { + eprintln!("error msg=portmap-disabled env=TS_DISABLE_PORTMAPPER"); + std::process::exit(1); + } let code = run_portmap_probe(&args)?; std::process::exit(code); } @@ -417,7 +435,9 @@ async fn log_status(dev: &Device) { return; } }; - // Go's netcheck `portmap=` summary (e.g. `active-_MC`), plus the mapping when one is held. + // Go's NetInfo `portmap=` summary (`tailcfg.NetInfo.portMapSummary`, e.g. `active-_MC`); + // `active-` here means an advertised mapping, where Go's means any held mapping. Plus the + // mapping when one is held. let (portmap, portmap_ext) = match dev.port_map_status() { Some(pm) => ( pm.summary(), @@ -531,6 +551,23 @@ mod tests { assert!(parse(&["--state-dir", "/tmp/x"]).is_ok()); } + /// `TS_DISABLE_PORTMAPPER` refuses the probe, parsed like the runtime's knob. + #[test] + fn probe_refused_under_ts_disable_portmapper() { + let env = |v: Option<&'static str>| { + move |k: &str| { + assert_eq!(k, "TS_DISABLE_PORTMAPPER"); + v.map(str::to_owned) + } + }; + for v in ["1", "t", "true", "TRUE", "yes"] { + assert!(portmap_disabled_by_env(env(Some(v))), "{v}"); + } + for v in [None, Some("0"), Some("false"), Some("")] { + assert!(!portmap_disabled_by_env(env(v)), "{v:?}"); + } + } + #[test] fn probe_flags_require_the_probe() { for flags in [ @@ -582,6 +619,13 @@ mod tests { Some("192.168.1.20".parse().unwrap()) ); assert!(parse(&["--portmap-probe", "--portmap-gateway", "nope"]).is_err()); + // The hold must be shorter than the lifetime (default 120 s). + let a = parse(&["--portmap-probe", "--portmap-hold", "120"]).unwrap(); + let err = a.probe_options().validate().unwrap_err(); + assert!( + err.contains("hold must be shorter than the lifetime"), + "{err}" + ); } /// The runner's port-mapper defaults (suppress non-public, renew while idle) and the flags