From e19b5af6be649c07b86d404a2c10a80256cb8c09 Mon Sep 17 00:00:00 2001 From: schickling-assistant <261620128+schickling-assistant@users.noreply.github.com> Date: Thu, 22 Oct 2026 10:17:45 +0200 Subject: [PATCH 1/2] feat(fleet): follow local transport order for automatic peer routes agent-identity: agent/dotfiles/fabric-mesh agent-persona: generalist agent-supervisor: unavailable agent-tool: OMP agent-tool-version: 18.9.1 agent-runtime: OMP 18.9.1 agent-model: anthropic/claude-opus-5-5 --- crates/fractal-peer/src/peer.rs | 125 ++++++++++++-- crates/fractal/src/fleet_cli.rs | 23 ++- crates/fractal/src/main.rs | 3 + crates/fractal/tests/fleet.rs | 59 +++++++ crates/smallclaims/src/fleet/file.rs | 2 + crates/smallclaims/src/fleet/transport.rs | 157 ++++++++++++++---- crates/smallclaims/src/sync/worker.rs | 21 ++- crates/smallclaims/src/sync/worker_tests.rs | 125 ++++++++++++++ docs/st3/cli-guided-tour.md | 7 + docs/st3/replication.md | 24 +++ docs/two-machines.md | 29 ++++ ...et-automatic-transport-preference-q33.json | 11 ++ 12 files changed, 526 insertions(+), 60 deletions(-) create mode 100644 release-notes/fleet-automatic-transport-preference-q33.json diff --git a/crates/fractal-peer/src/peer.rs b/crates/fractal-peer/src/peer.rs index 9c5f6362956..8e340c27909 100644 --- a/crates/fractal-peer/src/peer.rs +++ b/crates/fractal-peer/src/peer.rs @@ -322,6 +322,8 @@ pub struct ClientRelay { /// Reuse sealed membership across per-item reachability checks for the same short TTL. pub membership: Arc>>, pub fabric: Option, + /// Ordered local transport policy; absent for a legacy config-only fleet. + transports: Option>, legacy: bool, /// When each owner last refused this node's reads as stale, for provenance. fence_conflicts: @@ -433,6 +435,25 @@ impl ClientRelay { /// peer, then the peers with the shortest observed path to it. With no observation of the /// target at all, every peer is worth a try. Nodes the read already passed are never chosen. pub fn next_hops(&self, target: &str, visited: &[String]) -> Vec { + self.next_hops_with_transports( + target, + visited, + LocalTransports { + fabric: self.fabric.is_some(), + tailscale: local_addresses() + .iter() + .any(crate::fleet::transport::is_tailnet_address), + ..LocalTransports::default() + }, + ) + } + + fn next_hops_with_transports( + &self, + target: &str, + visited: &[String], + available: LocalTransports, + ) -> Vec { let view = self.fleet_view(); // Sync from a dial-out member supplies no reverse owner-RPC or PTY transport. // In particular, dropping its old links must not enable the unseen-owner fallback. @@ -441,12 +462,11 @@ impl ClientRelay { }) { return Vec::new(); } - let local = LocalTransports { - fabric: self.fabric.is_some(), - tailscale: local_addresses() - .iter() - .any(crate::fleet::transport::is_tailnet_address), - }; + let local = LocalTransports::configured( + self.transports.as_deref(), + available.tailscale, + available.fabric, + ); let dialable = dial_targets(&view, &self.node, &self.peers, local); let names = dialable .keys() @@ -512,14 +532,21 @@ impl ClientRelay { observed: Arc::default(), membership: Arc::default(), fence_conflicts: Arc::default(), - fabric: resolve_tool( - config - .fleet - .as_ref() - .and_then(|file| file.fabric.as_deref()), - "fabric", - ) - .map(Fabric::new), + transports: config.fleet.as_ref().map(|file| file.transports.clone()), + fabric: if config + .fleet + .as_ref() + .is_none_or(|file| file.transports.iter().any(|name| name == "fabric")) + || config.peers.iter().any(|peer| peer.url.starts_with("fabric://")) + { + resolve_tool( + config.fleet.as_ref().and_then(|file| file.fabric.as_deref()), + "fabric", + ) + .map(Fabric::new) + } else { + None + }, })) } @@ -2034,6 +2061,76 @@ mod tests { + #[test] + fn relay_next_hops_follow_local_transport_order_and_removal_with_legacy_peers() { + use serde_json::json; + + let view: FleetView = serde_json::from_value(json!({ + "members": [{ + "name": "owner", + "member_key": "owner-key", + "state": "current", + "mode": "listening", + "start": 1, + // Old peers advertise endpoints without an ordering or preference field. + "endpoints": [ + {"transport": "loopback", "address": "127.0.0.1:4000"}, + {"transport": "tailscale", "address": "100.101.102.103:4000"}, + {"transport": "fabric", "node": "owner-node", "protocol": "st3-peer-v1"} + ] + }] + })).unwrap(); + let mut relay = ClientRelay { + node: "gateway".into(), + peers: Vec::new(), + auth: FleetAuth::test("fleet-test", &[7; 32]), + http: reqwest::Client::new(), + links: Some(Arc::new(Store::open_memory("gateway").unwrap())), + observed: Arc::default(), + membership: Arc::new(std::sync::Mutex::new(Some(( + std::time::Instant::now(), + Arc::new(view), + )))), + fabric: None, + transports: None, + legacy: false, + fence_conflicts: Arc::default(), + }; + let available = LocalTransports { + tailscale: true, + fabric: true, + ..LocalTransports::default() + }; + let tailscale = "http://100.101.102.103:4000"; + let fabric = "fabric://owner-node/st3-peer-v1"; + let loopback = "http://127.0.0.1:4000"; + for (policy, expected) in [ + (None, vec![tailscale, fabric, loopback]), + (Some(vec!["tailscale", "fabric"]), vec![tailscale, fabric, loopback]), + (Some(vec!["fabric", "tailscale"]), vec![fabric, tailscale, loopback]), + (Some(vec!["fabric"]), vec![fabric, loopback]), + (Some(vec!["tailscale"]), vec![tailscale, loopback]), + (Some(Vec::new()), vec![loopback]), + ] { + relay.transports = policy.map(|names| names.into_iter().map(str::to_owned).collect()); + assert_eq!( + relay.next_hops_with_transports("owner", &[], available) + .into_iter() + .map(|peer| peer.url) + .collect::>(), + expected, + "{:?}", + relay.transports, + ); + } + // Explicit config routes retain their precedence for mixed config/member fleets. + relay.peers.push(PeerConfig { + name: "owner".into(), + url: fabric.into(), + }); + assert_eq!(relay.next_hops_with_transports("owner", &[], available)[0].url, fabric); + } + #[test] fn next_hops_prefer_the_owner_then_the_nearest_peer_that_reaches_it() { let links = |pairs: &[(&str, &str)]| { diff --git a/crates/fractal/src/fleet_cli.rs b/crates/fractal/src/fleet_cli.rs index 5cd27e02776..17df664a60b 100644 --- a/crates/fractal/src/fleet_cli.rs +++ b/crates/fractal/src/fleet_cli.rs @@ -685,8 +685,20 @@ async fn run_fleet_transports( .transports .iter() .any(|configured| configured == transport); - // A repeated add never reconfigures an existing exposure, even with a different flag. - if matches!(&command, FleetTransportsCommand::Add { .. }) && enabled { + // Re-adding can change local preference, but never reconfigures the exposure. + if let FleetTransportsCommand::Add { + prefer, no_service, .. + } = &command + && enabled + { + if *prefer && file.transports.first() != Some(transport) { + file.transports.retain(|configured| configured != transport); + file.transports.insert(0, transport.clone()); + file.save(&config.state_dir)?; + drop(lock); + println!("Fabric is now preferred; its existing protocol is unchanged."); + return refresh_fleet_worker(*no_service); + } println!("Fabric is already enabled; remove it before changing its protocol."); return Ok(()); } @@ -707,6 +719,7 @@ async fn run_fleet_transports( let no_service = match command { FleetTransportsCommand::Add { fabric_protocol, + prefer, no_service, .. } => { @@ -738,7 +751,11 @@ async fn run_fleet_transports( .expose(&protocol, &format!("127.0.0.1:{port}")) .await .context("Fabric exposure failed; fleet settings and endpoints are unchanged")?; - file.transports.push("fabric".into()); + if prefer { + file.transports.insert(0, "fabric".into()); + } else { + file.transports.push("fabric".into()); + } file.fabric_protocol = Some(protocol.clone()); endpoints.retain(|endpoint| endpoint["transport"] != "fabric"); endpoints.push(json!({"transport":"fabric", "node":node, "protocol":protocol})); diff --git a/crates/fractal/src/main.rs b/crates/fractal/src/main.rs index 880d517d01d..a7521a95196 100644 --- a/crates/fractal/src/main.rs +++ b/crates/fractal/src/main.rs @@ -930,6 +930,9 @@ enum FleetTransportsCommand { /// Use this protocol instead of the saved protocol or st3/fleet/FLEET_ID. #[arg(long)] fabric_protocol: Option, + /// Prefer Fabric over other automatic routes, including when already enabled. + #[arg(long)] + prefer: bool, /// Leave the replication worker restart to the operator. #[arg(long)] no_service: bool, diff --git a/crates/fractal/tests/fleet.rs b/crates/fractal/tests/fleet.rs index 847f20b0db8..da1282d22a9 100644 --- a/crates/fractal/tests/fleet.rs +++ b/crates/fractal/tests/fleet.rs @@ -3640,6 +3640,65 @@ async fn an_existing_member_adds_and_removes_fabric_without_rejoining() { } } +#[tokio::test(flavor = "multi_thread")] +async fn fabric_transport_preference_preserves_other_transports_and_existing_exposure() { + if fractal_daemon::test_support::supervise_test() { + return; + } + let root = tempfile::tempdir().unwrap(); + for prefer_on_add in [false, true] { + let name = if prefer_on_add { "preferred" } else { "appended" }; + let mut node = anchor(root.path(), name).await; + node.stop_worker(); + let mut file = configured_file(&node); + file.fabric = Some(fabric_shim(root.path(), name)); + file.transports = vec!["loopback".into(), "tailscale".into()]; + file.save(&node.state_dir()).unwrap(); + node.start_worker(); + node.wait_listening().await; + let mut args = vec!["fleet", "transports", "add", "fabric", "--no-service"]; + if prefer_on_add { + args.push("--prefer"); + } + node.st_ok(&args); + assert_eq!( + configured_file(&node).transports, + if prefer_on_add { + vec!["fabric", "loopback", "tailscale"] + } else { + vec!["loopback", "tailscale", "fabric"] + } + ); + // Re-adding without preference preserves the entire configured order. + let before = configured_file(&node); + node.st_ok(&["fleet", "transports", "add", "fabric", "--no-service"]); + assert_eq!(configured_file(&node).transports, before.transports); + let endpoints = node.st_json(&["fleet", "status"])["view"].clone(); + // Preference is local: it must not expose again or accept a new protocol. + fs::write( + root.path().join("fabric-registry").join(format!("fail-expose-{name}-fabric-id")), + "refuse", + ) + .unwrap(); + node.st_ok(&[ + "fleet", + "transports", + "add", + "fabric", + "--prefer", + "--fabric-protocol", + "ignored", + "--no-service", + ]); + let preferred = configured_file(&node); + assert_eq!(preferred.transports, ["fabric", "loopback", "tailscale"]); + assert_eq!(preferred.fabric_protocol, before.fabric_protocol); + assert_eq!(node.st_json(&["fleet", "status"])["view"], endpoints); + node.st_ok(&["fleet", "transports", "remove", "fabric", "--no-service"]); + assert_eq!(configured_file(&node).transports, ["loopback", "tailscale"]); + } +} + #[tokio::test(flavor = "multi_thread")] async fn fabric_transport_change_refusals_leave_settings_and_endpoints_unchanged() { if fractal_daemon::test_support::supervise_test() { diff --git a/crates/smallclaims/src/fleet/file.rs b/crates/smallclaims/src/fleet/file.rs index 655b3dcef8a..36e955e3299 100644 --- a/crates/smallclaims/src/fleet/file.rs +++ b/crates/smallclaims/src/fleet/file.rs @@ -142,6 +142,8 @@ pub struct FleetFile { pub sponsor_routes: Vec, pub mode: FleetMode, pub port: Option, + /// Enabled automatic transports, in dial preference order; loopback is always last. + /// Explicit config peer (`--peer`) routes retain precedence over this list. pub transports: Vec, pub fabric_protocol: Option, /// Accept HMAC-only exchanges from config peers. True only during a migration. diff --git a/crates/smallclaims/src/fleet/transport.rs b/crates/smallclaims/src/fleet/transport.rs index f0fae7b3214..cc970710ab1 100644 --- a/crates/smallclaims/src/fleet/transport.rs +++ b/crates/smallclaims/src/fleet/transport.rs @@ -593,15 +593,39 @@ pub fn parse_route(route: &str) -> Option { .then(|| Route::Http(route.trim_end_matches('/').into())) } -/// What this machine can use to dial. +/// What this machine can use to dial, filtered and ordered by its local configuration. #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] pub struct LocalTransports { pub tailscale: bool, pub fabric: bool, + pub fabric_first: bool, } -/// A member's routes from its advertised endpoints, in the design's order after any config -/// override: Tailscale, then Fabric, then loopback. Endpoints this machine cannot use, and any +impl LocalTransports { + /// Apply an explicit transport list to current availability. Without a fleet file, + /// retain the legacy Tailscale-first order and allow every available transport. + pub fn configured(transports: Option<&[String]>, tailscale: bool, fabric: bool) -> Self { + let Some(transports) = transports else { + return Self { + tailscale, + fabric, + fabric_first: false, + }; + }; + let tailscale_position = transports.iter().position(|name| name == "tailscale"); + let fabric_position = transports.iter().position(|name| name == "fabric"); + Self { + tailscale: tailscale && tailscale_position.is_some(), + fabric: fabric && fabric_position.is_some(), + fabric_first: fabric_position.is_some_and(|fabric| { + tailscale_position.is_none_or(|tailscale| fabric < tailscale) + }), + } + } +} + +/// A member's routes from its advertised endpoints, in the local configured order after +/// any config override, with loopback last. Endpoints this machine cannot use, and any /// address that is neither loopback nor tailnet, are skipped. pub fn routes_from_endpoints(endpoints: &[Value], local: LocalTransports) -> Vec { let text = |endpoint: &Value, field: &str| endpoint[field].as_str().map(str::to_owned); @@ -611,34 +635,33 @@ pub fn routes_from_endpoints(endpoints: &[Value], local: LocalTransports) -> Vec .filter(|address| is_permitted_route_address(&address.ip())) .map(|address| Route::Http(format!("http://{address}"))) }; + let tailscale = endpoints + .iter() + .filter(|endpoint| local.tailscale && endpoint["transport"] == "tailscale") + .filter_map(address) + .filter(|route| match route { + Route::Http(url) => url + .trim_start_matches("http://") + .parse::() + .is_ok_and(|address| is_tailnet_address(&address.ip())), + Route::Fabric { .. } => false, + }); + let fabric = endpoints + .iter() + .filter(|endpoint| local.fabric && endpoint["transport"] == "fabric") + .filter_map(|endpoint| { + Some(Route::Fabric { + node: text(endpoint, "node")?, + protocol: text(endpoint, "protocol")?, + }) + }); let mut routes = Vec::new(); - if local.tailscale { - routes.extend( - endpoints - .iter() - .filter(|endpoint| endpoint["transport"] == "tailscale") - .filter_map(address) - .filter(|route| match route { - Route::Http(url) => url - .trim_start_matches("http://") - .parse::() - .is_ok_and(|address| is_tailnet_address(&address.ip())), - Route::Fabric { .. } => false, - }), - ); - } - if local.fabric { - routes.extend( - endpoints - .iter() - .filter(|endpoint| endpoint["transport"] == "fabric") - .filter_map(|endpoint| { - Some(Route::Fabric { - node: text(endpoint, "node")?, - protocol: text(endpoint, "protocol")?, - }) - }), - ); + if local.fabric_first { + routes.extend(fabric); + routes.extend(tailscale); + } else { + routes.extend(tailscale); + routes.extend(fabric); } routes.extend( endpoints @@ -729,10 +752,7 @@ mod tests { json!({"transport": "tailscale", "address": "192.168.1.4:31313"}), json!({"transport": "loopback", "address": "192.168.1.4:4000"}), ]; - let all = LocalTransports { - tailscale: true, - fabric: true, - }; + let all = LocalTransports::configured(None, true, true); assert_eq!( routes_from_endpoints(&endpoints, all), vec![ @@ -750,6 +770,75 @@ mod tests { ); } + #[test] + fn configured_transports_filter_availability_and_preserve_local_order() { + for (names, tailscale, fabric, fabric_first) in [ + (None, true, true, false), + (Some(vec!["tailscale", "fabric"]), true, true, false), + (Some(vec!["fabric", "tailscale"]), true, true, true), + (Some(vec!["tailscale"]), true, false, false), + (Some(vec!["fabric"]), false, true, true), + (Some(vec![]), false, false, false), + ] { + let names = names.map(|names| names.into_iter().map(String::from).collect::>()); + for (available_tailscale, available_fabric) in + [(true, true), (false, true), (true, false), (false, false)] + { + assert_eq!( + LocalTransports::configured( + names.as_deref(), + available_tailscale, + available_fabric, + ), + LocalTransports { + tailscale: tailscale && available_tailscale, + fabric: fabric && available_fabric, + fabric_first, + }, + "{names:?}, available: tailscale={available_tailscale}, fabric={available_fabric}" + ); + } + } + } + + #[test] + fn routes_follow_local_order_and_filter_disabled_transports_with_loopback_last() { + let endpoints = [ + json!({"transport": "loopback", "address": "127.0.0.1:4000"}), + json!({"transport": "tailscale", "address": "100.101.102.103:31313"}), + json!({"transport": "fabric", "node": "abc", "protocol": "st3/fleet/x"}), + ]; + let tailscale = Route::Http("http://100.101.102.103:31313".into()); + let fabric = Route::Fabric { + node: "abc".into(), + protocol: "st3/fleet/x".into(), + }; + let loopback = Route::Http("http://127.0.0.1:4000".into()); + for (names, expected) in [ + ( + vec!["fabric", "tailscale"], + vec![fabric.clone(), tailscale.clone(), loopback.clone()], + ), + ( + vec!["tailscale", "fabric"], + vec![tailscale.clone(), fabric.clone(), loopback.clone()], + ), + (vec!["fabric"], vec![fabric, loopback.clone()]), + (vec!["tailscale"], vec![tailscale, loopback.clone()]), + (vec![], vec![loopback]), + ] { + let names = names.into_iter().map(String::from).collect::>(); + assert_eq!( + routes_from_endpoints( + &endpoints, + LocalTransports::configured(Some(&names), true, true), + ), + expected, + "{names:?}" + ); + } + } + fn shim(root: &Path, body: &str) -> PathBuf { let path = root.join("fabric"); std::fs::write(&path, format!("#!/bin/sh\n{body}\n")).unwrap(); diff --git a/crates/smallclaims/src/sync/worker.rs b/crates/smallclaims/src/sync/worker.rs index f7a8d267357..6768ab40d49 100644 --- a/crates/smallclaims/src/sync/worker.rs +++ b/crates/smallclaims/src/sync/worker.rs @@ -619,7 +619,7 @@ pub async fn run( config .fleet .as_ref() - .is_some_and(|file| file.transports.iter().any(|name| name == transport)) + .is_none_or(|file| file.transports.iter().any(|name| name == transport)) }; let tailscale = config .fleet @@ -653,10 +653,11 @@ pub async fn run( view: Arc::default(), view_refresh: Arc::default(), view_changed: watch::channel(0).0, - transports: Arc::new(std::sync::RwLock::new(LocalTransports { - tailscale: false, - fabric: fabric.is_some(), - })), + transports: Arc::new(std::sync::RwLock::new(LocalTransports::configured( + config.fleet.as_ref().map(|file| file.transports.as_slice()), + local_addresses().iter().any(is_tailnet_address), + fabric.is_some(), + ))), fabric: fabric.clone(), bootstrap: config.fleet.as_ref().and_then(|file| { let sponsor = file.sponsor.clone()?; @@ -777,6 +778,8 @@ pub async fn run( if let Some(address) = tailnet { endpoints.update(|set| set.tailscale = vec![address]); } + // Only an enabled transport gets a probe task, so availability changes cannot + // re-enable Tailscale when the explicit local transport list excludes it. if let Some(tailscale) = tailscale { tokio::spawn(crate::task_cpu::account( "replication.tailnet", @@ -1205,10 +1208,10 @@ async fn lock_current_settings( .await? } -/// Who this node dials, and the loopback URLs that reach each, most preferred first: every -/// current listening member other than itself, and every config peer that has never been a -/// member. A dial-out member is never dialed. A config peer entry for a member is that -/// member's first route from this machine. +/// Who this node dials, with routes most preferred first: every current listening member +/// other than itself, and every config peer that has never been a member. A dial-out +/// member is never dialed. An explicit config peer (`--peer`) route takes precedence over +/// the local transport list, including its disabled transports, for backward compatibility. pub fn dial_targets( view: &FleetView, own: &str, diff --git a/crates/smallclaims/src/sync/worker_tests.rs b/crates/smallclaims/src/sync/worker_tests.rs index 1490919a701..9fe7a8cad1f 100644 --- a/crates/smallclaims/src/sync/worker_tests.rs +++ b/crates/smallclaims/src/sync/worker_tests.rs @@ -1032,6 +1032,131 @@ fn the_dial_set_comes_from_membership_and_skips_dial_out_members() { ); } +fn transport_order_view() -> FleetView { + let member = |name: &str, endpoints| crate::fleet::MemberView { + name: name.into(), + member_key: format!("{name}-key"), + state: "current".into(), + mode: "listening".into(), + endpoints, + start: 1, + end: None, + ended: None, + removed_by: None, + removal_reason: None, + }; + FleetView { + members: vec![ + member( + "modern", + vec![ + serde_json::json!({"transport": "loopback", "address": "127.0.0.1:4000"}), + serde_json::json!({"transport": "tailscale", "address": "100.101.102.103:31313"}), + serde_json::json!({"transport": "fabric", "node": "modern-node", "protocol": "st3/fleet/x"}), + ], + ), + // An older member need not advertise Fabric to remain reachable. + member( + "legacy", + vec![ + serde_json::json!({"transport": "tailscale", "address": "100.101.102.104:31313"}), + ], + ), + ], + ..FleetView::default() + } +} + +#[test] +fn dial_targets_follow_local_transport_order_with_mixed_legacy_endpoints() { + let view = transport_order_view(); + let tailscale = Route::Http("http://100.101.102.103:31313".into()); + let fabric = Route::Fabric { + node: "modern-node".into(), + protocol: "st3/fleet/x".into(), + }; + let loopback = Route::Http("http://127.0.0.1:4000".into()); + for (names, expected, legacy_enabled) in [ + ( + None, + vec![tailscale.clone(), fabric.clone(), loopback.clone()], + true, + ), + ( + Some(vec!["fabric", "tailscale"]), + vec![fabric.clone(), tailscale.clone(), loopback.clone()], + true, + ), + ( + Some(vec!["tailscale", "fabric"]), + vec![tailscale.clone(), fabric.clone(), loopback.clone()], + true, + ), + ( + Some(vec!["fabric"]), + vec![fabric, loopback.clone()], + false, + ), + ( + Some(vec!["tailscale"]), + vec![tailscale, loopback.clone()], + true, + ), + (Some(vec![]), vec![loopback], false), + ] { + let names = names.map(|names| names.into_iter().map(String::from).collect::>()); + let targets = dial_targets( + &view, + "own", + &[], + LocalTransports::configured(names.as_deref(), true, true), + ); + let mut expected = BTreeMap::from([("modern".to_owned(), expected)]); + if legacy_enabled { + expected.insert( + "legacy".into(), + vec![Route::Http("http://100.101.102.104:31313".into())], + ); + } + assert_eq!(targets, expected, "{names:?}"); + } +} + +#[test] +fn explicit_peer_routes_precede_local_order_and_bypass_disabled_transports() { + let view = transport_order_view(); + let peers = [ + PeerConfig { + name: "modern".into(), + url: "http://100.101.102.105:31313".into(), + }, + PeerConfig { + name: "legacy".into(), + url: "fabric://legacy-node/st3/fleet/x".into(), + }, + PeerConfig { + name: "config-only".into(), + url: "fabric://config-node/st3/fleet/x".into(), + }, + ]; + for names in [vec!["fabric", "tailscale"], vec![]] { + let names = names.into_iter().map(String::from).collect::>(); + let local = LocalTransports::configured(Some(&names), true, true); + let targets = dial_targets(&view, "own", &peers, local); + for member in &view.members { + let peer = peers.iter().find(|peer| peer.name == member.name).unwrap(); + let mut expected = vec![parse_route(&peer.url).unwrap()]; + expected.extend(routes_from_endpoints(&member.endpoints, local)); + assert_eq!(targets[&member.name], expected, "{names:?}"); + } + assert_eq!( + targets["config-only"], + vec![parse_route(&peers[2].url).unwrap()], + "{names:?}" + ); + } +} + fn fleet_claim(store: &Store, kind: &str, subject: &str, fields: Value) { store .append_claim(&ClaimInput { diff --git a/docs/st3/cli-guided-tour.md b/docs/st3/cli-guided-tour.md index c63b14215bd..0c1b3655624 100644 --- a/docs/st3/cli-guided-tour.md +++ b/docs/st3/cli-guided-tour.md @@ -420,6 +420,13 @@ exit status, evidence, remediation, stable repair classes, approval token, and z Why: transport reachability and immutable-record convergence need explicit diagnostics rather than being mistaken for mission health. +For transport preference, inspect `st fleet transports add fabric --help`. +On an intentionally selected listening member, `st fleet transports add fabric --prefer` +enables Fabric or moves an existing Fabric route first without changing its exposure. +Omit `--prefer` to preserve existing transport order; see +[automatic transport selection](replication.md#automatic-transport-selection) for +explicit peer override precedence and the loopback-last fallback. + ```sh st replication --help st replication status --help diff --git a/docs/st3/replication.md b/docs/st3/replication.md index 517a5ccc278..bcea53c3242 100644 --- a/docs/st3/replication.md +++ b/docs/st3/replication.md @@ -11,6 +11,30 @@ or replica. Devices read and act through a member's client gateway and are not s Replication makes the logical authority equal across configured nodes. It does not make the SQLite files byte-identical. +## Automatic transport selection + +The local `fleet.toml` `transports` list is both an enablement filter and a +preference order for automatic replication and peer-relay routes. With +`["fabric", "tailscale"]`, available Fabric routes precede Tailscale; reversing +the list reverses that preference. A carrier absent from an explicit list is not +used automatically, even if its executable is installed. An empty list disables +both Fabric and Tailscale automatic routes. Loopback remains the last automatic +fallback, independent of its list position. + +Explicit config peer URLs (`[[peers]]` / `--peer`) retain precedence over automatic +routes for backward compatibility. To let fleet preference select the carrier, +remove the explicit override. Legacy operation without `fleet.toml` retains +Tailscale-first automatic routing among locally available carriers. + +Use `st fleet transports add fabric --prefer` to enable Fabric and move it to the +front, or to prefer an already enabled Fabric route without changing its saved +protocol or exposure. An add without `--prefer` appends a new carrier and leaves +an existing list unchanged. `st fleet transports remove fabric` keeps the relative +order of remaining carriers and refuses to remove the last usable transport. +These are local settings: choose the preference on each member independently. +Installed services restart automatically; otherwise restart +`st3 replication-worker` as instructed, including when using `--no-service`. + ## Local adapter membership and leave authority Local adapters may read `GET /v1/internal/fleet/membership` on the daemon's Unix diff --git a/docs/two-machines.md b/docs/two-machines.md index abe290aa800..a9e048cc31c 100644 --- a/docs/two-machines.md +++ b/docs/two-machines.md @@ -62,6 +62,35 @@ Beacon listens on its own loopback port and exposes it through Fabric. The invit With their user services installed, Tailscale, Fabric, and the Fractal replication worker restart after reboot and reconnect after network interruptions. A terminal does not need to stay open. +### Prefer Fabric while keeping Tailscale + +On an existing listening member, enable Fabric and make it the first automatic +route in one command: + +```sh +st fleet transports add fabric --prefer +st fleet status --json +``` + +Run this on each machine whose outbound connections should prefer Fabric. If Fabric +is already enabled, `--prefer` moves it to the front without changing its protocol +or exposure. Without `--prefer`, adding Fabric appends it; repeated adds preserve +the existing order. Removing it with `st fleet transports remove fabric` preserves +the remaining transports' order and requires another usable transport. + +The local `fleet.toml` transport list controls which automatic Tailscale/Fabric +routes are enabled and their order for replication and peer relay. For a new fleet +or joining machine, `--transports fabric,tailscale` selects Fabric first; +`--transports tailscale,fabric` selects Tailscale first. Loopback remains the last +automatic fallback regardless of its list position. Explicit config peer URLs +(`[[peers]]` / `--peer`) keep highest priority for backward compatibility, so remove +an old explicit route if automatic preference should choose the carrier. Legacy +machines without `fleet.toml` retain Tailscale-first automatic routing. + +Transport commands restart installed services. With `--no-service`, or without +installed services, restart `st3 replication-worker` as instructed to apply the +local settings. Fabric trust and service grants are still required. + ### Reconcile Fabric runtime policy `st-fabric-adapter` is a separate process, not part of the st daemon. Start Fabric diff --git a/release-notes/fleet-automatic-transport-preference-q33.json b/release-notes/fleet-automatic-transport-preference-q33.json new file mode 100644 index 00000000000..88f6565a0e5 --- /dev/null +++ b/release-notes/fleet-automatic-transport-preference-q33.json @@ -0,0 +1,11 @@ +{ + "version": 1, + "summary": "Choose automatic fleet transport preference with st fleet transports add fabric --prefer", + "replay": {"status": "none", "detail": "No claim replay or stored-history changes."}, + "schema": {"status": "none", "detail": "Uses the existing ordered fleet.toml transports list; no database schema or migration changes."}, + "rules": {"status": "none", "detail": "No checkpoint rules or coordinated member upgrade changes."}, + "compatibility": {"status": "changed", "detail": "The local fleet.toml transports list filters enabled automatic Tailscale/Fabric routes and determines their order for replication and peer relay. Loopback remains last. Legacy operation without fleet.toml remains Tailscale-first. Explicit config peer URLs and --peer overrides retain highest priority. Adding Fabric without --prefer preserves existing order; --prefer moves new or already enabled Fabric to the front without reconfiguring an existing exposure. Removal preserves remaining order."}, + "downtime": {"status": "none", "detail": "Transport commands refresh installed services. Without services or with --no-service, restart st3 replication-worker as instructed to apply local settings."}, + "manual": {"status": "none", "detail": "No action is required to retain existing ordered settings. To prefer Fabric on a member, run st fleet transports add fabric --prefer; Fabric trust and service grants remain required. Remove explicit peer overrides if automatic preference should choose the carrier."}, + "recovery": {"status": "none", "detail": "No stored-state recovery changes. Transport removal continues to require another usable transport."} +} From 663c5bee0bf61ffb89d1acdc01ba610ee014b87c Mon Sep 17 00:00:00 2001 From: schickling-assistant <261620128+schickling-assistant@users.noreply.github.com> Date: Thu, 22 Oct 2026 10:25:01 +0200 Subject: [PATCH 2/2] fix(fleet): preserve omitted-list defaults and require both service restarts agent-identity: agent/dotfiles/fabric-mesh agent-persona: generalist agent-supervisor: unavailable agent-tool: OMP agent-tool-version: 18.9.1 agent-runtime: OMP 18.9.1 agent-model: anthropic/claude-opus-5-5 --- crates/fractal-peer/src/peer.rs | 38 +++++++++++++++++++ crates/fractal/src/fleet_cli.rs | 16 ++++---- crates/fractal/src/main.rs | 4 +- crates/fractal/tests/fleet.rs | 11 ++++-- crates/smallclaims/src/fleet/file.rs | 36 ++++++++++++++++++ docs/st3/replication.md | 10 +++-- docs/two-machines.md | 5 ++- ...et-automatic-transport-preference-q33.json | 2 +- 8 files changed, 103 insertions(+), 19 deletions(-) diff --git a/crates/fractal-peer/src/peer.rs b/crates/fractal-peer/src/peer.rs index 8e340c27909..24c9a9c32a0 100644 --- a/crates/fractal-peer/src/peer.rs +++ b/crates/fractal-peer/src/peer.rs @@ -2123,6 +2123,44 @@ mod tests { relay.transports, ); } + // A daemon restart reconstructs the relay from the saved fleet file. Preference + // and removal must affect real next hops, not just the serialized setting. + let root = tempfile::tempdir().unwrap(); + let secret = root.path().join("fleet-secret"); + fs::write(&secret, [7_u8; 32]).unwrap(); + fs::set_permissions(&secret, fs::Permissions::from_mode(0o600)).unwrap(); + let key = root.path().join("node.key"); + MemberKey::load_or_create(&key).unwrap(); + let mut file = smallclaims::fleet::FleetFile { + fleet_id: "1f91ca65-7793-48cc-866e-ac15690130e1".into(), + node_key_file: key, + ..Default::default() + }; + for (policy, expected) in [ + (vec!["tailscale", "fabric"], vec![tailscale, fabric, loopback]), + (vec!["fabric", "tailscale"], vec![fabric, tailscale, loopback]), + (vec!["tailscale"], vec![tailscale, loopback]), + ] { + file.transports = policy.into_iter().map(str::to_owned).collect(); + file.save(root.path()).unwrap(); + let restarted = ClientRelay::from_config(&Config { + node: "gateway".into(), + fleet_id: Some(file.fleet_id.clone()), + shared_secret_file: Some(secret.clone()), + fleet: smallclaims::fleet::FleetFile::load(root.path()).unwrap(), + state_dir: root.path().into(), + ..Default::default() + }).unwrap().unwrap().with_links(relay.links.as_ref().unwrap().clone()); + *restarted.membership.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(( + std::time::Instant::now(), + relay.fleet_view(), + )); + assert_eq!( + restarted.next_hops_with_transports("owner", &[], available) + .into_iter().map(|peer| peer.url).collect::>(), + expected, + ); + } // Explicit config routes retain their precedence for mixed config/member fleets. relay.peers.push(PeerConfig { name: "owner".into(), diff --git a/crates/fractal/src/fleet_cli.rs b/crates/fractal/src/fleet_cli.rs index 17df664a60b..80719d0cfe0 100644 --- a/crates/fractal/src/fleet_cli.rs +++ b/crates/fractal/src/fleet_cli.rs @@ -459,7 +459,7 @@ pub(super) async fn run_fleet(endpoint: &Endpoint, command: FleetCommand, json_o "This member is now {}; it announces the change when its replication worker starts.", file.mode.as_str() ); - refresh_fleet_worker(args.no_service)?; + refresh_fleet_services(args.no_service)?; Ok(()) } FleetCommand::Transports { command } => { @@ -648,15 +648,15 @@ async fn save_and_publish_fleet_settings( }, ) .await - .context("fleet.toml saved; endpoint publication did not complete; restart the replication worker to announce the saved settings")?; + .context("fleet.toml saved; endpoint publication did not complete; restart st3 daemon and st3 replication-worker to apply saved relay settings and announce the saved endpoints")?; Ok(()) } -fn refresh_fleet_worker(no_service: bool) -> Result<()> { +fn refresh_fleet_services(no_service: bool) -> Result<()> { if !no_service && services_installed()? { fractal_daemon::service::install(Config::load_with_fleet(None)?)?; } else { - println!("Restart st3 replication-worker for the change to take effect."); + println!("Restart st3 daemon and st3 replication-worker for the change to take effect."); println!( "The old worker no longer publishes endpoints after its settings change; restart it to resume publication." ); @@ -697,7 +697,7 @@ async fn run_fleet_transports( file.save(&config.state_dir)?; drop(lock); println!("Fabric is now preferred; its existing protocol is unchanged."); - return refresh_fleet_worker(*no_service); + return refresh_fleet_services(*no_service); } println!("Fabric is already enabled; remove it before changing its protocol."); return Ok(()); @@ -807,8 +807,8 @@ async fn run_fleet_transports( None => Err(anyhow::anyhow!("fabric is not installed")), }; drop(lock); - // Refresh even when cleanup fails: no worker may keep using the removed carrier. - refresh_fleet_worker(no_service)?; + // Refresh even when cleanup fails: relay and worker must stop using the carrier. + refresh_fleet_services(no_service)?; if let Err(error) = result { println!( "Fabric endpoint removed; unexposure still needs cleanup: {error:#}. When Fabric is available, run fabric unexpose {protocol}" @@ -822,7 +822,7 @@ async fn run_fleet_transports( } }; drop(lock); - refresh_fleet_worker(no_service) + refresh_fleet_services(no_service) } /// Settings for a migrating node: its existing replication port unless one is given. diff --git a/crates/fractal/src/main.rs b/crates/fractal/src/main.rs index a7521a95196..d69f39e16b2 100644 --- a/crates/fractal/src/main.rs +++ b/crates/fractal/src/main.rs @@ -933,7 +933,7 @@ enum FleetTransportsCommand { /// Prefer Fabric over other automatic routes, including when already enabled. #[arg(long)] prefer: bool, - /// Leave the replication worker restart to the operator. + /// Leave the daemon and replication worker restarts to the operator. #[arg(long)] no_service: bool, }, @@ -941,7 +941,7 @@ enum FleetTransportsCommand { Remove { #[arg(value_parser = ["fabric"])] transport: String, - /// Leave the replication worker restart to the operator. + /// Leave the daemon and replication worker restarts to the operator. #[arg(long)] no_service: bool, }, diff --git a/crates/fractal/tests/fleet.rs b/crates/fractal/tests/fleet.rs index da1282d22a9..f1039e9d584 100644 --- a/crates/fractal/tests/fleet.rs +++ b/crates/fractal/tests/fleet.rs @@ -3660,7 +3660,8 @@ async fn fabric_transport_preference_preserves_other_transports_and_existing_exp if prefer_on_add { args.push("--prefer"); } - node.st_ok(&args); + let output = node.st_ok(&args); + assert!(output.contains("Restart st3 daemon and st3 replication-worker")); assert_eq!( configured_file(&node).transports, if prefer_on_add { @@ -3680,7 +3681,7 @@ async fn fabric_transport_preference_preserves_other_transports_and_existing_exp "refuse", ) .unwrap(); - node.st_ok(&[ + let output = node.st_ok(&[ "fleet", "transports", "add", @@ -3690,11 +3691,15 @@ async fn fabric_transport_preference_preserves_other_transports_and_existing_exp "ignored", "--no-service", ]); + if !prefer_on_add { + assert!(output.contains("Restart st3 daemon and st3 replication-worker")); + } let preferred = configured_file(&node); assert_eq!(preferred.transports, ["fabric", "loopback", "tailscale"]); assert_eq!(preferred.fabric_protocol, before.fabric_protocol); assert_eq!(node.st_json(&["fleet", "status"])["view"], endpoints); - node.st_ok(&["fleet", "transports", "remove", "fabric", "--no-service"]); + let output = node.st_ok(&["fleet", "transports", "remove", "fabric", "--no-service"]); + assert!(output.contains("Restart st3 daemon and st3 replication-worker")); assert_eq!(configured_file(&node).transports, ["loopback", "tailscale"]); } } diff --git a/crates/smallclaims/src/fleet/file.rs b/crates/smallclaims/src/fleet/file.rs index 36e955e3299..6bc863c08e5 100644 --- a/crates/smallclaims/src/fleet/file.rs +++ b/crates/smallclaims/src/fleet/file.rs @@ -143,7 +143,9 @@ pub struct FleetFile { pub mode: FleetMode, pub port: Option, /// Enabled automatic transports, in dial preference order; loopback is always last. + /// An omitted list defaults to Tailscale then Fabric; an explicit empty list disables both. /// Explicit config peer (`--peer`) routes retain precedence over this list. + #[serde(default = "default_transports")] pub transports: Vec, pub fabric_protocol: Option, /// Accept HMAC-only exchanges from config peers. True only during a migration. @@ -156,6 +158,10 @@ pub struct FleetFile { pub removed: Option, } +fn default_transports() -> Vec { + vec!["tailscale".into(), "fabric".into()] +} + impl FleetFile { /// Serialize refusal persistence and leave cleanup, including the read before a write. /// Keep the lock outside `fleet/`, and never unlink it: waiters must share one inode even @@ -287,3 +293,33 @@ impl FleetFile { Self::resolve(state_dir, path) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn omitted_transport_list_defaults_to_tailscale_then_fabric() { + let file: FleetFile = toml::from_str("").unwrap(); + assert_eq!(file.transports, ["tailscale", "fabric"]); + } + + #[test] + fn explicit_transport_lists_preserve_empty_disabled_and_reordered_values() { + for (input, expected) in [ + ("transports = []", vec![]), + ("transports = ['fabric']", vec!["fabric"]), + ("transports = ['tailscale']", vec!["tailscale"]), + ( + "transports = ['fabric', 'tailscale']", + vec!["fabric", "tailscale"], + ), + ] { + let file: FleetFile = toml::from_str(input).unwrap(); + assert_eq!(file.transports, expected, "{input}"); + let round_trip: FleetFile = + toml::from_str(&toml::to_string(&file).unwrap()).unwrap(); + assert_eq!(round_trip.transports, expected, "{input}"); + } + } +} diff --git a/docs/st3/replication.md b/docs/st3/replication.md index bcea53c3242..b0a9be08b01 100644 --- a/docs/st3/replication.md +++ b/docs/st3/replication.md @@ -32,8 +32,11 @@ protocol or exposure. An add without `--prefer` appends a new carrier and leaves an existing list unchanged. `st fleet transports remove fabric` keeps the relative order of remaining carriers and refuses to remove the last usable transport. These are local settings: choose the preference on each member independently. -Installed services restart automatically; otherwise restart +Installed services restart automatically; otherwise restart both `st3 daemon` and `st3 replication-worker` as instructed, including when using `--no-service`. +The daemon reloads peer-relay transport settings; the worker reloads replication +transport settings. Restarting only the worker leaves the running daemon's relay +preference and enabled transports unchanged. ## Local adapter membership and leave authority @@ -208,8 +211,9 @@ with `details.reason = "dial-out-owner"` and an explicit no-inbound-route messag conversation, terminal, raw attach, workspace and attachment reads, without trying listening peers as relays. The member's published mode overrides a configured direct URL. Correct an incorrect mode on the owning member with `st fleet mode listening`. This restarts installed -services; without installed services (or with `--no-service`), restart `st3 replication-worker` -as the command instructs. The restarted worker enables its listener and publishes the new +services; without installed services (or with `--no-service`), restart both `st3 daemon` +and `st3 replication-worker` as the command instructs. The restarted daemon reloads +relay settings; the restarted worker enables its listener and publishes the new mode/endpoints. Other nodes see it after replication and at most five seconds of membership caching. Reverse owner-RPC and PTY transport over a dial-out connection requires separate transport support. diff --git a/docs/two-machines.md b/docs/two-machines.md index a9e048cc31c..19f5a8670ef 100644 --- a/docs/two-machines.md +++ b/docs/two-machines.md @@ -88,8 +88,9 @@ an old explicit route if automatic preference should choose the carrier. Legacy machines without `fleet.toml` retain Tailscale-first automatic routing. Transport commands restart installed services. With `--no-service`, or without -installed services, restart `st3 replication-worker` as instructed to apply the -local settings. Fabric trust and service grants are still required. +installed services, restart both `st3 daemon` and `st3 replication-worker` as +instructed: the daemon reloads peer-relay settings, and the worker reloads +replication settings. Fabric trust and service grants are still required. ### Reconcile Fabric runtime policy diff --git a/release-notes/fleet-automatic-transport-preference-q33.json b/release-notes/fleet-automatic-transport-preference-q33.json index 88f6565a0e5..662cc3763a3 100644 --- a/release-notes/fleet-automatic-transport-preference-q33.json +++ b/release-notes/fleet-automatic-transport-preference-q33.json @@ -5,7 +5,7 @@ "schema": {"status": "none", "detail": "Uses the existing ordered fleet.toml transports list; no database schema or migration changes."}, "rules": {"status": "none", "detail": "No checkpoint rules or coordinated member upgrade changes."}, "compatibility": {"status": "changed", "detail": "The local fleet.toml transports list filters enabled automatic Tailscale/Fabric routes and determines their order for replication and peer relay. Loopback remains last. Legacy operation without fleet.toml remains Tailscale-first. Explicit config peer URLs and --peer overrides retain highest priority. Adding Fabric without --prefer preserves existing order; --prefer moves new or already enabled Fabric to the front without reconfiguring an existing exposure. Removal preserves remaining order."}, - "downtime": {"status": "none", "detail": "Transport commands refresh installed services. Without services or with --no-service, restart st3 replication-worker as instructed to apply local settings."}, + "downtime": {"status": "none", "detail": "Transport commands refresh installed services. Without services or with --no-service, restart both st3 daemon and st3 replication-worker as instructed: the daemon reloads peer-relay settings and the worker reloads replication settings."}, "manual": {"status": "none", "detail": "No action is required to retain existing ordered settings. To prefer Fabric on a member, run st fleet transports add fabric --prefer; Fabric trust and service grants remain required. Remove explicit peer overrides if automatic preference should choose the carrier."}, "recovery": {"status": "none", "detail": "No stored-state recovery changes. Transport removal continues to require another usable transport."} }