diff --git a/.config/daily-only.filterset b/.config/daily-only.filterset index 7f8e037b1ff..e31f4bcb8f2 100644 --- a/.config/daily-only.filterset +++ b/.config/daily-only.filterset @@ -23,6 +23,6 @@ package(=fractal) and binary(=integration) and test(/^boot_canaries::/) and not # doctor and machines page), #3611 (every invite refusal), #3614 (the frozen-member doctor # warning), and the existing fractal-store fleet_admission_tests (removal fenced through a relay), # smallclaims fleet/membership.rs, fleet/transport.rs and sync/worker_tests.rs. Kept per PR: the -# two fleet::action_coverage_ tests that docs/st3/action-coverage.json names, and the suite's two +# fleet::action_coverage_ tests, the transport_prefer_ command contracts, and the suite's two # fast self-tests. linux-fleet-compat runs its ignored test through cargo test and is unaffected. -package(=fractal) and binary(=integration) and test(/^fleet::/) and not test(/^fleet::(action_coverage_|fleet_workflows_have_no_path_filter$|fixture_port_reservation_excludes_other_processes_until_released$)/) +package(=fractal) and binary(=integration) and test(/^fleet::/) and not test(/^fleet::(action_coverage_|transport_prefer_|fleet_workflows_have_no_path_filter$|fixture_port_reservation_excludes_other_processes_until_released$)/) diff --git a/crates/fractal/src/fleet_cli.rs b/crates/fractal/src/fleet_cli.rs index 80719d0cfe0..85f21355df7 100644 --- a/crates/fractal/src/fleet_cli.rs +++ b/crates/fractal/src/fleet_cli.rs @@ -115,7 +115,8 @@ pub(super) async fn run_fleet(endpoint: &Endpoint, command: FleetCommand, json_o FleetCommand::Migrate(args) => !args.no_service, FleetCommand::Transports { command } => match command { FleetTransportsCommand::Add { no_service, .. } - | FleetTransportsCommand::Remove { no_service, .. } => !*no_service, + | FleetTransportsCommand::Remove { no_service, .. } + | FleetTransportsCommand::Prefer { no_service, .. } => !*no_service, }, _ => false, }; @@ -679,12 +680,23 @@ async fn run_fleet_transports( ); let transport = match &command { FleetTransportsCommand::Add { transport, .. } - | FleetTransportsCommand::Remove { transport, .. } => transport, + | FleetTransportsCommand::Remove { transport, .. } + | FleetTransportsCommand::Prefer { transport, .. } => transport, }; let enabled = file .transports .iter() .any(|configured| configured == transport); + if matches!(&command, FleetTransportsCommand::Prefer { .. }) { + anyhow::ensure!( + enabled, + "{transport} is not enabled on this member; preference only reorders enabled transports" + ); + if file.transports.first() == Some(transport) { + println!("{transport} is already preferred."); + return Ok(()); + } + } // Re-adding can change local preference, but never reconfigures the exposure. if let FleetTransportsCommand::Add { prefer, no_service, .. @@ -717,6 +729,20 @@ async fn run_fleet_transports( .clone() .unwrap_or_else(|| default_fabric_protocol(&file.fleet_id)); let no_service = match command { + FleetTransportsCommand::Prefer { + transport, + no_service, + } => { + let index = file + .transports + .iter() + .position(|configured| configured == &transport) + .expect("preferred transport was checked as enabled"); + file.transports[..=index].rotate_right(1); + save_and_publish_fleet_settings(client, config, &file, endpoints).await?; + println!("{transport} is now preferred; transport exposure is unchanged."); + no_service + } FleetTransportsCommand::Add { fabric_protocol, prefer, diff --git a/crates/fractal/src/main.rs b/crates/fractal/src/main.rs index 3cacc9e8112..440b9ddb6dc 100644 --- a/crates/fractal/src/main.rs +++ b/crates/fractal/src/main.rs @@ -951,6 +951,14 @@ enum FleetTransportsCommand { #[arg(long)] no_service: bool, }, + /// Prefer an enabled transport without changing exposure or removing any carrier. + Prefer { + #[arg(value_parser = ["tailscale", "fabric"])] + transport: String, + /// Leave the daemon and replication worker restarts to the operator. + #[arg(long)] + no_service: bool, + }, } #[derive(Args)] diff --git a/crates/fractal/tests/fleet.rs b/crates/fractal/tests/fleet.rs index d9f06e78113..1249ff2b9e1 100644 --- a/crates/fractal/tests/fleet.rs +++ b/crates/fractal/tests/fleet.rs @@ -3704,6 +3704,112 @@ async fn fabric_transport_preference_preserves_other_transports_and_existing_exp } } +#[tokio::test(flavor = "multi_thread")] +async fn transport_prefer_reorders_under_lock_without_changing_exposure() { + use std::os::unix::fs::MetadataExt as _; + + if fractal_daemon::test_support::supervise_test() { + return; + } + let root = tempfile::tempdir().unwrap(); + let mut node = anchor(root.path(), "preference").await; + node.stop_worker(); + let mut file = configured_file(&node); + file.fabric = Some(fabric_shim(root.path(), &node.name)); + file.transports = vec!["loopback".into(), "tailscale".into()]; + file.save(&node.state_dir()).unwrap(); + node.start_worker(); + node.wait_listening().await; + node.st_ok(&["fleet", "transports", "add", "fabric", "--no-service"]); + node.stop_worker(); + let registry = root.path().join("fabric-registry"); + let calls = fs::read(registry.join("calls")).unwrap(); + let endpoints = node.st_json(&["fleet", "status"])["view"].clone(); + let path = fractal_daemon::config::FleetFile::path(&node.state_dir()); + + // Observe actual lock contention, then change another setting while holding it. + // The CLI must load after acquiring the same settings lock, not before. + let lock = fractal_daemon::config::FleetFile::lock_settings(&node.state_dir()).unwrap(); + let barrier = root.path().join("prefer-lock"); + fs::create_dir(&barrier).unwrap(); + let mut prefer = node + .command(&["fleet", "transports", "prefer", "fabric", "--no-service"]) + .env("ST3_TEST_REMOVAL_WRITE_BARRIER", &barrier) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let cli = prefer.id().to_string(); + wait_until("preference CLI contends on settings lock", 20, || async { + fs::read_to_string(barrier.join("contended")) + .is_ok_and(|pids| pids.lines().any(|pid| pid == cli)) + }) + .await; + assert!(prefer.try_wait().unwrap().is_none()); + let mut before = configured_file(&node); + before.fabric_protocol = Some("preserved/under-lock".into()); + before.save(&node.state_dir()).unwrap(); + let old_file = fs::File::open(&path).unwrap(); + let old_inode = old_file.metadata().unwrap().ino(); + let old_bytes = fs::read(&path).unwrap(); + drop(lock); + let output = prefer.wait_with_output().unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert!( + String::from_utf8_lossy(&output.stdout) + .contains("Restart st3 daemon and st3 replication-worker") + ); + let preferred = configured_file(&node); + assert_eq!(preferred.transports, ["fabric", "loopback", "tailscale"]); + assert_eq!(preferred.fabric_protocol, before.fabric_protocol); + assert_ne!(fs::metadata(&path).unwrap().ino(), old_inode); + assert_eq!(fs::metadata(&path).unwrap().permissions().mode() & 0o777, 0o600); + use std::io::Read as _; + let mut retained_bytes = Vec::new(); + (&old_file).read_to_end(&mut retained_bytes).unwrap(); + assert_eq!(retained_bytes, old_bytes, "atomic save must replace, not truncate"); + + let inode = fs::metadata(&path).unwrap().ino(); + let bytes = fs::read(&path).unwrap(); + let output = node.st_ok(&["fleet", "transports", "prefer", "fabric", "--no-service"]); + assert!(output.contains("already preferred")); + assert!(!output.contains("Restart")); + assert_eq!(fs::metadata(&path).unwrap().ino(), inode); + assert_eq!(fs::read(&path).unwrap(), bytes); + node.st_ok(&["fleet", "transports", "prefer", "tailscale", "--no-service"]); + assert_eq!(configured_file(&node).transports, ["tailscale", "fabric", "loopback"]); + assert_eq!(configured_file(&node).fabric_protocol, before.fabric_protocol); + assert_eq!(node.st_json(&["fleet", "status"])["view"], endpoints); + assert_eq!(fs::read(registry.join("calls")).unwrap(), calls); +} + +#[tokio::test(flavor = "multi_thread")] +async fn transport_prefer_refuses_disabled_carriers_without_saving() { + if fractal_daemon::test_support::supervise_test() { + return; + } + let root = tempfile::tempdir().unwrap(); + let mut node = anchor(root.path(), "disabled-preference").await; + node.stop_worker(); + let path = fractal_daemon::config::FleetFile::path(&node.state_dir()); + let before = fs::read(&path).unwrap(); + let endpoints = node.st_json(&["fleet", "status"])["view"].clone(); + for transport in ["tailscale", "fabric"] { + let output = node.st(&["fleet", "transports", "prefer", transport, "--no-service"]); + assert!(!output.status.success()); + assert!( + String::from_utf8_lossy(&output.stderr) + .contains(&format!("{transport} is not enabled")) + ); + assert_eq!(fs::read(&path).unwrap(), before); + assert_eq!(node.st_json(&["fleet", "status"])["view"], endpoints); + } +} + #[tokio::test(flavor = "multi_thread")] async fn fabric_transport_change_refusals_leave_settings_and_endpoints_unchanged() { if fractal_daemon::test_support::supervise_test() { diff --git a/crates/fractal/tests/setup.rs b/crates/fractal/tests/setup.rs index ee6ce9938eb..a2de8ba7210 100644 --- a/crates/fractal/tests/setup.rs +++ b/crates/fractal/tests/setup.rs @@ -233,6 +233,7 @@ fn uninstall_and_fleet_conflicts_precede_settings_mutations() { vec!["fleet", "mode", "dial-out"], vec!["fleet", "transports", "add", "fabric"], vec!["fleet", "transports", "remove", "fabric"], + vec!["fleet", "transports", "prefer", "tailscale"], vec!["fleet", "migrate", "--finish"], vec!["fleet", "migrate", "--unfinish"], vec!["fleet", "migrate", "--anchor"], diff --git a/docs/fleet-join.md b/docs/fleet-join.md index f305c69dc7e..32d2f7f2d65 100644 --- a/docs/fleet-join.md +++ b/docs/fleet-join.md @@ -165,7 +165,8 @@ successful contact; authentication, admission, membership and projection failure ## Transports -The worker dials listening members using local overrides, Tailscale, Fabric, then loopback routes. +The worker dials listening members using local overrides, then enabled Tailscale/Fabric +routes in local `fleet.toml` preference order, then loopback routes. Successful inbound exchanges suppress redundant reverse dials. Graph/network wakes request prompt sync; unreachable peers use adaptive network backoff. This is connection recovery, not a retry of broken local work. A connection pushes and pulls history while its backlog drains. @@ -173,21 +174,25 @@ broken local work. A connection pushes and pulls history while its backlog drain ### Changing an existing member's transports ```text -st fleet transports add fabric [--fabric-protocol PROTOCOL] [--no-service] +st fleet transports add fabric [--prefer] [--fabric-protocol PROTOCOL] [--no-service] st fleet transports remove fabric [--no-service] +st fleet transports prefer [--no-service] ``` Run these commands on the member whose transports should change. They update the existing `fleet.toml`, publish `fleet.member-endpoints` immediately, and refresh the service using the same path as `st fleet mode`. Run them with the local daemon running. The member's name, identity, secret, signing key and graph history stay intact. -The command currently supports Fabric; changing Tailscale is outside this command's scope. +Add and remove support Fabric. Prefer reorders either enabled Tailscale or Fabric +without changing enablement or exposure; it rejects disabled carriers and is a +no-op when already first. It preserves the remaining carriers' relative order. Adding Fabric requires a listening member with a configured replication port and a working local `fabric` command. Use `st fleet mode listening` first for a dial-out member. Add exposes `127.0.0.1:PORT` before saving or announcing the Fabric endpoint; if exposure fails, settings and endpoints are unchanged. The protocol defaults to the saved `fabric_protocol`, or -`st3/fleet/FLEET_ID`. Re-adding an enabled transport changes nothing; to change its protocol, +`st3/fleet/FLEET_ID`. Re-adding an enabled transport changes nothing unless `--prefer` +moves it first; the saved protocol and exposure remain unchanged. To change its protocol, remove it and add it with `--fabric-protocol PROTOCOL`. Configure Fabric trust and grants on the peers as described below. diff --git a/docs/st3/action-coverage.json b/docs/st3/action-coverage.json index 3c54d437415..37b3cd77f99 100644 --- a/docs/st3/action-coverage.json +++ b/docs/st3/action-coverage.json @@ -1139,6 +1139,15 @@ ], "verification": "real CLI over isolated Unix daemons and workers with a Fabric shim; graph sync, restart restoration and refused exposure" }, + { + "command": "st fleet transports prefer", + "description": "Prefer an enabled transport without changing exposure or removing any carrier", + "tests": [ + "crates/fractal/tests/fleet.rs::transport_prefer_reorders_under_lock_without_changing_exposure", + "crates/fractal/tests/fleet.rs::transport_prefer_refuses_disabled_carriers_without_saving" + ], + "verification": "real CLI over isolated Unix daemons with a Fabric shim; both reorder directions, disabled-carrier refusal, exposure preservation, settings-lock contention, atomic save and no-op" + }, { "command": "st fleet transports remove", "description": "Withdraw Fabric from this fleet member while keeping a usable carrier", diff --git a/docs/st3/action-coverage.md b/docs/st3/action-coverage.md index 86da852b424..891dc6d2c6c 100644 --- a/docs/st3/action-coverage.md +++ b/docs/st3/action-coverage.md @@ -192,6 +192,7 @@ edits and removes retired subjects without changing unfiltered reads. | `st fleet remove` | existing CLI integration | [`action_coverage_fleet_controls_and_checkpoint_administration_survive_restarts`](https://github.com/compoundingtech/smalltalk/blob/main/crates/fractal/tests/fleet.rs) | | `st fleet status` | real CLI over private Unix socket; disk store reopened between actions | [`cli_reads_preserve_operational_views_after_restart`](https://github.com/compoundingtech/smalltalk/blob/main/crates/fractal/tests/action_coverage.rs) | | `st fleet transports add` | real CLI over isolated Unix daemons and workers with a Fabric shim; graph sync, restart restoration and refused exposure | [`an_existing_member_adds_and_removes_fabric_without_rejoining`](https://github.com/compoundingtech/smalltalk/blob/main/crates/fractal/tests/fleet.rs), [`fabric_transport_change_refusals_leave_settings_and_endpoints_unchanged`](https://github.com/compoundingtech/smalltalk/blob/main/crates/fractal/tests/fleet.rs) | +| `st fleet transports prefer` | real CLI over isolated Unix daemons with a Fabric shim; both reorder directions, disabled-carrier refusal, exposure preservation, settings-lock contention, atomic save and no-op | [`transport_prefer_reorders_under_lock_without_changing_exposure`](https://github.com/compoundingtech/smalltalk/blob/main/crates/fractal/tests/fleet.rs), [`transport_prefer_refuses_disabled_carriers_without_saving`](https://github.com/compoundingtech/smalltalk/blob/main/crates/fractal/tests/fleet.rs) | | `st fleet transports remove` | real CLI over isolated Unix daemons and workers with a Fabric shim; withdrawal before unexpose, remaining-carrier sync, guards and rollback without Fabric | [`an_existing_member_adds_and_removes_fabric_without_rejoining`](https://github.com/compoundingtech/smalltalk/blob/main/crates/fractal/tests/fleet.rs), [`fabric_transport_change_refusals_leave_settings_and_endpoints_unchanged`](https://github.com/compoundingtech/smalltalk/blob/main/crates/fractal/tests/fleet.rs) | | `st fleet wait` | existing CLI integration | [`fleet_wait_after_a_restart_waits_until_the_member_has_caught_up`](https://github.com/compoundingtech/smalltalk/blob/main/crates/fractal/tests/fleet.rs) | | `st gate cargo-test` | existing CLI integration | [`cargo_test_waits_for_its_target_and_breaks_on_a_build_this_host_cannot_make`](https://github.com/compoundingtech/smalltalk/blob/main/crates/fractal/tests/gate_kinds.rs) | diff --git a/docs/st3/actor-contract.json b/docs/st3/actor-contract.json index 8ef214ca27a..9fcf6cddabe 100644 --- a/docs/st3/actor-contract.json +++ b/docs/st3/actor-contract.json @@ -885,6 +885,15 @@ "actor_environment": null, "ignores_actor": false }, + "st fleet transports prefer": { + "actor": "none", + "accepts_person": false, + "can_carry_person": false, + "as_role": "none", + "alternate_actor_option": null, + "actor_environment": null, + "ignores_actor": false + }, "st fleet transports remove": { "actor": "none", "accepts_person": false, diff --git a/docs/st3/cli-guided-tour.md b/docs/st3/cli-guided-tour.md index 0c1b3655624..dd1f16de100 100644 --- a/docs/st3/cli-guided-tour.md +++ b/docs/st3/cli-guided-tour.md @@ -426,6 +426,11 @@ enables Fabric or moves an existing Fabric route first without changing its expo Omit `--prefer` to preserve existing transport order; see [automatic transport selection](replication.md#automatic-transport-selection) for explicit peer override precedence and the loopback-last fallback. +Use `st fleet transports prefer tailscale` to restore Tailscale-first routing, +or `st fleet transports prefer fabric` to prefer Fabric without enabling it. +Both require the carrier to be enabled and leave exposure unchanged; an already +preferred carrier is a no-op. With `--no-service`, restart both the daemon and +replication worker when instructed. ```sh st replication --help diff --git a/docs/st3/replication.md b/docs/st3/replication.md index dba5ce1f23f..08dd48b7b94 100644 --- a/docs/st3/replication.md +++ b/docs/st3/replication.md @@ -31,6 +31,12 @@ front, or to prefer an already enabled Fabric route without changing its saved protocol or exposure. An add without `--prefer` appends a new carrier and leaves an existing list unchanged. `st fleet transports remove fabric` keeps the relative order of remaining carriers and refuses to remove the last usable transport. +Use `st fleet transports prefer tailscale` or `st fleet transports prefer fabric` +to move an already enabled carrier first while preserving the remaining order. +Preference only reorders: it never exposes, unexposes, enables, or removes a +transport, and rejects a carrier absent from the list. An already-first carrier +is a no-op, including no service restart. Changes are locked, saved atomically, +and publish the existing enabled endpoints before refreshing services. These are local settings: choose the preference on each member independently. Installed services restart automatically; otherwise restart both `st3 daemon` and `st3 replication-worker` as instructed, including when using `--no-service`. diff --git a/docs/two-machines.md b/docs/two-machines.md index 19f5a8670ef..1af1afeddc5 100644 --- a/docs/two-machines.md +++ b/docs/two-machines.md @@ -78,6 +78,17 @@ or exposure. Without `--prefer`, adding Fabric appends it; repeated adds preserv the existing order. Removing it with `st fleet transports remove fabric` preserves the remaining transports' order and requires another usable transport. +To switch outbound preference back to Tailscale without disabling Fabric: + +```sh +st fleet transports prefer tailscale +``` + +Switch back with `st fleet transports prefer fabric`. Both commands require the +chosen carrier to be enabled, preserve the other carriers' order and existing +exposures, and do nothing if it is already first. They never enable, remove, +expose, or unexpose a transport. + The local `fleet.toml` transport list controls which automatic Tailscale/Fabric routes are enabled and their order for replication and peer relay. For a new fleet or joining machine, `--transports fabric,tailscale` selects Fabric first; diff --git a/release-notes/fleet-transport-prefer-enabled.json b/release-notes/fleet-transport-prefer-enabled.json new file mode 100644 index 00000000000..edc5fa2f517 --- /dev/null +++ b/release-notes/fleet-transport-prefer-enabled.json @@ -0,0 +1,32 @@ +{ + "version": 1, + "summary": "Switch between enabled Tailscale and Fabric routes with st fleet transports prefer", + "replay": { + "status": "none", + "detail": "No claim replay or projection changes. Preference publishes existing endpoints through the current fleet endpoint API." + }, + "schema": { + "status": "none", + "detail": "Reorders the existing fleet.toml transports list; no new fields or schema changes." + }, + "rules": { + "status": "none", + "detail": "Membership, trust and transport exposure authority are unchanged. Only enabled transports can be preferred." + }, + "compatibility": { + "status": "none", + "detail": "Existing readers already honor transport order. Add and remove commands retain their behavior." + }, + "downtime": { + "status": "none", + "detail": "Choosing a different preference refreshes installed daemon and replication-worker services, as existing transport changes do. An already-first transport does not restart services." + }, + "manual": { + "status": "none", + "detail": "No upgrade action required. Use st fleet transports prefer tailscale or st fleet transports prefer fabric on each member. With --no-service, restart both st3 daemon and st3 replication-worker as instructed." + }, + "recovery": { + "status": "none", + "detail": "Run the same command with the other enabled transport to restore its priority. No exposure is added or removed." + } +}