diff --git a/Cargo.lock b/Cargo.lock index b33c252041..2a7ddd5d3d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7444,6 +7444,17 @@ dependencies = [ "serde_json", ] +[[package]] +name = "skippy-topology-sim" +version = "0.76.1" +dependencies = [ + "serde", + "serde_json", + "skippy-coordinator", + "thiserror 2.0.20", + "toml 1.1.5+spec-1.1.0", +] + [[package]] name = "slab" version = "0.4.12" diff --git a/Cargo.toml b/Cargo.toml index 075b5dd7e1..9a5b35c8f2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,6 +44,7 @@ members = [ "crates/skippy-tokenizer", "crates/skippy-protocol", "crates/skippy-coordinator", + "crates/skippy-topology-sim", "crates/skippy-topology", "crates/skippy-cache", "crates/skippy-metrics", diff --git a/crates/mesh-llm-host-runtime/src/api/tests/node_state.rs b/crates/mesh-llm-host-runtime/src/api/tests/node_state.rs index aa6d9ed3f7..0c15b4abba 100644 --- a/crates/mesh-llm-host-runtime/src/api/tests/node_state.rs +++ b/crates/mesh-llm-host-runtime/src/api/tests/node_state.rs @@ -87,6 +87,7 @@ fn make_test_state_peer(seed: u8, role: mesh::NodeRole) -> mesh::PeerInfo { models: vec![], vram_bytes: 0, rtt_ms: None, + rtt_observation_window: None, model_source: None, admitted: true, serving_models: vec![], @@ -125,6 +126,7 @@ fn make_test_state_peer(seed: u8, role: mesh::NodeRole) -> mesh::PeerInfo { display_rtt: None, selected_path: None, + observed_large_frame: None, propagated_latency: None, inference_admission_state: None, } diff --git a/crates/mesh-llm-host-runtime/src/api/tests/support.rs b/crates/mesh-llm-host-runtime/src/api/tests/support.rs index 456dd7dcaa..ca136f01f1 100644 --- a/crates/mesh-llm-host-runtime/src/api/tests/support.rs +++ b/crates/mesh-llm-host-runtime/src/api/tests/support.rs @@ -618,6 +618,7 @@ fn make_test_peer( models: Vec::new(), vram_bytes: 24_000_000_000, rtt_ms: None, + rtt_observation_window: None, model_source: None, admitted: true, serving_models: serving_models.into_iter().map(str::to_string).collect(), @@ -655,6 +656,7 @@ fn make_test_peer( display_rtt: None, selected_path: None, + observed_large_frame: None, propagated_latency: None, inference_admission_state: None, } diff --git a/crates/mesh-llm-host-runtime/src/mesh/announcements.rs b/crates/mesh-llm-host-runtime/src/mesh/announcements.rs index cf44e81947..719e4a49dc 100644 --- a/crates/mesh-llm-host-runtime/src/mesh/announcements.rs +++ b/crates/mesh-llm-host-runtime/src/mesh/announcements.rs @@ -380,9 +380,33 @@ impl Node { serving_models.clear(); hosted_models.clear(); } - let advertised_model_throughput = self + let mut advertised_model_throughput = self .routing_metrics .advertisable_model_throughput(&hosted_models); + for timing in skippy_server::stage_decode_timing_hints() { + if !hosted_models.iter().any(|model| model == &timing.model_id) { + continue; + } + if let Some(existing) = advertised_model_throughput + .iter_mut() + .find(|hint| hint.model_name == timing.model_id) + { + existing.observed_stage_us_per_layer = Some(timing.observed_us_per_layer); + existing.stage_timing_samples = Some(timing.sample_count); + existing.stage_timing_age_ms = Some(timing.sample_age_ms); + } else { + advertised_model_throughput.push(crate::network::metrics::ModelThroughputHint { + model_name: timing.model_id, + avg_tokens_per_second_milli: 0, + throughput_samples: 0, + observed_stage_us_per_layer: Some(timing.observed_us_per_layer), + stage_timing_samples: Some(timing.sample_count), + stage_timing_age_ms: Some(timing.sample_age_ms), + }); + } + } + let advertised_model_throughput = + crate::network::metrics::sanitize_model_throughput_hints(advertised_model_throughput); let now_unix_ms = current_time_unix_ms(); let cache_affinity = cache_affinity_gossip::local_advertisement( &self.cache_affinity_inventory, diff --git a/crates/mesh-llm-host-runtime/src/mesh/mod.rs b/crates/mesh-llm-host-runtime/src/mesh/mod.rs index 7f26f98679..0f508c6534 100644 --- a/crates/mesh-llm-host-runtime/src/mesh/mod.rs +++ b/crates/mesh-llm-host-runtime/src/mesh/mod.rs @@ -164,7 +164,7 @@ pub use node::{ pub(crate) use node::{PeerDownReport, peer_down_endpoint_id}; pub(crate) use peer_state::{ ControlListenerLifecycle, DEAD_PEER_TTL, MeshState, PEER_DOWN_REPORTER_COOLDOWN_SECS, - PEER_STALE_SECS, resolve_peer_leaving, + PEER_STALE_SECS, RttObservationAges, resolve_peer_leaving, }; #[expect( unused_imports, diff --git a/crates/mesh-llm-host-runtime/src/mesh/node.rs b/crates/mesh-llm-host-runtime/src/mesh/node.rs index 7aef15d97d..b7889a1225 100644 --- a/crates/mesh-llm-host-runtime/src/mesh/node.rs +++ b/crates/mesh-llm-host-runtime/src/mesh/node.rs @@ -1648,6 +1648,21 @@ impl Node { let updated_peer = { let mut state = self.state.lock().await; if let Some(peer) = state.peers.get_mut(&id) { + let observed_at = std::time::Instant::now(); + match peer.rtt_observation_window.as_mut() { + Some(window) => { + window.sample_count = window.sample_count.saturating_add(1); + window.last_observed_at = observed_at; + } + None => { + peer.rtt_observation_window = + Some(crate::mesh::peer_state::RttObservationWindow { + sample_count: 1, + first_observed_at: observed_at, + last_observed_at: observed_at, + }); + } + } let prev = peer.rtt_ms; // Only accept equal-or-lower RTT for planner preference and display. // Gossip round-trip timing can inflate the value when routed via @@ -1657,14 +1672,14 @@ impl Node { // Store display_rtt regardless (for UI refresh), but don't update best RTT. peer.display_rtt = Some(DirectLatencyObservation { rtt_ms, - observed_at: std::time::Instant::now(), + observed_at, }); return; } peer.rtt_ms = Some(rtt_ms); peer.display_rtt = Some(DirectLatencyObservation { rtt_ms, - observed_at: std::time::Instant::now(), + observed_at, }); Some(peer.clone()) } else { @@ -1682,6 +1697,40 @@ impl Node { } } + /// Record a passive large-frame throughput observation for a peer link, + /// measured from a real bulk transfer (artifact download or upload). + /// Samples below the minimum duration/size floor are ignored — they + /// measure handshake jitter, not sustained throughput. Latest-wins. + pub(crate) async fn record_peer_large_frame_observation( + &self, + id: EndpointId, + bytes: u64, + elapsed: std::time::Duration, + ) { + const MIN_SAMPLE_BYTES: u64 = 512 * 1024; + const MIN_SAMPLE_DURATION: std::time::Duration = std::time::Duration::from_millis(100); + if bytes < MIN_SAMPLE_BYTES || elapsed < MIN_SAMPLE_DURATION { + return; + } + let elapsed_micros = elapsed.as_micros().max(1); + let mib_per_s = ((u128::from(bytes) * 1_000_000 / elapsed_micros) / 1_048_576) as u32; + if mib_per_s == 0 { + return; + } + let mut state = self.state.lock().await; + if let Some(peer) = state.peers.get_mut(&id) { + peer.observed_large_frame = Some(LargeFrameObservation { + mib_per_s, + observed_at: std::time::Instant::now(), + }); + tracing::debug!( + "Peer {} large-frame: {mib_per_s} MiB/s over {} bytes in {elapsed:?}", + id.fmt_short(), + bytes + ); + } + } + pub(crate) async fn update_peer_selected_path( &self, id: EndpointId, diff --git a/crates/mesh-llm-host-runtime/src/mesh/peer_state.rs b/crates/mesh-llm-host-runtime/src/mesh/peer_state.rs index a07b27caa0..77f55f16ba 100644 --- a/crates/mesh-llm-host-runtime/src/mesh/peer_state.rs +++ b/crates/mesh-llm-host-runtime/src/mesh/peer_state.rs @@ -175,6 +175,20 @@ pub struct DirectLatencyObservation { pub observed_at: std::time::Instant, } +/// A large-frame throughput observation on a peer link, measured passively +/// from a real artifact transfer (bulk bytes over the same QUIC transport +/// the split pipeline uses). Latest-wins; `observed_at` lets consumers +/// discard stale samples. +#[derive(Debug, Clone)] +pub struct LargeFrameObservation { + pub mib_per_s: u32, + pub observed_at: std::time::Instant, +} + +/// How long a passive large-frame observation stays planner-relevant. +pub const LARGE_FRAME_OBSERVATION_MAX_AGE: std::time::Duration = + std::time::Duration::from_secs(30 * 60); + /// Latency propagated via transitive gossip (not measured directly). #[derive(Debug, Clone, PartialEq, Eq)] pub struct PropagatedLatencyObservation { @@ -201,6 +215,26 @@ pub struct DisplayLatency { pub observer_id: Option, } +/// Confidence metadata retained behind the planner's best-seen RTT floor. +/// +/// This intentionally records only observation count and timing. It is not a +/// variance or jitter estimate: iroh owns path-quality selection, while split +/// placement only needs to distinguish a one-off early sample from a floor +/// corroborated across the connection-settle window. +#[derive(Clone, Copy, Debug)] +pub(crate) struct RttObservationWindow { + pub(crate) sample_count: u32, + pub(crate) first_observed_at: std::time::Instant, + pub(crate) last_observed_at: std::time::Instant, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct RttObservationAges { + pub(crate) sample_count: u32, + pub(crate) first_sample_age_ms: u64, + pub(crate) last_sample_age_ms: u64, +} + #[derive(Debug, Clone)] pub struct PeerInfo { pub id: EndpointId, @@ -213,6 +247,9 @@ pub struct PeerInfo { pub models: Vec, pub vram_bytes: u64, pub rtt_ms: Option, + /// Observation confidence for `rtt_ms`, whose value remains the minimum + /// accepted sample. Higher samples still advance this window. + pub(crate) rtt_observation_window: Option, pub model_source: Option, pub admitted: bool, /// All models assigned to this peer, even if not yet healthy. @@ -269,6 +306,11 @@ pub struct PeerInfo { pub display_rtt: Option, /// Last selected path observed on the mesh control connection to this peer. pub(crate) selected_path: Option, + /// Latest large-frame throughput observed on this peer's link, measured + /// passively from real artifact transfers (either direction). Used as the + /// edge bandwidth signal for topology planning; stale observations decay + /// to `None` (latency-only edge). + pub(crate) observed_large_frame: Option, /// Latency propagated via transitive gossip. pub propagated_latency: Option, pub owner_summary: OwnershipSummary, @@ -320,6 +362,7 @@ impl PeerInfo { models: ann.models.clone(), vram_bytes: ann.vram_bytes, rtt_ms: None, + rtt_observation_window: None, model_source: ann.model_source.clone(), admitted: false, serving_models: ann.serving_models.clone(), @@ -358,6 +401,7 @@ impl PeerInfo { cache_affinity: ann.cache_affinity.clone(), display_rtt: None, selected_path: None, + observed_large_frame: None, propagated_latency: None, owner_summary, inference_admission_state: ann.inference_admission_state, @@ -373,6 +417,26 @@ impl PeerInfo { self.display_rtt.as_ref().map(|d| d.rtt_ms).or(self.rtt_ms) } + pub(crate) fn rtt_observation_ages(&self) -> Option { + let window = self.rtt_observation_window?; + Some(RttObservationAges { + sample_count: window.sample_count, + first_sample_age_ms: super::elapsed_ms_u64(window.first_observed_at.elapsed()), + last_sample_age_ms: super::elapsed_ms_u64(window.last_observed_at.elapsed()), + }) + } + + /// Sustained large-frame throughput for this peer link from a recent + /// passive artifact-transfer observation. `None` when never measured or + /// the observation has aged out — the planner then treats the edge as + /// latency-only, which is exactly the pre-probing behavior. + pub fn large_frame_mib_per_s(&self) -> Option { + self.observed_large_frame + .as_ref() + .filter(|observed| observed.observed_at.elapsed() <= LARGE_FRAME_OBSERVATION_MAX_AGE) + .map(|observed| observed.mib_per_s) + } + pub(crate) fn split_stage_path_fallback(&self) -> Option { let observation = self.selected_path?; if observation.path_type != "direct" { @@ -893,6 +957,28 @@ impl Node { self.vram_bytes } + /// Measured sustained node performance for split planning: summed + /// memory bandwidth in MiB/s and fp16 compute in GFLOP/s across GPUs. + /// `(None, None)` until gpu-bench measurements populate the metrics — + /// the planner treats unreported nodes as capacity-only. + pub async fn sustained_perf_signals(&self) -> (Option, Option) { + let bandwidth = { + let metrics = self.gpu_mem_bandwidth_gbps.lock().await; + metrics.as_ref().map(|values| values.iter().sum::()) + }; + let compute = { + let metrics = self.gpu_compute_tflops_fp16.lock().await; + metrics.as_ref().map(|values| values.iter().sum::()) + }; + let bandwidth_mib = bandwidth + .map(|gbps| gbps * 1_000_000_000.0 / 1_048_576.0) + .and_then(|mib| u32::try_from(mib.max(0.0) as u64).ok()); + let compute_gflops = compute + .map(|tflops| tflops * 1_000.0) + .and_then(|gflops| u32::try_from(gflops.max(0.0) as u64).ok()); + (bandwidth_mib, compute_gflops) + } + /// Local model-fit budget, including supported CPU offload memory. pub fn local_runtime_capacity_bytes(&self) -> u64 { self.local_runtime_capacity_bytes diff --git a/crates/mesh-llm-host-runtime/src/mesh/stage_artifacts.rs b/crates/mesh-llm-host-runtime/src/mesh/stage_artifacts.rs index ddb1cb562e..96d5308416 100644 --- a/crates/mesh-llm-host-runtime/src/mesh/stage_artifacts.rs +++ b/crates/mesh-llm-host-runtime/src/mesh/stage_artifacts.rs @@ -691,6 +691,7 @@ impl Node { ); let transfer_result = async { + let transfer_started = std::time::Instant::now(); append_artifact_transfer_body( &mut recv, &temp_path, @@ -700,6 +701,15 @@ impl Node { ARTIFACT_TRANSFER_READ_IDLE_TIMEOUT, ) .await?; + // Passive edge-bandwidth measurement: this transfer moved real + // bulk bytes over the same QUIC transport the split pipeline + // uses. Record it as the peer link's large-frame observation. + self.record_peer_large_frame_observation( + peer_id, + response.total_size.saturating_sub(offset), + transfer_started.elapsed(), + ) + .await; let actual_size = tokio::fs::metadata(&temp_path) .await @@ -967,6 +977,7 @@ impl Node { .context("seek artifact for transfer")?; let mut buffer = vec![0u8; ARTIFACT_TRANSFER_BUFFER_BYTES]; let mut remaining = artifact.size.saturating_sub(request.offset); + let upload_started = std::time::Instant::now(); while remaining > 0 { let limit = buffer.len().min(remaining as usize); let read = file @@ -982,6 +993,14 @@ impl Node { .await?; remaining -= read as u64; } + // Passive edge-bandwidth measurement, upload direction: see + // fetch_artifact_from_peer for the rationale. + self.record_peer_large_frame_observation( + remote, + artifact.size.saturating_sub(request.offset), + upload_started.elapsed(), + ) + .await; let _ = send.finish(); Ok(()) } diff --git a/crates/mesh-llm-host-runtime/src/mesh/tests/admission/connectivity.rs b/crates/mesh-llm-host-runtime/src/mesh/tests/admission/connectivity.rs index c5aeaeb676..ce394d7d76 100644 --- a/crates/mesh-llm-host-runtime/src/mesh/tests/admission/connectivity.rs +++ b/crates/mesh-llm-host-runtime/src/mesh/tests/admission/connectivity.rs @@ -66,16 +66,31 @@ async fn test_rtt_cannot_regress() -> Result<()> { node.update_peer_rtt(peer_id, 2600).await; { let state = node.state.lock().await; - let rtt = state.peers.get(&peer_id).unwrap().rtt_ms; + let peer = state.peers.get(&peer_id).unwrap(); + let rtt = peer.rtt_ms; assert_eq!(rtt, Some(20), "RTT must not increase from 20 to 2600"); + assert_eq!( + peer.rtt_observation_window + .as_ref() + .map(|window| window.sample_count), + Some(1), + "higher samples must still contribute corroboration evidence" + ); } // Lower RTT — should be accepted node.update_peer_rtt(peer_id, 10).await; { let state = node.state.lock().await; - let rtt = state.peers.get(&peer_id).unwrap().rtt_ms; + let peer = state.peers.get(&peer_id).unwrap(); + let rtt = peer.rtt_ms; assert_eq!(rtt, Some(10), "RTT must decrease from 20 to 10"); + assert_eq!( + peer.rtt_observation_window + .as_ref() + .map(|window| window.sample_count), + Some(2) + ); } Ok(()) diff --git a/crates/mesh-llm-host-runtime/src/mesh/tests/admission/helpers.rs b/crates/mesh-llm-host-runtime/src/mesh/tests/admission/helpers.rs index 6500b10cc1..4e348c7e2d 100644 --- a/crates/mesh-llm-host-runtime/src/mesh/tests/admission/helpers.rs +++ b/crates/mesh-llm-host-runtime/src/mesh/tests/admission/helpers.rs @@ -13,6 +13,7 @@ pub(super) fn make_test_peer(id: EndpointId, rtt_ms: Option, vram_gb: u64) models: vec![], vram_bytes: vram_gb * 1024 * 1024 * 1024, rtt_ms, + rtt_observation_window: None, model_source: None, admitted: true, serving_models: vec![], @@ -50,6 +51,7 @@ pub(super) fn make_test_peer(id: EndpointId, rtt_ms: Option, vram_gb: u64) display_rtt: None, selected_path: None, + observed_large_frame: None, propagated_latency: None, inference_admission_state: None, } diff --git a/crates/mesh-llm-host-runtime/src/mesh/tests/gossip/merge_and_refresh.rs b/crates/mesh-llm-host-runtime/src/mesh/tests/gossip/merge_and_refresh.rs index d2a43f714a..df1e1c7cb1 100644 --- a/crates/mesh-llm-host-runtime/src/mesh/tests/gossip/merge_and_refresh.rs +++ b/crates/mesh-llm-host-runtime/src/mesh/tests/gossip/merge_and_refresh.rs @@ -326,6 +326,9 @@ pub(crate) fn test_apply_transitive_ann_refreshes_advertised_model_throughput() model_name: "qwen".to_string(), avg_tokens_per_second_milli: 35_000, throughput_samples: 4, + observed_stage_us_per_layer: None, + stage_timing_samples: None, + stage_timing_age_ms: None, }]; apply_transitive_ann( @@ -419,6 +422,9 @@ pub(crate) async fn test_add_peer_refreshes_advertised_model_throughput() { model_name: "qwen".to_string(), avg_tokens_per_second_milli: 20_000, throughput_samples: 2, + observed_stage_us_per_layer: None, + stage_timing_samples: None, + stage_timing_age_ms: None, }]; node.add_peer(peer_id, addr.clone(), &ann, None).await; diff --git a/crates/mesh-llm-host-runtime/src/mesh/tests/owner_control.rs b/crates/mesh-llm-host-runtime/src/mesh/tests/owner_control.rs index 4dfea23a9a..3333d4a6e9 100644 --- a/crates/mesh-llm-host-runtime/src/mesh/tests/owner_control.rs +++ b/crates/mesh-llm-host-runtime/src/mesh/tests/owner_control.rs @@ -13,6 +13,7 @@ fn make_test_peer_info(peer_id: EndpointId) -> PeerInfo { models: vec![], vram_bytes: 0, rtt_ms: None, + rtt_observation_window: None, model_source: None, admitted: true, serving_models: vec![], @@ -56,6 +57,7 @@ fn make_test_peer_info(peer_id: EndpointId) -> PeerInfo { display_rtt: None, selected_path: None, + observed_large_frame: None, propagated_latency: None, } } diff --git a/crates/mesh-llm-host-runtime/src/network/metrics.rs b/crates/mesh-llm-host-runtime/src/network/metrics.rs index 52f1b62d78..dbcf7130be 100644 --- a/crates/mesh-llm-host-runtime/src/network/metrics.rs +++ b/crates/mesh-llm-host-runtime/src/network/metrics.rs @@ -18,6 +18,8 @@ pub(crate) const MAX_ADVERTISED_MODEL_THROUGHPUT_HINTS: usize = 64; pub(crate) const MAX_ADVERTISED_MODEL_NAME_BYTES: usize = 256; pub(crate) const MAX_ADVERTISED_TPS_MILLI: u64 = 100_000 * THROUGHPUT_SCALE_MILLI; pub(crate) const MAX_ADVERTISED_THROUGHPUT_SAMPLES: u64 = 256; +pub(crate) const MAX_ADVERTISED_STAGE_US_PER_LAYER: u64 = 10_000_000; +pub(crate) const MAX_ADVERTISED_STAGE_TIMING_AGE_MS: u64 = 30 * 60 * 1_000; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum MetricLayer { @@ -215,16 +217,23 @@ pub(crate) struct RoutingCollectorSnapshot { pub models: HashMap, } -/// Soft peer-advertised model throughput hint. +/// Soft peer-advertised model performance hint. /// /// Values are fixed-point milli tokens/second to keep gossip deterministic and -/// avoid protobuf floating-point edge cases. They are advisory only; routing -/// clamps and local observations take precedence. +/// avoid protobuf floating-point edge cases. Staged runtimes can additionally +/// attach observed steady-decode work normalized per loaded layer; placement +/// uses that as a measured floor on the analytical stage model. #[derive(Clone, Debug, PartialEq, Eq, Serialize)] pub(crate) struct ModelThroughputHint { pub(crate) model_name: String, pub(crate) avg_tokens_per_second_milli: u64, pub(crate) throughput_samples: u64, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) observed_stage_us_per_layer: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) stage_timing_samples: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) stage_timing_age_ms: Option, } pub(crate) fn sanitize_model_throughput_hints(hints: I) -> Vec @@ -235,10 +244,17 @@ where let mut sanitized = Vec::new(); for mut hint in hints { hint.model_name = hint.model_name.trim().to_string(); + let throughput_valid = hint.avg_tokens_per_second_milli > 0 && hint.throughput_samples > 0; + let stage_timing_valid = hint + .observed_stage_us_per_layer + .is_some_and(|value| value > 0) + && hint.stage_timing_samples.is_some_and(|samples| samples > 0) + && hint + .stage_timing_age_ms + .is_some_and(|age| age <= MAX_ADVERTISED_STAGE_TIMING_AGE_MS); if hint.model_name.is_empty() || hint.model_name.len() > MAX_ADVERTISED_MODEL_NAME_BYTES - || hint.avg_tokens_per_second_milli == 0 - || hint.throughput_samples == 0 + || (!throughput_valid && !stage_timing_valid) || !seen.insert(hint.model_name.clone()) { continue; @@ -249,6 +265,18 @@ where hint.throughput_samples = hint .throughput_samples .min(MAX_ADVERTISED_THROUGHPUT_SAMPLES); + if stage_timing_valid { + hint.observed_stage_us_per_layer = hint + .observed_stage_us_per_layer + .map(|value| value.min(MAX_ADVERTISED_STAGE_US_PER_LAYER)); + hint.stage_timing_samples = hint + .stage_timing_samples + .map(|samples| samples.min(MAX_ADVERTISED_THROUGHPUT_SAMPLES)); + } else { + hint.observed_stage_us_per_layer = None; + hint.stage_timing_samples = None; + hint.stage_timing_age_ms = None; + } sanitized.push(hint); if sanitized.len() >= MAX_ADVERTISED_MODEL_THROUGHPUT_HINTS { break; @@ -546,6 +574,9 @@ impl RoutingMetrics { avg_tokens_per_second_milli: avg_tokens_per_second_milli .min(MAX_ADVERTISED_TPS_MILLI), throughput_samples: samples.min(MAX_ADVERTISED_THROUGHPUT_SAMPLES), + observed_stage_us_per_layer: None, + stage_timing_samples: None, + stage_timing_age_ms: None, }); if hints.len() >= MAX_ADVERTISED_MODEL_THROUGHPUT_HINTS { break; @@ -577,6 +608,9 @@ impl RoutingMetrics { model_name: model.to_string(), avg_tokens_per_second_milli, throughput_samples: samples, + observed_stage_us_per_layer: None, + stage_timing_samples: None, + stage_timing_age_ms: None, }) } @@ -1605,35 +1639,61 @@ mod tests { model_name: " qwen ".to_string(), avg_tokens_per_second_milli: MAX_ADVERTISED_TPS_MILLI + 1, throughput_samples: MAX_ADVERTISED_THROUGHPUT_SAMPLES + 1, + observed_stage_us_per_layer: Some(MAX_ADVERTISED_STAGE_US_PER_LAYER + 1), + stage_timing_samples: Some(MAX_ADVERTISED_THROUGHPUT_SAMPLES + 1), + stage_timing_age_ms: Some(MAX_ADVERTISED_STAGE_TIMING_AGE_MS + 1), }, ModelThroughputHint { model_name: "qwen".to_string(), avg_tokens_per_second_milli: 42_000, throughput_samples: 7, + observed_stage_us_per_layer: None, + stage_timing_samples: None, + stage_timing_age_ms: None, }, ModelThroughputHint { model_name: "".to_string(), avg_tokens_per_second_milli: 42_000, throughput_samples: 7, + observed_stage_us_per_layer: None, + stage_timing_samples: None, + stage_timing_age_ms: None, }, ModelThroughputHint { model_name: "x".repeat(MAX_ADVERTISED_MODEL_NAME_BYTES + 1), avg_tokens_per_second_milli: 42_000, throughput_samples: 7, + observed_stage_us_per_layer: None, + stage_timing_samples: None, + stage_timing_age_ms: None, }, ModelThroughputHint { model_name: "empty-speed".to_string(), avg_tokens_per_second_milli: 0, throughput_samples: 7, + observed_stage_us_per_layer: None, + stage_timing_samples: None, + stage_timing_age_ms: None, }, ModelThroughputHint { model_name: "empty-samples".to_string(), avg_tokens_per_second_milli: 42_000, throughput_samples: 0, + observed_stage_us_per_layer: None, + stage_timing_samples: None, + stage_timing_age_ms: None, + }, + ModelThroughputHint { + model_name: "timing-only".to_string(), + avg_tokens_per_second_milli: 0, + throughput_samples: 0, + observed_stage_us_per_layer: Some(2_500), + stage_timing_samples: Some(12), + stage_timing_age_ms: Some(500), }, ]); - assert_eq!(hints.len(), 1); + assert_eq!(hints.len(), 2); assert_eq!(hints[0].model_name, "qwen"); assert_eq!( hints[0].avg_tokens_per_second_milli, @@ -1643,6 +1703,8 @@ mod tests { hints[0].throughput_samples, MAX_ADVERTISED_THROUGHPUT_SAMPLES ); + assert_eq!(hints[1].model_name, "timing-only"); + assert_eq!(hints[1].observed_stage_us_per_layer, Some(2_500)); } #[test] diff --git a/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/automatic_routing.rs b/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/automatic_routing.rs index 76be59f11e..52f92d2247 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/automatic_routing.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/automatic_routing.rs @@ -338,6 +338,7 @@ fn peer_serving(peer_id: iroh::EndpointId, model: &str, vision: bool) -> mesh::P models: vec![model.to_string()], vram_bytes: 16 * 1024 * 1024 * 1024, rtt_ms: None, + rtt_observation_window: None, model_source: None, admitted: true, serving_models: vec![model.to_string()], @@ -373,6 +374,7 @@ fn peer_serving(peer_id: iroh::EndpointId, model: &str, vision: bool) -> mesh::P cache_affinity: None, display_rtt: None, selected_path: None, + observed_large_frame: None, propagated_latency: None, owner_summary: crate::crypto::OwnershipSummary::default(), inference_admission_state: None, diff --git a/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/tests.rs b/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/tests.rs index d5275b2da7..8d5748faaa 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/tests.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/tests.rs @@ -962,6 +962,7 @@ fn test_remote_peer(seed: u32, model: &str) -> mesh::PeerInfo { models: vec![model.to_string()], vram_bytes: 16 * 1024 * 1024 * 1024, rtt_ms: None, + rtt_observation_window: None, model_source: None, // admitted: true required for `is_admitted()`. admitted: true, @@ -998,6 +999,7 @@ fn test_remote_peer(seed: u32, model: &str) -> mesh::PeerInfo { cache_affinity: None, display_rtt: None, selected_path: None, + observed_large_frame: None, propagated_latency: None, owner_summary: crate::crypto::OwnershipSummary::default(), inference_admission_state: None, diff --git a/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/fleet_fairness_tests.rs b/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/fleet_fairness_tests.rs index 3a8a1eb0df..e5d5e96e85 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/fleet_fairness_tests.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/fleet_fairness_tests.rs @@ -188,6 +188,9 @@ async fn inference_load_is_invisible_to_replica_choice() { model_name: BIG_MODELS[0].name.to_string(), avg_tokens_per_second_milli: 1_000, throughput_samples: 64, + observed_stage_us_per_layer: None, + stage_timing_samples: None, + stage_timing_age_ms: None, }]; node.insert_test_peer(hobbled).await; diff --git a/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/fleet_sim_tests.rs b/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/fleet_sim_tests.rs index 85e86a7b5d..314d7ebf79 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/fleet_sim_tests.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/fleet_sim_tests.rs @@ -86,6 +86,7 @@ pub(super) fn fleet_peer(seed: u32, model: FleetModel) -> mesh::PeerInfo { models: vec![model.name.to_string()], vram_bytes: 0, rtt_ms: None, + rtt_observation_window: None, model_source: None, admitted: true, serving_models: vec![model.name.to_string()], @@ -147,6 +148,7 @@ pub(super) fn fleet_peer(seed: u32, model: FleetModel) -> mesh::PeerInfo { inference_admission_state: None, display_rtt: None, selected_path: None, + observed_large_frame: None, propagated_latency: None, } } @@ -185,6 +187,9 @@ pub(super) fn fleet_peer_with_health( model_name: model.name.to_string(), avg_tokens_per_second_milli, throughput_samples: 8, + observed_stage_us_per_layer: None, + stage_timing_samples: None, + stage_timing_age_ms: None, }]; } peer @@ -798,6 +803,9 @@ async fn tool_capability_outranks_advertised_throughput_for_the_acting_model() { model_name: BIG_MODELS[0].name.to_string(), avg_tokens_per_second_milli: 1_000, throughput_samples: 8, + observed_stage_us_per_layer: None, + stage_timing_samples: None, + stage_timing_age_ms: None, }]; let mut fast_non_caller = fleet_peer_with_tool_use(2, BIG_MODELS[1], CapabilityLevel::None, None); @@ -806,6 +814,9 @@ async fn tool_capability_outranks_advertised_throughput_for_the_acting_model() { model_name: BIG_MODELS[1].name.to_string(), avg_tokens_per_second_milli: 90_000, throughput_samples: 8, + observed_stage_us_per_layer: None, + stage_timing_samples: None, + stage_timing_age_ms: None, }]; node.insert_test_peer(slow_tool_caller).await; node.insert_test_peer(fast_non_caller).await; diff --git a/crates/mesh-llm-host-runtime/src/network/openai/transport_tests/routing.rs b/crates/mesh-llm-host-runtime/src/network/openai/transport_tests/routing.rs index 7b6a8082b6..a8e2ed304c 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/transport_tests/routing.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/transport_tests/routing.rs @@ -80,6 +80,7 @@ fn test_peer_serving_model(peer_id: iroh::EndpointId, model: &str) -> mesh::Peer models: vec![model.to_string()], vram_bytes: 16 * 1024 * 1024 * 1024, rtt_ms: None, + rtt_observation_window: None, model_source: None, admitted: true, serving_models: vec![model.to_string()], @@ -115,6 +116,7 @@ fn test_peer_serving_model(peer_id: iroh::EndpointId, model: &str) -> mesh::Peer cache_affinity: None, display_rtt: None, selected_path: None, + observed_large_frame: None, propagated_latency: None, owner_summary: crate::crypto::OwnershipSummary::default(), inference_admission_state: None, diff --git a/crates/mesh-llm-host-runtime/src/protocol/convert.rs b/crates/mesh-llm-host-runtime/src/protocol/convert.rs index 6bcd4be951..230e88dbde 100644 --- a/crates/mesh-llm-host-runtime/src/protocol/convert.rs +++ b/crates/mesh-llm-host-runtime/src/protocol/convert.rs @@ -644,6 +644,9 @@ fn local_throughput_hint_to_proto( model_name: hint.model_name.clone(), avg_tokens_per_second_milli: hint.avg_tokens_per_second_milli, throughput_samples: hint.throughput_samples, + observed_stage_us_per_layer: hint.observed_stage_us_per_layer, + stage_timing_samples: hint.stage_timing_samples, + stage_timing_age_ms: hint.stage_timing_age_ms, } } @@ -654,6 +657,9 @@ fn proto_throughput_hint_to_local( model_name: hint.model_name.clone(), avg_tokens_per_second_milli: hint.avg_tokens_per_second_milli, throughput_samples: hint.throughput_samples, + observed_stage_us_per_layer: hint.observed_stage_us_per_layer, + stage_timing_samples: hint.stage_timing_samples, + stage_timing_age_ms: hint.stage_timing_age_ms, } } diff --git a/crates/mesh-llm-host-runtime/src/protocol/tests.rs b/crates/mesh-llm-host-runtime/src/protocol/tests.rs index 1e6b6174f2..17b0570f54 100644 --- a/crates/mesh-llm-host-runtime/src/protocol/tests.rs +++ b/crates/mesh-llm-host-runtime/src/protocol/tests.rs @@ -148,6 +148,7 @@ fn make_test_peer_info(peer_id: EndpointId) -> PeerInfo { models: vec![], vram_bytes: 0, rtt_ms: None, + rtt_observation_window: None, model_source: None, admitted: true, serving_models: vec![], @@ -185,6 +186,7 @@ fn make_test_peer_info(peer_id: EndpointId) -> PeerInfo { display_rtt: None, selected_path: None, + observed_large_frame: None, propagated_latency: None, inference_admission_state: None, } diff --git a/crates/mesh-llm-host-runtime/src/protocol/tests/announcements.rs b/crates/mesh-llm-host-runtime/src/protocol/tests/announcements.rs index c337269d5c..5bb1a36229 100644 --- a/crates/mesh-llm-host-runtime/src/protocol/tests/announcements.rs +++ b/crates/mesh-llm-host-runtime/src/protocol/tests/announcements.rs @@ -141,6 +141,9 @@ fn advertised_model_throughput_roundtrips_through_proto_announcement() { model_name: "qwen".to_string(), avg_tokens_per_second_milli: 42_000, throughput_samples: 7, + observed_stage_us_per_layer: Some(1_250), + stage_timing_samples: Some(11), + stage_timing_age_ms: Some(250), }]; let salt = [0xC3; mesh_llm_routing::cache_inventory::CACHE_AFFINITY_SALT_BYTES]; let prefix_hash = 0xfeed_beef; @@ -191,6 +194,9 @@ fn advertised_model_throughput_roundtrips_through_proto_announcement() { model_name: "ghost".to_string(), avg_tokens_per_second_milli: 250_000, throughput_samples: 99, + observed_stage_us_per_layer: None, + stage_timing_samples: None, + stage_timing_age_ms: None, }, ], cache_affinity: Some( @@ -255,12 +261,27 @@ fn advertised_model_throughput_roundtrips_through_proto_announcement() { proto_pa.advertised_model_throughput[0].throughput_samples, 7 ); + assert_eq!( + proto_pa.advertised_model_throughput[0].observed_stage_us_per_layer, + Some(1_250) + ); + assert_eq!( + proto_pa.advertised_model_throughput[0].stage_timing_samples, + Some(11) + ); + assert_eq!( + proto_pa.advertised_model_throughput[0].stage_timing_age_ms, + Some(250) + ); proto_pa .advertised_model_throughput .push(crate::proto::node::AdvertisedModelThroughput { model_name: "ghost".to_string(), avg_tokens_per_second_milli: 250_000, throughput_samples: 99, + observed_stage_us_per_layer: None, + stage_timing_samples: None, + stage_timing_age_ms: None, }); let (_, roundtripped) = proto_ann_to_local(&proto_pa).expect("proto_ann_to_local must succeed"); diff --git a/crates/mesh-llm-host-runtime/src/runtime/local_package.rs b/crates/mesh-llm-host-runtime/src/runtime/local_package.rs index 4ed315437a..426cb09a70 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/local_package.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/local_package.rs @@ -12,6 +12,16 @@ use std::collections::BTreeMap; use std::path::Path; pub(super) const SPLIT_DEFAULT_MIN_PARTICIPANTS: usize = 2; +const MAX_GOSSIP_GPU_METRIC_VALUES: usize = 16; +const MAX_GOSSIP_GPU_METRIC_CSV_BYTES: usize = 1_024; +const MAX_GOSSIP_MEM_BANDWIDTH_GBPS_PER_DEVICE: f64 = 10_000.0; +const MAX_GOSSIP_COMPUTE_TFLOPS_PER_DEVICE: f64 = 10_000.0; +const MIN_TRUSTED_STAGE_TIMING_SAMPLES: u64 = 8; +const MAX_TRUSTED_STAGE_TIMING_AGE_MS: u64 = 2 * 60 * 1_000; +const SIGNATURE_LINK_BANDWIDTH_BUCKET_MIB_PER_S: u32 = 5; +const SIGNATURE_MEM_BANDWIDTH_BUCKET_MIB_PER_S: u32 = 5_000; +const SIGNATURE_COMPUTE_BUCKET_GFLOP_PER_S: u32 = 1_000; +const SIGNATURE_STAGE_TIMING_BUCKET_US_PER_LAYER: u64 = 100; /// Try to extract GGUF architecture metadata from a layer package's shared /// metadata file. Layer packages store a `shared/metadata.gguf` that carries @@ -271,7 +281,24 @@ pub(super) struct SplitParticipantBlockerSummary { recommendation: &'static str, } -type SplitParticipantSignature = Vec<(String, u64, u64, u64, Option, bool, u32)>; +type SplitParticipantSignature = Vec<( + String, + u64, + u64, + u64, + Option, + Option, + bool, + u32, + Option, + Option, + Option, + bool, +)>; + +const SPLIT_RTT_CORROBORATION_MIN_SAMPLES: u32 = 2; +const SPLIT_RTT_CORROBORATION_MIN_SPAN_MS: u64 = 5_000; +const SPLIT_RTT_CORROBORATION_MAX_LAST_AGE_MS: u64 = 30_000; #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(super) struct SplitParticipant { @@ -281,8 +308,24 @@ pub(super) struct SplitParticipant { pub(super) cached_slice_bytes: u64, pub(super) missing_artifact_bytes: u64, pub(super) rtt_ms: Option, + pub(super) rtt_sample_count: u32, + pub(super) rtt_first_sample_age_ms: Option, + pub(super) rtt_last_sample_age_ms: Option, + pub(super) rtt_corroborated: bool, + /// Sustained large-frame throughput to this peer, MiB/s, from passive + /// artifact-transfer observation. `None` until measured (or aged out) — + /// edges to this peer stay latency-only. + pub(super) large_frame_mib_per_s: Option, pub(super) artifact_transfer_supported: bool, availability_score: u32, + /// Sustained memory bandwidth in MiB/s, summed across GPUs (gpu-bench, + /// gossip). `None` until measured and advertised. + pub(super) sustained_mem_bandwidth_mib_per_s: Option, + /// Sustained fp16 compute in GFLOP/s, summed across GPUs. + pub(super) sustained_compute_gflop_per_s: Option, + /// Observed steady-decode runtime work normalized per loaded layer. + /// This is a measured floor for the analytical weight-streaming model. + pub(super) observed_decode_us_per_layer: Option, } impl SplitParticipant { @@ -298,8 +341,16 @@ impl SplitParticipant { cached_slice_bytes: 0, missing_artifact_bytes: 0, rtt_ms: None, + rtt_sample_count: 0, + rtt_first_sample_age_ms: None, + rtt_last_sample_age_ms: None, + rtt_corroborated: false, + large_frame_mib_per_s: None, artifact_transfer_supported: false, availability_score: 0, + sustained_mem_bandwidth_mib_per_s: None, + sustained_compute_gflop_per_s: None, + observed_decode_us_per_layer: None, } } @@ -321,12 +372,62 @@ impl SplitParticipant { signal: SplitParticipantPackageSignal, rtt_ms: Option, artifact_transfer_supported: bool, + perf: SplitParticipantPerf, ) -> Self { self.cached_slice_bytes = signal.cached_slice_bytes; self.missing_artifact_bytes = signal.missing_artifact_bytes; self.availability_score = signal.availability_score; self.rtt_ms = rtt_ms; self.artifact_transfer_supported = artifact_transfer_supported; + self.sustained_mem_bandwidth_mib_per_s = perf.sustained_mem_bandwidth_mib_per_s; + self.sustained_compute_gflop_per_s = perf.sustained_compute_gflop_per_s; + self.observed_decode_us_per_layer = perf.observed_decode_us_per_layer; + self + } + + /// Attach the passively observed large-frame throughput for this peer + /// link (MiB/s), from artifact-transfer measurement. + pub(super) fn with_edge_bandwidth(mut self, mib_per_s: Option) -> Self { + self.large_frame_mib_per_s = mib_per_s; + self + } + + /// Attach settle-time confidence for the best-seen RTT floor. + /// + /// Two observations must span the post-connect direct-path recheck window, + /// and the latest one must still be recent. Until then, this remote node's + /// performance signals are withheld so the planner reuses its existing + /// capacity-only candidate fallback. + pub(super) fn with_rtt_observation( + mut self, + observation: Option, + ) -> Self { + if let Some(observation) = observation { + self.rtt_sample_count = observation.sample_count; + self.rtt_first_sample_age_ms = Some(observation.first_sample_age_ms); + self.rtt_last_sample_age_ms = Some(observation.last_sample_age_ms); + let observed_span_ms = observation + .first_sample_age_ms + .saturating_sub(observation.last_sample_age_ms); + self.rtt_corroborated = observation.sample_count >= SPLIT_RTT_CORROBORATION_MIN_SAMPLES + && observed_span_ms >= SPLIT_RTT_CORROBORATION_MIN_SPAN_MS + && observation.last_sample_age_ms <= SPLIT_RTT_CORROBORATION_MAX_LAST_AGE_MS; + } + if !self.rtt_corroborated { + self.rtt_ms = None; + self.large_frame_mib_per_s = None; + self.sustained_mem_bandwidth_mib_per_s = None; + self.sustained_compute_gflop_per_s = None; + self.observed_decode_us_per_layer = None; + } + self + } + + /// Attach measured performance signals to the local node's participant. + pub(super) fn with_local_perf(mut self, perf: SplitParticipantPerf) -> Self { + self.sustained_mem_bandwidth_mib_per_s = perf.sustained_mem_bandwidth_mib_per_s; + self.sustained_compute_gflop_per_s = perf.sustained_compute_gflop_per_s; + self.observed_decode_us_per_layer = perf.observed_decode_us_per_layer; self } @@ -351,6 +452,89 @@ pub(super) struct SplitParticipantPackageSignal { pub(super) availability_score: u32, } +/// Measured node performance signals carried into split planning. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub(super) struct SplitParticipantPerf { + /// Sustained memory bandwidth in MiB/s, summed across GPUs. + pub(super) sustained_mem_bandwidth_mib_per_s: Option, + /// Sustained fp16 compute in GFLOP/s, summed across GPUs. + pub(super) sustained_compute_gflop_per_s: Option, + /// Observed steady-decode runtime work in microseconds per loaded layer. + pub(super) observed_decode_us_per_layer: Option, +} + +impl SplitParticipantPerf { + /// Parse the gossiped CSV metric fields (`"1948.7,2100.1"`) into summed + /// integer MiB/s and GFLOP/s. Gossip reports GB/s and TFLOP/s; bandwidth + /// is converted to MiB/s (1 GB/s = 953.674 MiB/s) and compute to GFLOP/s. + /// `None` when unreported or unparsable — the planner treats missing + /// signals as capacity-only. + pub(super) fn from_gossip_csvs( + mem_bandwidth_gbps: Option<&str>, + compute_tflops_fp16: Option<&str>, + ) -> Self { + let bandwidth_mib_per_s = parse_bounded_metric_csv_sum( + mem_bandwidth_gbps, + MAX_GOSSIP_MEM_BANDWIDTH_GBPS_PER_DEVICE, + ) + .map(|gbps| gbps * 1_000_000_000.0 / 1_048_576.0) + .and_then(|mib| u32::try_from(mib.trunc() as u64).ok()); + let compute_gflop_per_s = + parse_bounded_metric_csv_sum(compute_tflops_fp16, MAX_GOSSIP_COMPUTE_TFLOPS_PER_DEVICE) + .map(|tflops| tflops * 1_000.0) + .and_then(|gflops| u32::try_from(gflops.trunc() as u64).ok()); + Self { + sustained_mem_bandwidth_mib_per_s: bandwidth_mib_per_s, + sustained_compute_gflop_per_s: compute_gflop_per_s, + observed_decode_us_per_layer: None, + } + } + + fn with_stage_timing( + mut self, + hint: Option<&crate::network::metrics::ModelThroughputHint>, + ) -> Self { + self.observed_decode_us_per_layer = hint.and_then(|hint| { + if hint.stage_timing_samples.unwrap_or_default() >= MIN_TRUSTED_STAGE_TIMING_SAMPLES + && hint.stage_timing_age_ms.unwrap_or(u64::MAX) <= MAX_TRUSTED_STAGE_TIMING_AGE_MS + { + hint.observed_stage_us_per_layer + } else { + None + } + }); + self + } +} + +/// Sum a bounded comma-separated peer metric list. Limits are deliberately +/// above current hardware, but prevent a fabricated near-`u32::MAX` aggregate +/// from dominating placement or exhausting parse work. +fn parse_bounded_metric_csv_sum(field: Option<&str>, max_per_value: f64) -> Option { + let field = field?; + if field.len() > MAX_GOSSIP_GPU_METRIC_CSV_BYTES { + return None; + } + let mut total = 0.0f64; + let mut value_count = 0usize; + for (entry_index, entry) in field.split(',').enumerate() { + if entry_index >= MAX_GOSSIP_GPU_METRIC_VALUES { + return None; + } + let entry = entry.trim(); + if entry.is_empty() { + continue; + } + let value: f64 = entry.parse().ok()?; + if !value.is_finite() || value <= 0.0 || value > max_per_value { + return None; + } + value_count += 1; + total += value; + } + (value_count > 0).then_some(total) +} + impl SplitParticipantPackageSignal { pub(super) fn can_stage_with( self, @@ -478,11 +662,19 @@ pub(super) async fn collect_split_participant_membership( model_ref: &str, local_source_required: bool, ) -> SplitParticipantSnapshot { - let mut participants = vec![SplitParticipant::new( - node.id(), - node.vram_bytes(), - Some(node.first_joined_mesh_ts().await.unwrap_or(0)), - )]; + let local_perf = node.sustained_perf_signals().await; + let mut participants = vec![ + SplitParticipant::new( + node.id(), + node.vram_bytes(), + Some(node.first_joined_mesh_ts().await.unwrap_or(0)), + ) + .with_local_perf(SplitParticipantPerf { + sustained_mem_bandwidth_mib_per_s: local_perf.0, + sustained_compute_gflop_per_s: local_perf.1, + observed_decode_us_per_layer: None, + }), + ]; let mut excluded = Vec::new(); for peer in node.peers().await { if let Some(reason) = split_peer_preflight_exclusion_reason( @@ -521,12 +713,27 @@ pub(super) async fn collect_split_participants( local_vram_override: Option, local_source_required: bool, ) -> SplitParticipantSnapshot { - let mut participants = vec![SplitParticipant::local_package( - node.id(), - local_vram_override.unwrap_or_else(|| node.vram_bytes()), - Some(node.first_joined_mesh_ts().await.unwrap_or(0)), - package, - )]; + let local_perf = node.sustained_perf_signals().await; + let local_stage_timing = skippy_server::stage_decode_timing_hints() + .into_iter() + .find(|hint| hint.model_id == model_ref || hint.model_id == model_name); + let mut participants = vec![ + SplitParticipant::local_package( + node.id(), + local_vram_override.unwrap_or_else(|| node.vram_bytes()), + Some(node.first_joined_mesh_ts().await.unwrap_or(0)), + package, + ) + .with_local_perf(SplitParticipantPerf { + sustained_mem_bandwidth_mib_per_s: local_perf.0, + sustained_compute_gflop_per_s: local_perf.1, + observed_decode_us_per_layer: local_stage_timing.as_ref().and_then(|hint| { + (hint.sample_count >= MIN_TRUSTED_STAGE_TIMING_SAMPLES + && hint.sample_age_ms <= MAX_TRUSTED_STAGE_TIMING_AGE_MS) + .then_some(hint.observed_us_per_layer) + }), + }), + ]; let mut excluded = Vec::new(); for peer in node.peers().await { if let Some(reason) = split_peer_preflight_exclusion_reason( @@ -555,13 +762,25 @@ pub(super) async fn collect_split_participants( .await { Ok(package_signal) => { + let stage_timing = peer + .advertised_model_throughput + .iter() + .find(|hint| hint.model_name == model_ref || hint.model_name == model_name); + let perf = SplitParticipantPerf::from_gossip_csvs( + peer.gpu_mem_bandwidth_gbps.as_deref(), + peer.gpu_compute_tflops_fp16.as_deref(), + ) + .with_stage_timing(stage_timing); participants.push( SplitParticipant::new(peer.id, peer.vram_bytes, peer.first_joined_mesh_ts) .with_package_signals( package_signal, peer.rtt_ms, artifact_transfer_allowed, - ), + perf, + ) + .with_edge_bandwidth(peer.large_frame_mib_per_s()) + .with_rtt_observation(peer.rtt_observation_ages()), ); } Err(reason) => { @@ -817,23 +1036,73 @@ fn split_inventory_covered_layers<'a>( pub(super) fn split_participant_signature( participants: &[SplitParticipant], +) -> SplitParticipantSignature { + split_participant_signature_with_perf( + participants, + super::split_planning::perf_aware_placement_enabled(), + ) +} + +fn split_participant_signature_with_perf( + participants: &[SplitParticipant], + include_perf: bool, ) -> SplitParticipantSignature { participants .iter() .map(|participant| { + let link_bandwidth = if include_perf { + participant.large_frame_mib_per_s + } else { + None + }; + let mem_bandwidth = if include_perf { + participant.sustained_mem_bandwidth_mib_per_s + } else { + None + }; + let compute = if include_perf { + participant.sustained_compute_gflop_per_s + } else { + None + }; + let stage_timing = if include_perf { + participant.observed_decode_us_per_layer + } else { + None + }; ( participant.node_id.to_string(), participant.vram_bytes, participant.cached_slice_bytes, participant.missing_artifact_bytes, participant.rtt_ms, + link_bandwidth.map(|value| { + quantize_nonzero_u32(value, SIGNATURE_LINK_BANDWIDTH_BUCKET_MIB_PER_S) + }), participant.artifact_transfer_supported, participant.availability_score, + mem_bandwidth.map(|value| { + quantize_nonzero_u32(value, SIGNATURE_MEM_BANDWIDTH_BUCKET_MIB_PER_S) + }), + compute + .map(|value| quantize_nonzero_u32(value, SIGNATURE_COMPUTE_BUCKET_GFLOP_PER_S)), + stage_timing.map(|value| { + quantize_nonzero_u64(value, SIGNATURE_STAGE_TIMING_BUCKET_US_PER_LAYER) + }), + participant.rtt_corroborated, ) }) .collect() } +fn quantize_nonzero_u32(value: u32, bucket: u32) -> u32 { + value.max(bucket) / bucket * bucket +} + +fn quantize_nonzero_u64(value: u64, bucket: u64) -> u64 { + value.max(bucket) / bucket * bucket +} + pub(super) fn split_participant_set_hash(participants: &[SplitParticipant]) -> String { let mut hasher = Sha256::new(); for participant in split_participant_signature(participants) { @@ -842,8 +1111,13 @@ pub(super) fn split_participant_set_hash(participants: &[SplitParticipant]) -> S hasher.update(participant.2.to_le_bytes()); hasher.update(participant.3.to_le_bytes()); hasher.update(participant.4.unwrap_or_default().to_le_bytes()); - hasher.update([u8::from(participant.5)]); - hasher.update(participant.6.to_le_bytes()); + hasher.update(participant.5.unwrap_or_default().to_le_bytes()); + hasher.update([u8::from(participant.6)]); + hasher.update(participant.7.to_le_bytes()); + hasher.update(participant.8.unwrap_or_default().to_le_bytes()); + hasher.update(participant.9.unwrap_or_default().to_le_bytes()); + hasher.update(participant.10.unwrap_or_default().to_le_bytes()); + hasher.update([u8::from(participant.11)]); } hex::encode(hasher.finalize()) } @@ -1009,3 +1283,93 @@ pub(super) fn log_topology_plan_diagnostics( ); } } + +#[cfg(test)] +mod perf_signal_tests { + use super::*; + use crate::network::metrics::ModelThroughputHint; + + #[test] + fn peer_gpu_metrics_reject_implausible_or_oversized_csvs() { + let plausible = + SplitParticipantPerf::from_gossip_csvs(Some("1948.7,2100.1"), Some("850.0,900.0")); + assert!(plausible.sustained_mem_bandwidth_mib_per_s.is_some()); + assert!(plausible.sustained_compute_gflop_per_s.is_some()); + + let implausible = SplitParticipantPerf::from_gossip_csvs(Some("4294967295"), None); + assert_eq!(implausible.sustained_mem_bandwidth_mib_per_s, None); + + let too_many = std::iter::repeat_n("100", MAX_GOSSIP_GPU_METRIC_VALUES + 1) + .collect::>() + .join(","); + let oversized = SplitParticipantPerf::from_gossip_csvs(Some(&too_many), None); + assert_eq!(oversized.sustained_mem_bandwidth_mib_per_s, None); + + let too_long = "1".repeat(MAX_GOSSIP_GPU_METRIC_CSV_BYTES + 1); + let oversized = SplitParticipantPerf::from_gossip_csvs(Some(&too_long), None); + assert_eq!(oversized.sustained_mem_bandwidth_mib_per_s, None); + } + + #[test] + fn stage_timing_requires_fresh_multi_sample_evidence() { + let hint = |samples, age_ms| ModelThroughputHint { + model_name: "model".to_string(), + avg_tokens_per_second_milli: 0, + throughput_samples: 0, + observed_stage_us_per_layer: Some(2_500), + stage_timing_samples: Some(samples), + stage_timing_age_ms: Some(age_ms), + }; + + assert_eq!( + SplitParticipantPerf::default() + .with_stage_timing(Some(&hint(MIN_TRUSTED_STAGE_TIMING_SAMPLES, 500))) + .observed_decode_us_per_layer, + Some(2_500) + ); + assert_eq!( + SplitParticipantPerf::default() + .with_stage_timing(Some(&hint(MIN_TRUSTED_STAGE_TIMING_SAMPLES - 1, 500))) + .observed_decode_us_per_layer, + None + ); + assert_eq!( + SplitParticipantPerf::default() + .with_stage_timing(Some(&hint( + MIN_TRUSTED_STAGE_TIMING_SAMPLES, + MAX_TRUSTED_STAGE_TIMING_AGE_MS + 1, + ))) + .observed_decode_us_per_layer, + None + ); + } + + #[test] + fn participant_signature_ignores_sub_bucket_perf_noise() { + let mut first = SplitParticipant::new( + iroh::SecretKey::from_bytes(&[41; 32]).public(), + 40_000_000_000, + None, + ); + first.sustained_mem_bandwidth_mib_per_s = Some(400_001); + first.sustained_compute_gflop_per_s = Some(20_001); + first.observed_decode_us_per_layer = Some(2_501); + first.large_frame_mib_per_s = Some(101); + let mut noisy = first; + noisy.sustained_mem_bandwidth_mib_per_s = Some(404_999); + noisy.sustained_compute_gflop_per_s = Some(20_999); + noisy.observed_decode_us_per_layer = Some(2_599); + noisy.large_frame_mib_per_s = Some(104); + + assert_eq!( + split_participant_signature_with_perf(&[first], true), + split_participant_signature_with_perf(&[noisy], true) + ); + + noisy.observed_decode_us_per_layer = Some(2_600); + assert_ne!( + split_participant_signature_with_perf(&[first], true), + split_participant_signature_with_perf(&[noisy], true) + ); + } +} diff --git a/crates/mesh-llm-host-runtime/src/runtime/local_split/test_support.rs b/crates/mesh-llm-host-runtime/src/runtime/local_split/test_support.rs index 8e17ec7171..38b03981da 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/local_split/test_support.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/local_split/test_support.rs @@ -146,6 +146,7 @@ pub(super) fn split_test_peer( models: Vec::new(), vram_bytes: 24_000_000_000, rtt_ms: None, + rtt_observation_window: None, model_source: None, admitted: true, serving_models: Vec::new(), @@ -182,6 +183,7 @@ pub(super) fn split_test_peer( display_rtt: None, selected_path: None, + observed_large_frame: None, propagated_latency: None, owner_summary: crate::crypto::OwnershipSummary::default(), inference_admission_state: None, diff --git a/crates/mesh-llm-host-runtime/src/runtime/local_split/tests.rs b/crates/mesh-llm-host-runtime/src/runtime/local_split/tests.rs index bf5a857e16..abb2f20530 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/local_split/tests.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/local_split/tests.rs @@ -338,6 +338,7 @@ fn canonical_coordinator_is_identical_with_divergent_observer_signals() { }, Some(u32::from(seed) * 40), true, + SplitParticipantPerf::default(), ) }) .collect::>(); @@ -353,6 +354,7 @@ fn canonical_coordinator_is_identical_with_divergent_observer_signals() { }, Some(u32::from(4 - seed)), false, + SplitParticipantPerf::default(), ) }) .collect::>(); @@ -362,6 +364,61 @@ fn canonical_coordinator_is_identical_with_divergent_observer_signals() { assert_eq!(canonical_split_coordinator(&observer_b), Some(expected)); } +#[test] +fn rtt_floor_requires_settle_window_corroboration_for_remote_perf() { + let signal = SplitParticipantPackageSignal { + cached_slice_bytes: 100_000, + missing_artifact_bytes: 0, + availability_score: 4, + }; + let perf = SplitParticipantPerf { + sustained_mem_bandwidth_mib_per_s: Some(400_000), + sustained_compute_gflop_per_s: Some(15_000), + observed_decode_us_per_layer: Some(2_500), + }; + + let uncorroborated = SplitParticipant::new(make_id(1), 32_000_000_000, None) + .with_package_signals(signal, Some(8), true, perf) + .with_rtt_observation(Some(crate::mesh::RttObservationAges { + sample_count: 1, + first_sample_age_ms: 200, + last_sample_age_ms: 200, + })); + assert!(!uncorroborated.rtt_corroborated); + assert_eq!(uncorroborated.rtt_ms, None); + assert_eq!(uncorroborated.sustained_mem_bandwidth_mib_per_s, None); + assert_eq!(uncorroborated.sustained_compute_gflop_per_s, None); + assert_eq!(uncorroborated.observed_decode_us_per_layer, None); + + let corroborated = SplitParticipant::new(make_id(1), 32_000_000_000, None) + .with_package_signals(signal, Some(8), true, perf) + .with_rtt_observation(Some(crate::mesh::RttObservationAges { + sample_count: 2, + first_sample_age_ms: 5_500, + last_sample_age_ms: 200, + })); + assert!(corroborated.rtt_corroborated); + assert_eq!( + corroborated.sustained_mem_bandwidth_mib_per_s, + perf.sustained_mem_bandwidth_mib_per_s + ); + assert_eq!( + corroborated.observed_decode_us_per_layer, + perf.observed_decode_us_per_layer + ); + + let missing_confidence = SplitParticipant::new(make_id(1), 32_000_000_000, None) + .with_package_signals(signal, Some(8), true, perf) + .with_rtt_observation(None); + assert_eq!(missing_confidence.rtt_ms, None); + assert_eq!(missing_confidence.sustained_mem_bandwidth_mib_per_s, None); + assert_ne!( + split_participant_set_hash(&[uncorroborated]), + split_participant_set_hash(&[corroborated]), + "settle confidence transition must invalidate the placement signature" + ); +} + #[test] fn noncanonical_gate_returns_standby_without_invoking_package_planning() { let local = SplitParticipant::new(make_id(1), 24_000_000_000, None); @@ -420,6 +477,7 @@ fn resource_planner_keeps_canonical_coordinator_at_stage_zero() { }, Some(200), true, + SplitParticipantPerf::default(), ); let fast_a = SplitParticipant::new(make_id(2), 32_000_000_000, None).with_package_signals( SplitParticipantPackageSignal { @@ -429,6 +487,7 @@ fn resource_planner_keeps_canonical_coordinator_at_stage_zero() { }, Some(1), true, + SplitParticipantPerf::default(), ); let fast_b = SplitParticipant::new(make_id(3), 32_000_000_000, None).with_package_signals( SplitParticipantPackageSignal { @@ -438,6 +497,7 @@ fn resource_planner_keeps_canonical_coordinator_at_stage_zero() { }, Some(1), true, + SplitParticipantPerf::default(), ); let participants = [canonical, fast_a, fast_b]; let package = package(40); @@ -472,6 +532,7 @@ fn split_topology_planner_prefers_cached_participant_in_runtime_path() { }, Some(80), true, + SplitParticipantPerf::default(), ); let warm = SplitParticipant::new(make_id(2), 24_000_000_000, None).with_package_signals( SplitParticipantPackageSignal { @@ -481,6 +542,7 @@ fn split_topology_planner_prefers_cached_participant_in_runtime_path() { }, Some(5), true, + SplitParticipantPerf::default(), ); let stages = plan_runtime_slice_topology( @@ -1288,6 +1350,7 @@ fn split_participant_signature_includes_package_signals_for_claim_identity() { }, Some(20), true, + SplitParticipantPerf::default(), ), ]; diff --git a/crates/mesh-llm-host-runtime/src/runtime/split_participant_settle.rs b/crates/mesh-llm-host-runtime/src/runtime/split_participant_settle.rs index fac62ab53c..4441b12f73 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/split_participant_settle.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/split_participant_settle.rs @@ -314,7 +314,7 @@ fn split_membership_node_ids(participants: &[SplitParticipant]) -> Vec { mod tests { use super::*; use crate::runtime::local_package::{ - SplitParticipantPackageSignal, split_participant_signature, + SplitParticipantPackageSignal, SplitParticipantPerf, split_participant_signature, }; fn make_id(seed: u8) -> iroh::EndpointId { @@ -379,6 +379,7 @@ mod tests { }, Some(80), true, + SplitParticipantPerf::default(), ); participants[1] = participants[1].with_package_signals( SplitParticipantPackageSignal { @@ -388,6 +389,7 @@ mod tests { }, Some(4), true, + SplitParticipantPerf::default(), ); assert!(barrier.observe(&participants, start + Duration::from_secs(8))); diff --git a/crates/mesh-llm-host-runtime/src/runtime/split_planning.rs b/crates/mesh-llm-host-runtime/src/runtime/split_planning.rs index d3fe54a7df..6eea4e700f 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/split_planning.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/split_planning.rs @@ -54,6 +54,17 @@ pub(super) struct SplitTopologyPlanInput { pub(super) target_decode_tpot_ms: Option, pub(super) minimum_nodes: usize, pub(super) nodes: Vec, + pub(super) edges: Vec, + pub(super) activation_frame_bytes: u64, +} + +/// Directed link measurement carried into the coordinator planner. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(super) struct SplitTopologyPlanEdge { + pub(super) source_node_id: String, + pub(super) target_node_id: String, + pub(super) rtt_ms: u32, + pub(super) large_frame_mib_per_s: Option, } #[derive(Clone, Debug, Eq, PartialEq)] @@ -63,6 +74,9 @@ pub(super) struct SplitTopologyPlanNode { pub(super) max_vram_bytes: Option, pub(super) runtime_headroom_bytes: u64, pub(super) stage_transfer_latency_ms: Option, + pub(super) sustained_mem_bandwidth_mib_per_s: Option, + pub(super) sustained_compute_gflop_per_s: Option, + pub(super) observed_decode_us_per_layer: Option, } #[derive(Clone, Debug, Eq, PartialEq)] @@ -146,11 +160,28 @@ fn topology_planning_input(input: SplitTopologyPlanInput) -> TopologyPlanningInp max_vram_bytes: node.max_vram_bytes, runtime_headroom_bytes: node.runtime_headroom_bytes, stage_transfer_latency_ms: node.stage_transfer_latency_ms, + sustained_mem_bandwidth_mib_per_s: node.sustained_mem_bandwidth_mib_per_s, + sustained_compute_gflop_per_s: node.sustained_compute_gflop_per_s, + observed_decode_us_per_layer: node.observed_decode_us_per_layer, }) .collect(), context_length_override: input.context_length_override, parallel_lanes_override: input.parallel_lanes_override, target_decode_tpot_ms: input.target_decode_tpot_ms, + // MoE active-fraction metadata is not yet plumbed from the package + // identity; dense (1000 permil) is the conservative default. + active_weight_fraction_permil: 1000, + edges: input + .edges + .into_iter() + .map(|edge| skippy_coordinator::topology::TopologyEdge { + source_node_id: edge.source_node_id, + target_node_id: edge.target_node_id, + rtt_ms: edge.rtt_ms, + large_frame_mib_per_s: edge.large_frame_mib_per_s, + }) + .collect(), + activation_frame_bytes: input.activation_frame_bytes, } } @@ -351,6 +382,20 @@ fn runtime_slice_plan_input( package: &skippy::SkippyPackageIdentity, participants: &[SplitParticipant], resources: SplitTopologyResourceInputs, +) -> SplitTopologyPlanInput { + let mut plan_input = runtime_slice_plan_input_unfiltered(package, participants, resources); + + if !perf_aware_placement_enabled() { + strip_perf_aware_signals(&mut plan_input); + } + + plan_input +} + +fn runtime_slice_plan_input_unfiltered( + package: &skippy::SkippyPackageIdentity, + participants: &[SplitParticipant], + resources: SplitTopologyResourceInputs, ) -> SplitTopologyPlanInput { SplitTopologyPlanInput { native_context_length: resources.native_context_length, @@ -372,11 +417,98 @@ fn runtime_slice_plan_input( max_vram_bytes: Some(participant.vram_bytes), runtime_headroom_bytes: default_runtime_headroom_bytes(participant.vram_bytes), stage_transfer_latency_ms: participant.rtt_ms, + sustained_mem_bandwidth_mib_per_s: participant.sustained_mem_bandwidth_mib_per_s, + sustained_compute_gflop_per_s: participant.sustained_compute_gflop_per_s, + observed_decode_us_per_layer: participant.observed_decode_us_per_layer, }) .collect(), + edges: participant_edges(participants), + // The stage wire protocol carries raw f32 activations. Family-specific + // sidebands can multiply this at particular boundaries; until the + // automatic planner consumes that legality metadata, use the exact + // dense-boundary payload instead of the stale f16 assumption. + activation_frame_bytes: skippy_topology::wire_payload_bytes_per_token( + package.activation_width, + ), } } +/// Strip performance-aware planning signals when the experimental mode is not +/// explicitly enabled. Fields that pre-date +/// perf-aware planning — per-node RTT (`stage_transfer_latency_ms`) and the +/// decode TPOT target — are deliberately kept: the legacy planner consumed +/// both, so stripping them would change capacity-only placement instead of +/// reproducing it. Tested by `kill_switch_strip_keeps_pre_perf_aware_fields`. +fn strip_perf_aware_signals(plan_input: &mut SplitTopologyPlanInput) { + for node in &mut plan_input.nodes { + node.sustained_mem_bandwidth_mib_per_s = None; + node.sustained_compute_gflop_per_s = None; + node.observed_decode_us_per_layer = None; + } + plan_input.edges = Vec::new(); + plan_input.activation_frame_bytes = 0; +} + +/// Whether performance-aware placement is enabled. The optimizer remains an +/// explicit opt-in while its measurement trust and replan-adoption contracts +/// are being hardened: only `1`, `true`, `on`, or `yes` enables it. Unset, +/// disable spellings, and unknown values preserve capacity-only placement. +/// The value is read afresh on every planning attempt. +pub(super) fn perf_aware_placement_enabled() -> bool { + perf_aware_enabled_from_value(std::env::var("MESH_TOPOLOGY_PERF_AWARE").ok().as_deref()) +} + +fn perf_aware_enabled_from_value(value: Option<&str>) -> bool { + value.is_some_and(|value| { + matches!( + value.trim().to_ascii_lowercase().as_str(), + "1" | "true" | "on" | "yes" + ) + }) +} + +/// Directed edge measurements between participants. The mesh does not relay +/// peer-to-peer pair measurements, so both RTT and bandwidth are synthesized +/// from each participant's coordinator-observed link (RTT from gossip round +/// trips; bandwidth from passive artifact-transfer observation). RTT takes the +/// conservative `max` of the two sides so the estimate can never +/// underestimate a real hop; bandwidth takes the conservative `min` (a +/// stage's egress is limited by the slower direction's sustain). Missing +/// observations fall back per-signal: no RTT on either side ⇒ no edge; no +/// bandwidth ⇒ latency-only edge, preserving pre-probing behavior. +fn participant_edges(participants: &[SplitParticipant]) -> Vec { + let mut edges = Vec::new(); + for (index, source) in participants.iter().enumerate() { + for target in participants.iter().skip(index + 1) { + let Some(rtt_ms) = source.rtt_ms.into_iter().chain(target.rtt_ms).max() else { + continue; + }; + let large_frame_mib_per_s = source + .large_frame_mib_per_s + .into_iter() + .chain(target.large_frame_mib_per_s) + .min(); + let (forward, reverse) = ( + SplitTopologyPlanEdge { + source_node_id: source.node_id.to_string(), + target_node_id: target.node_id.to_string(), + rtt_ms, + large_frame_mib_per_s, + }, + SplitTopologyPlanEdge { + source_node_id: target.node_id.to_string(), + target_node_id: source.node_id.to_string(), + rtt_ms, + large_frame_mib_per_s, + }, + ); + edges.push(forward); + edges.push(reverse); + } + } + edges +} + fn package_layer_weight_bytes(package: &skippy::SkippyPackageIdentity) -> Vec { if package.layer_weight_bytes.len() == package.layer_count as usize { return package.layer_weight_bytes.clone(); @@ -524,12 +656,20 @@ pub(super) fn split_participant_labels(participants: &[SplitParticipant]) -> Vec .iter() .map(|participant| { format!( - "{}:{} cached={} missing={} rtt={}ms transfer={}", + "{}:{} cached={} missing={} rtt={}ms rtt_samples={} rtt_first_age={}ms rtt_last_age={}ms rtt_corroborated={} transfer={}", participant.node_id.fmt_short(), format_gb(participant.vram_bytes), format_gb(participant.cached_slice_bytes), format_gb(participant.missing_artifact_bytes), participant.rtt_ms.unwrap_or_default(), + participant.rtt_sample_count, + participant + .rtt_first_sample_age_ms + .map_or_else(|| "-".to_string(), |age| age.to_string()), + participant + .rtt_last_sample_age_ms + .map_or_else(|| "-".to_string(), |age| age.to_string()), + participant.rtt_corroborated, participant.artifact_transfer_supported ) }) @@ -717,6 +857,18 @@ mod tests { participant } + fn participant_with_perf( + seed: u8, + vram_bytes: u64, + rtt_ms: u32, + bandwidth_mib_per_s: u32, + ) -> SplitParticipant { + let mut participant = participant_with_rtt(seed, vram_bytes, rtt_ms); + participant.sustained_mem_bandwidth_mib_per_s = Some(bandwidth_mib_per_s); + participant.sustained_compute_gflop_per_s = Some(15_000); + participant + } + #[test] fn default_runtime_headroom_reserves_decode_margin() { // This fixed reserve is 1/10 (10%) of the advertised budget — the @@ -972,4 +1124,128 @@ mod tests { assert!(reason.contains("max_layers=0")); assert!(reason.contains("missing_model_source")); } + + #[test] + fn perf_aware_planning_requires_explicit_enable_value() { + assert!(!perf_aware_enabled_from_value(None)); + assert!(!perf_aware_enabled_from_value(Some("0"))); + assert!(!perf_aware_enabled_from_value(Some("false"))); + assert!(!perf_aware_enabled_from_value(Some("off"))); + assert!(!perf_aware_enabled_from_value(Some("no"))); + assert!(!perf_aware_enabled_from_value(Some("perf"))); + assert!(!perf_aware_enabled_from_value(Some(""))); + + assert!(perf_aware_enabled_from_value(Some("1"))); + assert!(perf_aware_enabled_from_value(Some("true"))); + assert!(perf_aware_enabled_from_value(Some("on"))); + assert!(perf_aware_enabled_from_value(Some("yes"))); + assert!(perf_aware_enabled_from_value(Some("ON"))); + assert!(perf_aware_enabled_from_value(Some(" yes "))); + } + + #[test] + fn disabled_mode_strip_keeps_pre_perf_aware_fields() { + // The default-off parity contract reproduces pre-PR capacity-only + // placement, which consumed per-node RTT and the decode TPOT target. + // The strip removes only signals introduced by perf-aware planning. + let mut plan_input = runtime_slice_plan_input_unfiltered( + &package(40, 40_000_000_000), + &[ + participant_with_perf(1, 26_000_000_000, 5, 400_000), + participant_with_perf(2, 26_000_000_000, 9, 120_000), + ], + SplitTopologyResourceInputs { + native_context_length: 262_144, + kv_bytes_per_token: 64 * 1024, + recurrent_bytes_per_sequence_by_layer: Vec::new(), + ctx_size_override: None, + parallel_override: None, + }, + ); + assert!( + plan_input.target_decode_tpot_ms.is_some(), + "fixture must set the TPOT target for the assertion to mean anything" + ); + assert!( + plan_input + .nodes + .iter() + .all(|node| node.stage_transfer_latency_ms.is_some()), + "fixture must set RTT for the assertion to mean anything" + ); + assert_eq!( + plan_input.activation_frame_bytes, + u64::from(896_u32) * 4, + "automatic planning must price the raw-f32 wire payload" + ); + + strip_perf_aware_signals(&mut plan_input); + + // Pre-perf-aware fields survive the strip. + assert!(plan_input.target_decode_tpot_ms.is_some()); + assert!( + plan_input + .nodes + .iter() + .all(|node| node.stage_transfer_latency_ms.is_some()) + ); + // Perf-aware signals are stripped. + assert!(plan_input.nodes.iter().all( + |node| node.sustained_mem_bandwidth_mib_per_s.is_none() + && node.sustained_compute_gflop_per_s.is_none() + && node.observed_decode_us_per_layer.is_none() + )); + assert!(plan_input.edges.is_empty()); + assert_eq!(plan_input.activation_frame_bytes, 0); + } + + #[test] + fn participant_edges_take_conservative_rtt_max_and_bandwidth_min() { + let mut fast_link = participant_with_rtt(1, 40_000_000_000, 5); + fast_link.large_frame_mib_per_s = Some(800); + let mut slow_link = participant_with_rtt(2, 40_000_000_000, 25); + slow_link.large_frame_mib_per_s = Some(120); + + let edges = participant_edges(&[fast_link, slow_link]); + assert_eq!(edges.len(), 2, "one edge per direction"); + // RTT is the max of the two sides' coordinator observations; the + // estimate must never under-estimate a real hop. + assert_eq!(edges[0].rtt_ms, 25); + assert_eq!(edges[1].rtt_ms, 25); + // Bandwidth is the min of the two directions' sustained throughput: + // the link runs at the slower side's pace. + assert_eq!(edges[0].large_frame_mib_per_s, Some(120)); + assert_eq!(edges[1].large_frame_mib_per_s, Some(120)); + assert_eq!(edges[0].source_node_id, edges[1].target_node_id); + assert_eq!(edges[0].target_node_id, edges[1].source_node_id); + } + + #[test] + fn participant_edges_without_bandwidth_stay_latency_only() { + // No passive observation on either side: latency-only edges, + // exactly the pre-probing behavior. + let participants = vec![ + participant_with_rtt(1, 40_000_000_000, 5), + participant_with_rtt(2, 40_000_000_000, 8), + ]; + let edges = participant_edges(&participants); + assert_eq!(edges.len(), 2); + assert!( + edges + .iter() + .all(|edge| edge.large_frame_mib_per_s.is_none()) + ); + assert_eq!(edges[0].rtt_ms, 8); + + // A single observed side propagates to the pair (the mesh only + // observes coordinator links, so partial coverage is the norm). + let mut one_sided = participants; + one_sided[0].large_frame_mib_per_s = Some(400); + let edges = participant_edges(&one_sided); + assert!( + edges + .iter() + .all(|edge| edge.large_frame_mib_per_s == Some(400)) + ); + } } diff --git a/crates/mesh-llm-host-runtime/src/runtime_data/mod.rs b/crates/mesh-llm-host-runtime/src/runtime_data/mod.rs index dce2875360..134d20d5ae 100644 --- a/crates/mesh-llm-host-runtime/src/runtime_data/mod.rs +++ b/crates/mesh-llm-host-runtime/src/runtime_data/mod.rs @@ -515,6 +515,7 @@ pub(crate) mod tests { models: vec!["Peer-Model".into()], vram_bytes: 32_000_000_000, rtt_ms: Some(7), + rtt_observation_window: None, model_source: None, admitted: true, serving_models: vec!["Peer-Model".into()], @@ -555,6 +556,7 @@ pub(crate) mod tests { cache_affinity: None, display_rtt: None, selected_path: None, + observed_large_frame: None, propagated_latency: None, owner_summary: crate::crypto::OwnershipSummary::default(), inference_admission_state: None, @@ -660,6 +662,7 @@ pub(crate) mod tests { models: vec!["Qwen/Qwen3-Coder".into()], vram_bytes: 32_000_000_000, rtt_ms: Some(7), + rtt_observation_window: None, model_source: None, admitted: true, serving_models: vec!["Qwen/Qwen3-Coder".into()], @@ -695,10 +698,14 @@ pub(crate) mod tests { model_name: "Qwen/Qwen3-Coder".into(), avg_tokens_per_second_milli: 13_400, throughput_samples: 27, + observed_stage_us_per_layer: None, + stage_timing_samples: None, + stage_timing_age_ms: None, }], cache_affinity: None, display_rtt: None, selected_path: None, + observed_large_frame: None, propagated_latency: None, owner_summary: crate::crypto::OwnershipSummary::default(), inference_admission_state: None, diff --git a/crates/mesh-llm-protocol/proto/node.proto b/crates/mesh-llm-protocol/proto/node.proto index d15c77e653..b661b2883d 100644 --- a/crates/mesh-llm-protocol/proto/node.proto +++ b/crates/mesh-llm-protocol/proto/node.proto @@ -155,6 +155,9 @@ message AdvertisedModelThroughput { string model_name = 1; uint64 avg_tokens_per_second_milli = 2; uint64 throughput_samples = 3; + optional uint64 observed_stage_us_per_layer = 4; + optional uint64 stage_timing_samples = 5; + optional uint64 stage_timing_age_ms = 6; } message CacheAffinityAdvertisement { diff --git a/crates/mesh-llm-protocol/src/proto/node.rs b/crates/mesh-llm-protocol/src/proto/node.rs index 36b009cfeb..901197bec2 100644 --- a/crates/mesh-llm-protocol/src/proto/node.rs +++ b/crates/mesh-llm-protocol/src/proto/node.rs @@ -141,6 +141,12 @@ pub struct AdvertisedModelThroughput { pub avg_tokens_per_second_milli: u64, #[prost(uint64, tag = "3")] pub throughput_samples: u64, + #[prost(uint64, optional, tag = "4")] + pub observed_stage_us_per_layer: ::core::option::Option, + #[prost(uint64, optional, tag = "5")] + pub stage_timing_samples: ::core::option::Option, + #[prost(uint64, optional, tag = "6")] + pub stage_timing_age_ms: ::core::option::Option, } #[derive(Clone, PartialEq, ::prost::Message)] pub struct CacheAffinityAdvertisement { diff --git a/crates/skippy-coordinator/src/topology.rs b/crates/skippy-coordinator/src/topology.rs index 8859952da4..d685f1706a 100644 --- a/crates/skippy-coordinator/src/topology.rs +++ b/crates/skippy-coordinator/src/topology.rs @@ -1,7 +1,24 @@ use std::cmp::Ordering; +use std::collections::HashMap; mod locked; +/// Calibrated per-stage software overhead for one decode step (dispatch, +/// kernel-launch slop), in microseconds. Inherited from the execution sim's +/// calibration against the BENCHMARKS.md anchors (see +/// `skippy-topology-sim/scenarios/benchmarks_anchor_pair.toml`, +/// `per_stage_overhead_ms = 1.3`). The `planner_model_matches_execution_sim` +/// test locks the planner and sim to the same values. +pub const CALIBRATED_PER_STAGE_OVERHEAD_US: u128 = 1_300; + +/// Calibrated per-hop overhead beyond RTT + activation transfer for one +/// decode token (QUIC stream setup, copies, scheduling), in microseconds. +/// Inherited from the execution sim's calibration: BENCHMARKS.md names +/// per-token RPC latency the dominant split cost, and 13 ms/hop is the +/// back-solved coefficient from the 2-way anchor. Without this term the +/// planner under-prices every WAN edge by ~10 ms per hop. +pub const CALIBRATED_PER_HOP_OVERHEAD_US: u128 = 13_000; + pub use locked::{LockedTopologyStage, plan_locked_topology}; const MINIMUM_AUTO_CONTEXT_LENGTH: u32 = 65_536; @@ -47,6 +64,34 @@ pub struct TopologyPlanningInput { pub context_length_override: Option, pub parallel_lanes_override: Option, pub target_decode_tpot_ms: Option, + /// Fraction of layer weights actually streamed per decode token, in + /// per-mille of total (1000 = dense). MoE models touch only the active + /// experts; the calibrated anchor scenario uses 340 (0.34). Default for + /// callers without MoE metadata: 1000 — dense over-estimates TPOT + /// uniformly, which is conservative for target-met and does not change + /// relative candidate ordering. + pub active_weight_fraction_permil: u32, + /// Directed node-pair link measurements. An empty vector keeps the + /// legacy hop-count × worst-RTT network estimate, so callers without + /// edge data reproduce today's behavior exactly. + pub edges: Vec, + /// Activation frame size in bytes sent per token between stages at the + /// package's wire dtype (`activation_width × dtype size`). Used only for + /// edge transfer-time terms when edge bandwidth is known; `0` disables + /// bandwidth terms (latency-only edges). + pub activation_frame_bytes: u64, +} + +/// Directed link measurement between two candidate stage nodes. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct TopologyEdge { + pub source_node_id: String, + pub target_node_id: String, + /// Round-trip latency in milliseconds for this direction. + pub rtt_ms: u32, + /// Large-frame (activation-sized) throughput in MiB/s. `None` when the + /// edge has latency data but no bandwidth measurement yet. + pub large_frame_mib_per_s: Option, } #[derive(Clone, Debug, Eq, PartialEq)] @@ -56,6 +101,18 @@ pub struct TopologyNode { pub max_vram_bytes: Option, pub runtime_headroom_bytes: u64, pub stage_transfer_latency_ms: Option, + /// Sustained memory bandwidth in MiB/s, measured (gpu-bench) and gossiped. + /// `None` keeps this node capacity-only: performance-aware span balancing + /// is only active when every node in the planned subset reports it, so + /// signal-less fleets reproduce capacity-only placement exactly. + pub sustained_mem_bandwidth_mib_per_s: Option, + /// Sustained fp16 compute in GFLOP/s, measured and gossiped. Secondary + /// signal (decode is usually memory-bound); `None` = unreported. + pub sustained_compute_gflop_per_s: Option, + /// Observed steady-decode runtime work, normalized per loaded layer. + /// When present, the planner uses it as a measured floor on the + /// analytical weight-streaming service-time estimate. + pub observed_decode_us_per_layer: Option, } #[derive(Clone, Debug, Eq, PartialEq)] @@ -65,6 +122,17 @@ pub struct TopologyPlan { pub stages: Vec, pub estimated_decode_network_ms_per_token: Option, pub decode_tpot_target_met: Option, + /// Modeled single-stream decode TPOT in microseconds: serial form — + /// Σ stage service times + Σ hop times (including the prediction + /// return), each stage charged its weight-streaming time plus the + /// calibrated per-stage overhead, each hop its RTT + activation + /// transfer + the calibrated per-hop overhead. `None` unless every + /// node in the chosen subset reports sustained memory bandwidth + /// (capacity-only plans carry no model). Matches the calibrated + /// execution sim (`skippy-topology-sim`); the + /// `planner_model_matches_execution_sim` calibration test locks the + /// two together on the BENCHMARKS.md anchor scenario. + pub modeled_decode_tpot_us: Option, } #[derive(Clone, Debug, Eq, PartialEq)] @@ -183,9 +251,10 @@ fn plan_topology_with_required_stage0( }); if let Some(candidate) = best_for_count { if latency_aware { - if best_latency_candidate.as_ref().is_none_or(|current| { - latency_candidate_better(&candidate, current, input) - }) { + if best_latency_candidate + .as_ref() + .is_none_or(|current| latency_candidate_better(&candidate, current)) + { best_latency_candidate = Some(candidate); } continue; @@ -298,6 +367,9 @@ struct UsableNode { node_id: String, usable_vram_bytes: u64, stage_transfer_latency_ms: Option, + sustained_mem_bandwidth_mib_per_s: Option, + sustained_compute_gflop_per_s: Option, + observed_decode_us_per_layer: Option, } fn usable_nodes(nodes: &[TopologyNode]) -> Vec { @@ -312,6 +384,9 @@ fn usable_nodes(nodes: &[TopologyNode]) -> Vec { node_id: node.node_id.clone(), usable_vram_bytes: capped.saturating_sub(node.runtime_headroom_bytes), stage_transfer_latency_ms: node.stage_transfer_latency_ms, + sustained_mem_bandwidth_mib_per_s: node.sustained_mem_bandwidth_mib_per_s, + sustained_compute_gflop_per_s: node.sustained_compute_gflop_per_s, + observed_decode_us_per_layer: node.observed_decode_us_per_layer, } }) .collect::>(); @@ -356,6 +431,10 @@ struct CandidatePlan { plan: TopologyPlan, minimum_remaining_vram: u64, total_remaining_vram: u128, + /// Modeled per-token decode time (serial stage service + network) in + /// microseconds; present only when every node in the subset reports + /// sustained bandwidth. Drives candidate preference when comparable. + modeled_decode_tpot_us: Option, } impl Ord for CandidatePlan { @@ -424,6 +503,70 @@ fn fit_candidate( let mut minimum_remaining_vram = u64::MAX; let mut total_remaining_vram = 0u128; + // Performance-aware span assignment: when every node in the subset reports + // sustained memory bandwidth, minimize modeled single-stream serial decode + // time (weight streaming dominates quantized decode). Any missing signal + // falls back to the exact capacity-greedy walk below, so signal-less fleets + // keep bit-identical placement. + let streamed_layer_weights = streamed_layer_weight_bytes(input); + if let Some((spans, _stage_service_us)) = serial_optimized_spans( + &streamed_layer_weights, + &layer_required_bytes, + &capacities, + input.layer_count as usize, + ) { + for (stage_index, (node, span)) in capacities.iter().zip(spans).enumerate() { + let layer_start = next_layer; + let layer_end = layer_start + span as u32; + let range = layer_start as usize..layer_end as usize; + let parameter_bytes = sum_u64(&layer_weights[range.clone()]); + let required_bytes = sum_u64(&layer_required_bytes[range]); + debug_assert!(required_bytes <= node.usable_vram_bytes); + let remaining = node.usable_vram_bytes - required_bytes; + minimum_remaining_vram = minimum_remaining_vram.min(remaining); + total_remaining_vram += u128::from(remaining); + stages.push(TopologyStagePlan { + stage_id: format!("stage-{stage_index}"), + stage_index: stage_index as u32, + node_id: node.node_id.clone(), + layer_start, + layer_end, + parameter_bytes, + }); + next_layer = layer_end; + } + debug_assert_eq!(next_layer, input.layer_count); + + let estimated_decode_network_ms_per_token = + candidate_network_ms_per_token(&stages, nodes, input); + // Modeled single-stream decode TPOT, serial form: every token + // traverses every stage and returns, so TPOT = Σ stage service + // times + Σ hop times. This is the form the BENCHMARKS.md anchors + // prove (see the execution sim's calibration); the previous + // bottleneck-stage form under-priced multi-stage plans. Both + // calibrated overhead terms are included so the planner's number + // matches the calibrated sim. + let modeled_decode_tpot_us = modeled_serial_decode_tpot_us(&stages, input); + // Target-met is scored against the modeled decode TPOT, the best + // estimate of it this plan has. Network-only scoring would mark + // single-stage plans as trivially meeting any target. + let decode_tpot_target_met = + decode_tpot_target_met_us(modeled_decode_tpot_us, input.target_decode_tpot_ms); + return Some(CandidatePlan { + plan: TopologyPlan { + context_length, + parallel_lanes, + stages, + estimated_decode_network_ms_per_token, + decode_tpot_target_met, + modeled_decode_tpot_us, + }, + minimum_remaining_vram, + total_remaining_vram, + modeled_decode_tpot_us, + }); + } + for (stage_index, node) in capacities.iter().enumerate() { let remaining_layers = input.layer_count - next_layer; let remaining_nodes = capacities.len() - stage_index; @@ -465,20 +608,25 @@ fn fit_candidate( return None; } - let estimated_decode_network_ms_per_token = estimate_decode_network_ms_per_token(nodes); + let estimated_decode_network_ms_per_token = + candidate_network_ms_per_token(&stages, nodes, input); Some(CandidatePlan { plan: TopologyPlan { context_length, parallel_lanes, stages, estimated_decode_network_ms_per_token, - decode_tpot_target_met: decode_tpot_target_met( + // Network time is a lower bound on full decode TPOT: it can prove + // a miss, but it cannot prove success without compute signals. + decode_tpot_target_met: decode_tpot_target_from_network_lower_bound( estimated_decode_network_ms_per_token, input.target_decode_tpot_ms, ), + modeled_decode_tpot_us: None, }, minimum_remaining_vram, total_remaining_vram, + modeled_decode_tpot_us: None, }) } @@ -502,58 +650,94 @@ fn candidate_has_required_stage0( } fn candidate_better_for_same_shape(candidate: &CandidatePlan, current: &CandidatePlan) -> bool { - let candidate_estimate = candidate - .plan - .estimated_decode_network_ms_per_token - .unwrap_or_default(); - let current_estimate = current - .plan - .estimated_decode_network_ms_per_token - .unwrap_or_default(); - candidate_estimate < current_estimate - || (candidate_estimate == current_estimate && candidate.cmp(current) == Ordering::Greater) + // This comparison is between different node subsets of the same count, + // so signal completeness can differ. Prefer a complete TPOT model, then + // compare like-for-like estimates; never turn a missing estimate into + // zero latency. + estimate_completeness(candidate) + .cmp(&estimate_completeness(current)) + .then_with(|| { + lower_option_is_better( + candidate.modeled_decode_tpot_us, + current.modeled_decode_tpot_us, + ) + }) + .then_with(|| { + lower_option_is_better( + candidate.plan.estimated_decode_network_ms_per_token, + current.plan.estimated_decode_network_ms_per_token, + ) + }) + .then_with(|| candidate.cmp(current)) + == Ordering::Greater } -fn latency_candidate_better( - candidate: &CandidatePlan, - current: &CandidatePlan, - input: &TopologyPlanningInput, -) -> bool { - latency_candidate_ordering(candidate, current, input) == Ordering::Greater +fn latency_candidate_better(candidate: &CandidatePlan, current: &CandidatePlan) -> bool { + latency_candidate_ordering(candidate, current) == Ordering::Greater } -fn latency_candidate_ordering( - left: &CandidatePlan, - right: &CandidatePlan, - input: &TopologyPlanningInput, -) -> Ordering { - let left_estimate = left - .plan - .estimated_decode_network_ms_per_token - .unwrap_or_default(); - let right_estimate = right - .plan - .estimated_decode_network_ms_per_token - .unwrap_or_default(); - let left_target_met = decode_tpot_target_met( - left.plan.estimated_decode_network_ms_per_token, - input.target_decode_tpot_ms, - ) - .unwrap_or(true); - let right_target_met = decode_tpot_target_met( - right.plan.estimated_decode_network_ms_per_token, - input.target_decode_tpot_ms, - ) - .unwrap_or(true); - - left_target_met - .cmp(&right_target_met) - .then_with(|| right_estimate.cmp(&left_estimate)) +fn latency_candidate_ordering(left: &CandidatePlan, right: &CandidatePlan) -> Ordering { + estimate_completeness(left) + .cmp(&estimate_completeness(right)) + .then_with(|| { + if left.modeled_decode_tpot_us.is_some() && right.modeled_decode_tpot_us.is_some() { + target_status_rank(left.plan.decode_tpot_target_met) + .cmp(&target_status_rank(right.plan.decode_tpot_target_met)) + } else { + Ordering::Equal + } + }) + .then_with(|| { + lower_option_is_better(left.modeled_decode_tpot_us, right.modeled_decode_tpot_us) + }) + .then_with(|| { + lower_option_is_better( + left.plan.estimated_decode_network_ms_per_token, + right.plan.estimated_decode_network_ms_per_token, + ) + }) .then_with(|| left.plan.context_length.cmp(&right.plan.context_length)) .then_with(|| left.plan.parallel_lanes.cmp(&right.plan.parallel_lanes)) .then_with(|| left.cmp(right)) } +/// Within equally complete full-TPOT estimates, known target success wins, +/// followed by an unconfigured target and then a known miss. Estimate +/// completeness is compared first so withholding compute cannot improve rank. +fn target_status_rank(status: Option) -> u8 { + match status { + Some(true) => 2, + None => 1, + Some(false) => 0, + } +} + +/// Full modeled TPOT is more decision-useful than a network-only estimate, +/// which is more useful than no estimate. Numeric values are only compared by +/// `lower_option_is_better` when both candidates carry the same signal kind. +fn estimate_completeness(candidate: &CandidatePlan) -> u8 { + if candidate.modeled_decode_tpot_us.is_some() { + 2 + } else if candidate + .plan + .estimated_decode_network_ms_per_token + .is_some() + { + 1 + } else { + 0 + } +} + +fn lower_option_is_better(left: Option, right: Option) -> Ordering { + match (left, right) { + (Some(left), Some(right)) => right.cmp(&left), + (Some(_), None) => Ordering::Greater, + (None, Some(_)) => Ordering::Less, + (None, None) => Ordering::Equal, + } +} + fn estimate_decode_network_ms_per_token(nodes: &[UsableNode]) -> Option { let hop_latency = nodes .iter() @@ -562,8 +746,237 @@ fn estimate_decode_network_ms_per_token(nodes: &[UsableNode]) -> Option { Some(hop_latency.saturating_mul(nodes.len() as u32)) } -fn decode_tpot_target_met(estimate: Option, target: Option) -> Option { - Some(estimate? <= target?) +/// Network time for one decode step across the pipeline stages, in +/// microseconds, from directed edge measurements. Each hop is charged its +/// measured RTT plus the activation-frame transfer time when the edge also +/// reports bandwidth. Hops are matched directed-first, then by their reverse +/// edge, then fall back to that node's coordinator RTT; an unmatched hop +/// with no fallback aborts edge-based estimation (caller keeps the legacy +/// estimate). Returns `None` when the input carries no edge data at all. +fn pipeline_network_time_us( + stages: &[TopologyStagePlan], + nodes: &[UsableNode], + input: &TopologyPlanningInput, +) -> Option { + if input.edges.is_empty() { + return None; + } + if stages.len() < 2 { + return Some(0); + } + let rtt_by_node: HashMap<&str, u32> = nodes + .iter() + .filter_map(|node| { + node.stage_transfer_latency_ms + .map(|rtt| (node.node_id.as_str(), rtt)) + }) + .collect(); + // Charge one hop: directed edge first, then the reverse edge (same pair, + // measured), then the endpoint nodes' coordinator RTT. An unmatched hop + // with no fallback aborts edge-based estimation. + let hop_rtt_ms = |source: &TopologyStagePlan, target: &TopologyStagePlan| -> Option { + let edge = input + .edges + .iter() + .find(|edge| { + edge.source_node_id == source.node_id && edge.target_node_id == target.node_id + }) + .or_else(|| { + input.edges.iter().find(|edge| { + edge.source_node_id == target.node_id && edge.target_node_id == source.node_id + }) + }); + edge.map(|edge| edge.rtt_ms).or_else(|| { + rtt_by_node + .get(target.node_id.as_str()) + .copied() + .or_else(|| rtt_by_node.get(source.node_id.as_str()).copied()) + }) + }; + let hop_transfer_us = |source: &TopologyStagePlan, target: &TopologyStagePlan| -> u128 { + let bandwidth = input + .edges + .iter() + .find(|edge| { + edge.source_node_id == source.node_id && edge.target_node_id == target.node_id + }) + .or_else(|| { + input.edges.iter().find(|edge| { + edge.source_node_id == target.node_id && edge.target_node_id == source.node_id + }) + }) + .and_then(|edge| edge.large_frame_mib_per_s); + match bandwidth { + Some(bandwidth) if bandwidth > 0 && input.activation_frame_bytes > 0 => { + u128::from(input.activation_frame_bytes) * 1_000_000 + / (u128::from(bandwidth) * 1_048_576) + } + _ => 0, + } + }; + let mut total_us = 0u128; + for window in stages.windows(2) { + let rtt_ms = hop_rtt_ms(&window[0], &window[1])?; + total_us += u128::from(rtt_ms) * 1_000; + total_us += hop_transfer_us(&window[0], &window[1]); + } + // The final stage returns predictions to stage 0 — charge that hop too, + // matching the legacy estimate's per-node accounting. + let last = stages.last().expect("stages.len() >= 2"); + let first = stages.first().expect("stages.len() >= 2"); + let return_rtt_ms = hop_rtt_ms(last, first)?; + total_us += u128::from(return_rtt_ms) * 1_000; + total_us += hop_transfer_us(last, first); + Some(total_us) +} + +/// Usable per-candidate network estimate in whole milliseconds: the +/// edge-based model when available, else the legacy hop-count estimate. +fn candidate_network_ms_per_token( + stages: &[TopologyStagePlan], + nodes: &[UsableNode], + input: &TopologyPlanningInput, +) -> Option { + match pipeline_network_time_us(stages, nodes, input) { + Some(us) => Some(u32::try_from(us / 1_000).unwrap_or(u32::MAX)), + None => estimate_decode_network_ms_per_token(nodes), + } +} + +fn decode_tpot_target_met_us(estimate_us: Option, target_ms: Option) -> Option { + Some(estimate_us? <= u128::from(target_ms?).saturating_mul(1_000)) +} + +fn decode_tpot_target_from_network_lower_bound( + network_ms: Option, + target_ms: Option, +) -> Option { + (network_ms? > target_ms?).then_some(false) +} + +/// Modeled single-stream decode TPOT for a planned stage sequence, serial +/// form: Σ per-stage service times + Σ per-hop times (including the +/// prediction return). Stage service time = stage weight-streaming time at +/// the node's sustained bandwidth + calibrated per-stage overhead; hop time +/// = edge RTT + activation transfer + calibrated per-hop overhead (falling +/// back the same way the network estimate does). Requires every stage's +/// node to report bandwidth; `None` otherwise (capacity-only plan). +fn modeled_serial_decode_tpot_us( + stages: &[TopologyStagePlan], + input: &TopologyPlanningInput, +) -> Option { + if stages.is_empty() { + return None; + } + // Per-stage service times from the same per-layer weight table the DP + // used (scaled by the active weight fraction for MoE models); `None` + // if any node lacks a bandwidth signal. + let layer_weights = streamed_layer_weight_bytes(input); + let mut total_us = 0u128; + for stage in stages { + let node = input + .nodes + .iter() + .find(|node| node.node_id == stage.node_id)?; + let bandwidth = node + .sustained_mem_bandwidth_mib_per_s + .filter(|bw| *bw > 0)?; + let range = stage.layer_start as usize..stage.layer_end as usize; + let weight_bytes: u64 = layer_weights.get(range.clone()).map_or(0, sum_u64); + total_us += modeled_stage_time_us_from( + bandwidth, + weight_bytes, + node.observed_decode_us_per_layer, + u64::from(stage.layer_end.saturating_sub(stage.layer_start)), + ); + total_us += CALIBRATED_PER_STAGE_OVERHEAD_US; + } + // Hop times: reuse the edge model's per-hop accounting (RTT + + // transfer), and add the calibrated per-hop overhead per hop, including + // the prediction return. Node RTTs are looked up directly (the edge + // model works on UsableNode slices; here input.nodes suffices). + let hop_rtt_ms = |source: &TopologyStagePlan, target: &TopologyStagePlan| -> Option { + input + .edges + .iter() + .find(|edge| { + edge.source_node_id == source.node_id && edge.target_node_id == target.node_id + }) + .or_else(|| { + input.edges.iter().find(|edge| { + edge.source_node_id == target.node_id && edge.target_node_id == source.node_id + }) + }) + .map(|edge| edge.rtt_ms) + .or_else(|| { + input + .nodes + .iter() + .find(|node| node.node_id == target.node_id) + .and_then(|node| node.stage_transfer_latency_ms) + .or_else(|| { + input + .nodes + .iter() + .find(|node| node.node_id == source.node_id) + .and_then(|node| node.stage_transfer_latency_ms) + }) + }) + }; + let hop_transfer_us = |source: &TopologyStagePlan, target: &TopologyStagePlan| -> u128 { + input + .edges + .iter() + .find(|edge| { + edge.source_node_id == source.node_id && edge.target_node_id == target.node_id + }) + .or_else(|| { + input.edges.iter().find(|edge| { + edge.source_node_id == target.node_id && edge.target_node_id == source.node_id + }) + }) + .and_then(|edge| edge.large_frame_mib_per_s) + .map_or(0, |bandwidth| { + if bandwidth > 0 && input.activation_frame_bytes > 0 { + u128::from(input.activation_frame_bytes) * 1_000_000 + / (u128::from(bandwidth) * 1_048_576) + } else { + 0 + } + }) + }; + let mut hop_count = 0u128; + for window in stages.windows(2) { + let rtt_ms = hop_rtt_ms(&window[0], &window[1])?; + total_us += u128::from(rtt_ms) * 1_000; + total_us += hop_transfer_us(&window[0], &window[1]); + hop_count += 1; + } + if stages.len() > 1 { + let last = stages.last().expect("len > 1"); + let first = stages.first().expect("len > 1"); + let return_rtt_ms = hop_rtt_ms(last, first)?; + total_us += u128::from(return_rtt_ms) * 1_000; + total_us += hop_transfer_us(last, first); + hop_count += 1; + } + total_us += hop_count * CALIBRATED_PER_HOP_OVERHEAD_US; + Some(total_us) +} + +/// Weight-streaming time in microseconds for `weight_bytes` at +/// `bandwidth_mib_per_s` (bytes × 1e6 / (MiB/s × 2^20)). +fn modeled_stage_time_us_from( + bandwidth_mib_per_s: u32, + weight_bytes: u64, + observed_us_per_layer: Option, + layer_count: u64, +) -> u128 { + let analytical = + u128::from(weight_bytes) * 1_000_000 / (u128::from(bandwidth_mib_per_s) * 1_048_576); + let observed = u128::from(observed_us_per_layer.unwrap_or_default()) + .saturating_mul(u128::from(layer_count)); + analytical.max(observed) } fn layer_weight_bytes(input: &TopologyPlanningInput) -> Vec { @@ -576,6 +989,21 @@ fn layer_weight_bytes(input: &TopologyPlanningInput) -> Vec { vec![weight_per_layer; input.layer_count as usize] } +/// Layer weights actually streamed per decode token: the full table scaled +/// by `active_weight_fraction_permil` (per-mille; 1000 = dense). MoE models +/// touch only the active experts — the calibrated anchor scenario uses 340 +/// (0.34). Clamped to [1, 1000] so a zero fraction can never make stage +/// service time vanish. Used only by the modeled-TPOT path; capacity +/// accounting always uses full weights. +fn streamed_layer_weight_bytes(input: &TopologyPlanningInput) -> Vec { + let weights = layer_weight_bytes(input); + let fraction_permil = input.active_weight_fraction_permil.clamp(1, 1000); + weights + .into_iter() + .map(|bytes| bytes * u64::from(fraction_permil) / 1_000) + .collect() +} + fn candidate_bytes_per_layer( weight_per_layer: u64, kv_per_layer: u64, @@ -625,6 +1053,132 @@ fn recurrent_bytes_by_layer(input: &TopologyPlanningInput) -> Vec { vec![0; input.layer_count as usize] } +/// Modeled per-stage decode service time in microseconds, using the dominant +/// term for quantized decode: streaming the stage's weights from memory. +/// Integer microseconds keep candidate comparisons deterministic. +fn modeled_stage_time_us(node: &UsableNode, weight_bytes: u64, layer_count: usize) -> Option { + let bandwidth = node.sustained_mem_bandwidth_mib_per_s?; + if bandwidth == 0 { + return None; + } + Some(modeled_stage_time_us_from( + bandwidth, + weight_bytes, + node.observed_decode_us_per_layer, + layer_count as u64, + )) +} + +/// Performance-aware contiguous span assignment via DP over layer boundaries. +/// +/// Nodes arrive in the planner's deterministic stage order (VRAM-descending, +/// node id tie-break). For each contiguous split of the layer sequence across +/// the stages, every stage's memory requirement must fit its node's ceiling +/// (checked with prefix sums in O(1)); among feasible assignments we minimize +/// the serial sum of modeled stage service times, matching the single-stream +/// TPOT evaluator used to rank the resulting plan. Ties prefer the smaller +/// bottleneck stage time, then the lexicographically smallest boundary vector +/// for determinism. Returns `None` unless every node reports +/// sustained memory bandwidth — the caller then keeps today's capacity-greedy +/// walk, which guarantees signal-less fleets keep identical placement. +fn serial_optimized_spans( + layer_weights: &[u64], + linearized_required_bytes: &[u64], + capacities: &[UsableNode], + layer_count: usize, +) -> Option<(Vec, u128)> { + if capacities.is_empty() || layer_weights.len() != layer_count { + return None; + } + // All-or-nothing on the dominant signal: partial signals would make the + // modeled comparison between stages meaningless. + if capacities + .iter() + .any(|node| node.sustained_mem_bandwidth_mib_per_s.is_none()) + { + return None; + } + + // Prefix sums over the linearized memory requirement (u128 guards against + // overflow when context is large). + let mut prefix_required = vec![0u128; layer_count + 1]; + for (index, bytes) in linearized_required_bytes.iter().enumerate() { + prefix_required[index + 1] = prefix_required[index] + u128::from(*bytes); + } + let mut prefix_weights = vec![0u128; layer_count + 1]; + for (index, bytes) in layer_weights.iter().enumerate() { + prefix_weights[index + 1] = prefix_weights[index] + u128::from(*bytes); + } + + // dp[stage][boundary] = best (total stage time, max stage time) for + // assigning layers 0..boundary to stages 0..=stage, plus the parent + // boundary for reconstruction. + let mut dp = vec![vec![(u128::MAX, u128::MAX, 0usize); layer_count + 1]; capacities.len()]; + for (stage_index, node) in capacities.iter().enumerate() { + for boundary in 0..=layer_count { + if stage_index == 0 { + // Stage 0 owns layers 0..boundary and must be non-empty in the + // final plan; dp[0][0] stays unreachable so no chain can leave + // a stage empty. + let weight = prefix_weights[boundary]; + if boundary == 0 { + continue; + } + if let Some(time) = modeled_stage_time_us(node, weight.try_into().ok()?, boundary) { + let fits = prefix_required[boundary] <= u128::from(node.usable_vram_bytes); + if fits { + dp[0][boundary] = (time, time, 0); + } + } + continue; + } + // Non-final stages may not consume all remaining layers; leave at + // least one for each later stage. + let max_boundary = layer_count - (capacities.len() - 1 - stage_index); + if boundary > max_boundary { + continue; + } + let mut best = (u128::MAX, u128::MAX, 0usize); + for previous in 0..boundary { + let (prev_total, prev_max, _) = dp[stage_index - 1][previous]; + if prev_total == u128::MAX { + continue; + } + let weight = prefix_weights[boundary] - prefix_weights[previous]; + let Some(time) = + modeled_stage_time_us(node, weight.try_into().ok()?, boundary - previous) + else { + continue; + }; + let required = prefix_required[boundary] - prefix_required[previous]; + if required > u128::from(node.usable_vram_bytes) { + continue; + } + let candidate = (prev_total + time, prev_max.max(time), previous); + if candidate < best { + best = candidate; + } + } + dp[stage_index][boundary] = best; + } + } + let final_stage = capacities.len() - 1; + let (best_total, _, _) = dp[final_stage][layer_count]; + if best_total == u128::MAX { + return None; + } + // Reconstruct boundary chain. + let mut spans = Vec::with_capacity(capacities.len()); + let mut boundary = layer_count; + for stage_index in (0..capacities.len()).rev() { + let previous = dp[stage_index][boundary].2; + spans.push(boundary - previous); + boundary = previous; + } + spans.reverse(); + Some((spans, best_total)) +} + fn max_contiguous_layers_from( layer_required_bytes: &[u64], start: usize, @@ -670,6 +1224,9 @@ mod tests { max_vram_bytes: None, runtime_headroom_bytes: 0, stage_transfer_latency_ms: None, + sustained_mem_bandwidth_mib_per_s: None, + sustained_compute_gflop_per_s: None, + observed_decode_us_per_layer: None, } } @@ -680,6 +1237,13 @@ mod tests { } } + fn perf_node(id: &str, gib: u64, mem_bandwidth_mib_per_s: u32) -> TopologyNode { + TopologyNode { + sustained_mem_bandwidth_mib_per_s: Some(mem_bandwidth_mib_per_s), + ..node(id, gib) + } + } + fn input(nodes: Vec) -> TopologyPlanningInput { TopologyPlanningInput { native_context_length: 65_536, @@ -694,6 +1258,9 @@ mod tests { context_length_override: None, parallel_lanes_override: None, target_decode_tpot_ms: None, + active_weight_fraction_permil: 1000, + edges: Vec::new(), + activation_frame_bytes: 0, } } @@ -711,6 +1278,9 @@ mod tests { context_length_override: None, parallel_lanes_override: None, target_decode_tpot_ms: None, + active_weight_fraction_permil: 1000, + edges: Vec::new(), + activation_frame_bytes: 0, } } @@ -722,6 +1292,412 @@ mod tests { (0..count).map(|index| qwen_node(index, gib)).collect() } + #[test] + fn edge_data_replaces_hop_count_estimate() { + // Two latency-aware nodes with 5 ms coordinator RTT each. Without + // edges the legacy estimate is hop_count x max RTT = 10 ms. With + // directed edges at 5 ms each the edge model also yields 10 ms here, + // but with an asymmetric edge (2 ms) the edge model must charge the + // honest per-hop latency (2 + 5 = 7 ms), not 2 x max(5) = 10 ms. + let mut planning = input(vec![latency_node("a", 48, 5), latency_node("b", 48, 5)]); + planning.minimum_nodes = 2; + let legacy = plan_topology(&planning).expect("legacy plan"); + planning.edges = vec![TopologyEdge { + source_node_id: "a".into(), + target_node_id: "b".into(), + rtt_ms: 5, + large_frame_mib_per_s: None, + }]; + let symmetric = plan_topology(&planning).expect("symmetric edge plan"); + planning.edges = vec![TopologyEdge { + source_node_id: "a".into(), + target_node_id: "b".into(), + rtt_ms: 2, + large_frame_mib_per_s: None, + }]; + let asymmetric = plan_topology(&planning).expect("asymmetric edge plan"); + assert_eq!( + legacy.estimated_decode_network_ms_per_token, + Some(10), + "legacy estimate is hop count x max RTT" + ); + assert_eq!( + symmetric.estimated_decode_network_ms_per_token, + Some(10), + "symmetric edges sum forward + return hop RTT" + ); + assert_eq!( + asymmetric.estimated_decode_network_ms_per_token, + Some(4), + "asymmetric edge charges forward + reverse-matched return (2 + 2)" + ); + } + + #[test] + fn edge_bandwidth_charges_activation_transfer_time() { + // Same topology as above; the edge now reports 1 MiB/s large-frame + // bandwidth with a 1 MiB activation frame: transfer adds ~1.05 s + // per token hop, dwarfing latency and failing a 33 ms TPOT target. + let mut planning = input(vec![latency_node("a", 48, 5), latency_node("b", 48, 5)]); + planning.minimum_nodes = 2; + planning.target_decode_tpot_ms = Some(33); + planning.activation_frame_bytes = 1024 * 1024; + planning.edges = vec![TopologyEdge { + source_node_id: "a".into(), + target_node_id: "b".into(), + rtt_ms: 5, + large_frame_mib_per_s: Some(1), + }]; + let plan = plan_topology(&planning).expect("plan"); + assert!( + plan.estimated_decode_network_ms_per_token.unwrap_or(0) > 1_000, + "slow edge bandwidth must charge activation transfer time" + ); + assert_eq!(plan.decode_tpot_target_met, Some(false)); + } + + #[test] + fn missing_edge_falls_back_to_node_rtt() { + // Edge data exists for one hop only; the unmatched hop falls back to + // the node's coordinator RTT instead of aborting the estimate. + let mut planning = input(vec![ + latency_node("a", 48, 5), + latency_node("b", 48, 7), + latency_node("c", 48, 9), + ]); + planning.minimum_nodes = 3; + planning.edges = vec![TopologyEdge { + source_node_id: "a".into(), + target_node_id: "b".into(), + rtt_ms: 1, + large_frame_mib_per_s: None, + }]; + let plan = plan_topology(&planning).expect("plan"); + // a->b edge (1 ms) + b->c fallback to c's 9 ms RTT + c->a return + // fallback to a's 5 ms RTT = 15 ms. + assert_eq!(plan.estimated_decode_network_ms_per_token, Some(15)); + } + + #[test] + fn empty_edges_keep_legacy_estimate() { + let mut planning = input(vec![latency_node("a", 48, 5), latency_node("b", 48, 5)]); + planning.minimum_nodes = 2; + let plan = plan_topology(&planning).expect("plan"); + assert_eq!(plan.estimated_decode_network_ms_per_token, Some(10)); + } + + #[test] + fn single_stream_objective_assigns_only_required_work_to_slower_node() { + // For single-stream TPOT, stage times are serial. With ample memory, + // the faster node should therefore receive every layer except the one + // required to keep the slower stage non-empty. + let fast = perf_node("fast", 48, 546_000); + let slow = perf_node("slow", 48, 273_000); + let mut planning = input(vec![fast, slow]); + planning.minimum_nodes = 2; + let plan = plan_topology(&planning).expect("plan"); + assert_eq!(plan.stages.len(), 2); + let fast_stage = plan + .stages + .iter() + .find(|stage| stage.node_id == "fast") + .expect("fast stage"); + let slow_stage = plan + .stages + .iter() + .find(|stage| stage.node_id == "slow") + .expect("slow stage"); + assert_eq!(slow_stage.layer_end - slow_stage.layer_start, 1); + assert_eq!(fast_stage.layer_end - fast_stage.layer_start, 39); + } + + #[test] + fn observed_stage_timing_corrects_analytical_span_balance() { + let fast = perf_node("fast", 48, 400_000); + let mut slow = perf_node("slow", 48, 400_000); + // Both nodes advertise identical bandwidth, but live decode shows + // that the second runtime takes substantially longer per layer. + slow.observed_decode_us_per_layer = Some(10_000); + let mut planning = input(vec![fast, slow]); + planning.minimum_nodes = 2; + + let plan = plan_topology(&planning).expect("plan"); + let fast_layers = plan + .stages + .iter() + .find(|stage| stage.node_id == "fast") + .map(|stage| stage.layer_end - stage.layer_start) + .expect("fast stage"); + let slow_layers = plan + .stages + .iter() + .find(|stage| stage.node_id == "slow") + .map(|stage| stage.layer_end - stage.layer_start) + .expect("slow stage"); + + assert!( + fast_layers > slow_layers, + "measured slow stage must receive fewer layers: fast={fast_layers} slow={slow_layers}" + ); + } + + #[test] + fn missing_perf_signals_keep_capacity_only_placement() { + // Any node without a bandwidth signal reproduces the capacity-only + // plan exactly: same stage boundaries and node assignment. + let nodes_signal = vec![perf_node("a", 48, 400_000), perf_node("b", 24, 400_000)]; + let mut nodes_plain = nodes_signal.clone(); + for node in &mut nodes_plain { + node.sustained_mem_bandwidth_mib_per_s = None; + node.sustained_compute_gflop_per_s = None; + } + let mut planning = input(nodes_plain.clone()); + planning.minimum_nodes = 2; + let plain = plan_topology(&planning).expect("plain plan"); + let signaled = plan_topology(&input(nodes_signal)).expect("signaled plan"); + let spans: Vec<(String, u32, u32)> = plain + .stages + .iter() + .map(|stage| (stage.node_id.clone(), stage.layer_start, stage.layer_end)) + .collect(); + let _spans_signaled: Vec<(String, u32, u32)> = signaled + .stages + .iter() + .map(|stage| (stage.node_id.clone(), stage.layer_start, stage.layer_end)) + .collect(); + // With equal bandwidths on both nodes the perf-aware path may still + // rebalance; the guarantee under test is that *removing* signals + // yields the capacity-only result, asserted against the greedy + // expectations: node a (48 GiB) should hold more layers than b (24). + let _ = signaled; + let a_stage = spans.iter().find(|(id, _, _)| id == "a").unwrap(); + let b_stage = spans.iter().find(|(id, _, _)| id == "b").unwrap(); + assert!(a_stage.2 - a_stage.1 > b_stage.2 - b_stage.1); + // And the fallback is exercised: partial signals on the signaled + // input must produce identical output to the plain input. + let mut nodes_partial = nodes_plain.clone(); + nodes_partial[0].sustained_mem_bandwidth_mib_per_s = Some(400_000); + let mut planning_partial = input(nodes_partial); + planning_partial.minimum_nodes = 2; + let partial = plan_topology(&planning_partial).expect("partial plan"); + let spans_partial: Vec<(String, u32, u32)> = partial + .stages + .iter() + .map(|stage| (stage.node_id.clone(), stage.layer_start, stage.layer_end)) + .collect(); + assert_eq!( + spans, spans_partial, + "partial signals must fall back to capacity-only placement" + ); + } + + #[test] + fn signalless_subset_placement_unchanged_by_other_nodes_signals() { + // The fallback is per-subset, not fleet-wide: a node without a + // bandwidth signal keeps its capacity-only span assignment even when + // other fleet nodes do report signals (a heterogeneous fleet). The + // two signaled nodes are too small to host the model alone or as a + // pair, so every feasible candidate contains the plain node — the + // assertion is never vacuous. + let plain = node("plain", 60); + let mut planning_mixed = input(vec![ + plain.clone(), + perf_node("signaled", 10, 400_000), + perf_node("other", 10, 400_000), + ]); + planning_mixed.minimum_nodes = 2; + let mut planning_plain_twin = input(vec![plain, node("signaled", 10), node("other", 10)]); + planning_plain_twin.minimum_nodes = 2; + let mixed = plan_topology(&planning_mixed).expect("mixed plan"); + let plain_twin = plan_topology(&planning_plain_twin).expect("plain twin plan"); + let span_of = |plan: &TopologyPlan, id: &str| { + plan.stages + .iter() + .find(|stage| stage.node_id == id) + .map(|stage| (stage.layer_start, stage.layer_end)) + }; + assert!( + span_of(&mixed, "plain").is_some() && span_of(&plain_twin, "plain").is_some(), + "fixture must select the plain node in both plans for the test to mean anything" + ); + assert_eq!( + span_of(&mixed, "plain"), + span_of(&plain_twin, "plain"), + "a signal-less node's span must not change because other fleet nodes report signals" + ); + } + + #[test] + fn edge_data_changes_capacity_greedy_candidate_ordering() { + // Documented behavior, not a bug: any non-empty edge data switches + // the network estimate to per-hop edge-aware accounting for every + // candidate — including capacity-greedy plans from signal-less + // nodes. This test pins that: with plain nodes (no bandwidth + // signals), asymmetric edge data changes the selected plan's + // network estimate vs the legacy hop-count × max-RTT number. + let mut planning = input(vec![latency_node("a", 48, 5), latency_node("b", 48, 5)]); + planning.minimum_nodes = 2; + let legacy = plan_topology(&planning).expect("legacy plan"); + let mut edged = planning.clone(); + edged.edges = vec![TopologyEdge { + source_node_id: "a".into(), + target_node_id: "b".into(), + rtt_ms: 2, + large_frame_mib_per_s: None, + }]; + let edge_plan = plan_topology(&edged).expect("edge plan"); + assert_ne!( + legacy.estimated_decode_network_ms_per_token, + edge_plan.estimated_decode_network_ms_per_token, + "edge data must change the network estimate for capacity-greedy plans too" + ); + assert_eq!(legacy.estimated_decode_network_ms_per_token, Some(10)); + assert_eq!(edge_plan.estimated_decode_network_ms_per_token, Some(4)); + } + + #[test] + fn decode_tpot_target_met_uses_modeled_tpot() { + // Target-met must be scored against the modeled decode TPOT + // (stage service time + network), not the network-only + // estimate. Single-stage plans have zero network time but still + // carry the full weight-streaming time of the model. + // Model: 40 layers, 40 GiB weights (1 GiB/layer), KV 0. + let mut planning = input(vec![perf_node("solo", 80, 400_000)]); + planning.kv_bytes_per_token = 1; // negligible KV; weights dominate + planning.target_decode_tpot_ms = Some(10); + let plan = plan_topology(&planning).expect("plan"); + // Network-only estimate may be None (no RTT data); the modeled TPOT + // is what the target must be scored against. + // 40 GiB at 400_000 MiB/s = 104.9 ms/token modeled decode TPOT — + // far over a 10 ms target. + assert_eq!(plan.decode_tpot_target_met, Some(false)); + } + + #[test] + fn modeled_tpot_target_comparison_keeps_microsecond_precision() { + assert_eq!( + decode_tpot_target_met_us(Some(33_000), Some(33)), + Some(true) + ); + assert_eq!( + decode_tpot_target_met_us(Some(33_001), Some(33)), + Some(false), + "a fractional-millisecond overrun must not be rounded into the target" + ); + } + + #[test] + fn candidate_ordering_does_not_reward_missing_full_tpot() { + let candidate = |network_ms, target_met, modeled_us| CandidatePlan { + plan: TopologyPlan { + context_length: 65_536, + parallel_lanes: 1, + stages: Vec::new(), + estimated_decode_network_ms_per_token: Some(network_ms), + decode_tpot_target_met: target_met, + modeled_decode_tpot_us: modeled_us, + }, + minimum_remaining_vram: 0, + total_remaining_vram: 0, + modeled_decode_tpot_us: modeled_us, + }; + let modeled_miss = candidate(20, Some(false), Some(100_000)); + let fallback_unknown = candidate(1, None, None); + + assert!( + latency_candidate_better(&modeled_miss, &fallback_unknown), + "withholding compute data must not turn a network-only estimate into a target success" + ); + } + + #[test] + fn missing_network_estimate_is_not_treated_as_zero() { + let candidate = |network_ms| CandidatePlan { + plan: TopologyPlan { + context_length: 65_536, + parallel_lanes: 1, + stages: Vec::new(), + estimated_decode_network_ms_per_token: network_ms, + decode_tpot_target_met: None, + modeled_decode_tpot_us: None, + }, + minimum_remaining_vram: 0, + total_remaining_vram: 0, + modeled_decode_tpot_us: None, + }; + let measured = candidate(Some(20)); + let missing = candidate(None); + + assert!(candidate_better_for_same_shape(&measured, &missing)); + assert!(!candidate_better_for_same_shape(&missing, &measured)); + } + + #[test] + fn tpot_target_met_outranks_modeled_tpot_in_candidate_ordering() { + // Locks the candidate-ordering priority: decode-TPOT-target-met + // outranks the modeled TPOT (and context/lanes). Note that after + // scoring target-met against the *modeled* TPOT (see + // `decode_tpot_target_met_uses_modeled_tpot`), met is monotone in + // modeled TPOT, so on the fully-signaled path the two keys cannot + // conflict; this ordering matters for mixed-signal comparisons and + // keeps legacy key priority. Constructed via stage-0 binding, the + // only input surface that forces different candidate sets from one + // fleet. + let mut planning = input(vec![ + perf_node("large", 80, 400_000), + perf_node("small", 30, 150_000), + perf_node("tiny", 20, 150_000), + ]); + // Hops need RTT data for the modeled TPOT to exist: without any + // RTT signal the planner declines to model TPOT (None) rather + // than pretending hops are free. + for node in &mut planning.nodes { + node.stage_transfer_latency_ms = Some(3); + } + planning.kv_bytes_per_token = 1; // negligible KV; weights dominate + planning.target_decode_tpot_ms = Some(110); + // Binding stage 0 to the large node: it fits the 40 GiB model solo + // (104.9 ms/token at 400_000 MiB/s), so a target-meeting plan + // exists and must be returned. + let large_stage0 = + plan_topology_with_stage0(&planning, "large").expect("large stage0 plan"); + assert_eq!(large_stage0.decode_tpot_target_met, Some(true)); + // Binding stage 0 to the small node rules out every subset where + // the large node would be stage 0 (stage order is VRAM-descending), + // leaving {small, tiny}: 40 GiB across two 150_000 MiB/s nodes is a + // ~133 ms/token bottleneck - over the 110 ms target. + let small_stage0 = + plan_topology_with_stage0(&planning, "small").expect("small stage0 plan"); + assert_eq!(small_stage0.stages.len(), 2); + assert_eq!(small_stage0.decode_tpot_target_met, Some(false)); + } + + #[test] + fn perf_balancing_respects_memory_ceilings() { + // The slow node has a much smaller ceiling; the DP must not assign it + // more layers than fit, no matter how attractive the time balance. + let fast = perf_node("fast", 96, 500_000); + let slow = perf_node("slow", 16, 500_000); + let mut planning = input(vec![fast, slow]); + planning.minimum_nodes = 2; + let plan = plan_topology(&planning).expect("plan"); + for stage in &plan.stages { + assert!(stage.layer_end > stage.layer_start, "no empty stages"); + } + } + + #[test] + fn perf_signals_do_not_break_latency_aware_planning() { + // Latency-aware ordering still applies when perf signals are present; + // the plan remains valid and stage 0 binding is respected. + let mut a = perf_node("a", 48, 400_000); + a.stage_transfer_latency_ms = Some(30); + let mut b = perf_node("b", 48, 400_000); + b.stage_transfer_latency_ms = Some(30); + let plan = plan_topology_with_stage0(&input(vec![a, b]), "a").expect("plan"); + assert_eq!(plan.stages.first().unwrap().node_id, "a"); + } + #[test] fn lane_planning_rejects_exhausted_sequence_ids() { assert_eq!( @@ -756,6 +1732,9 @@ mod tests { context_length_override: Some(65_536), parallel_lanes_override: Some(LANES), target_decode_tpot_ms: None, + active_weight_fraction_permil: 1000, + edges: Vec::new(), + activation_frame_bytes: 0, }; let layer_weights = layer_weight_bytes(&request); let kv_per_layer = request.kv_bytes_per_token.div_ceil(u64::from(LAYERS)); @@ -830,6 +1809,9 @@ mod tests { // reported by the local runtime. runtime_headroom_bytes: 0, stage_transfer_latency_ms: None, + sustained_mem_bandwidth_mib_per_s: None, + sustained_compute_gflop_per_s: None, + observed_decode_us_per_layer: None, } } @@ -956,7 +1938,10 @@ mod tests { assert_eq!(plan.context_length, 65_536); assert_eq!(plan.stages.len(), 2); assert_eq!(plan.estimated_decode_network_ms_per_token, Some(20)); - assert_eq!(plan.decode_tpot_target_met, Some(true)); + assert_eq!( + plan.decode_tpot_target_met, None, + "network time below target is not proof that full TPOT meets it" + ); } #[test] diff --git a/crates/skippy-coordinator/src/topology/locked.rs b/crates/skippy-coordinator/src/topology/locked.rs index f7a534ca8c..4b57b288de 100644 --- a/crates/skippy-coordinator/src/topology/locked.rs +++ b/crates/skippy-coordinator/src/topology/locked.rs @@ -1,8 +1,9 @@ use super::{ CandidatePlan, TopologyPlan, TopologyPlanError, TopologyPlanningInput, TopologyStagePlan, - UsableNode, context_candidates, decode_tpot_target_met, estimate_decode_network_ms_per_token, - layer_required_bytes, layer_weight_bytes, minimum_valid_context, parallel_lane_candidates, - recurrent_bytes_by_layer, sum_u64, usable_nodes, validate_input, + UsableNode, context_candidates, decode_tpot_target_from_network_lower_bound, + estimate_decode_network_ms_per_token, layer_required_bytes, layer_weight_bytes, + minimum_valid_context, parallel_lane_candidates, recurrent_bytes_by_layer, sum_u64, + usable_nodes, validate_input, }; #[derive(Clone, Debug, Eq, PartialEq)] @@ -164,13 +165,15 @@ fn fit_locked_candidate( parallel_lanes, stages, estimated_decode_network_ms_per_token, - decode_tpot_target_met: decode_tpot_target_met( + decode_tpot_target_met: decode_tpot_target_from_network_lower_bound( estimated_decode_network_ms_per_token, input.target_decode_tpot_ms, ), + modeled_decode_tpot_us: None, }, minimum_remaining_vram, total_remaining_vram, + modeled_decode_tpot_us: None, }) } @@ -188,6 +191,9 @@ mod tests { max_vram_bytes: None, runtime_headroom_bytes: 0, stage_transfer_latency_ms: None, + sustained_mem_bandwidth_mib_per_s: None, + sustained_compute_gflop_per_s: None, + observed_decode_us_per_layer: None, } } @@ -205,6 +211,9 @@ mod tests { context_length_override: None, parallel_lanes_override: None, target_decode_tpot_ms: None, + active_weight_fraction_permil: 1000, + edges: Vec::new(), + activation_frame_bytes: 0, } } diff --git a/crates/skippy-server/src/binary_transport/binary_messaging/connection.rs b/crates/skippy-server/src/binary_transport/binary_messaging/connection.rs index 56a7589690..caa2d75815 100644 --- a/crates/skippy-server/src/binary_transport/binary_messaging/connection.rs +++ b/crates/skippy-server/src/binary_transport/binary_messaging/connection.rs @@ -658,6 +658,7 @@ fn handle_binary_connection_messages( result }; let compute_ms = elapsed_ms(compute_started); + crate::stage_performance::record_stage_decode_timing(config, &message, compute_ms); compute_end_unix_nanos = now_unix_nanos() as u64; (result.0, result.1, result.2, result.3, compute_ms) }; diff --git a/crates/skippy-server/src/lib.rs b/crates/skippy-server/src/lib.rs index 5f26e08d23..32b8782c2b 100644 --- a/crates/skippy-server/src/lib.rs +++ b/crates/skippy-server/src/lib.rs @@ -37,7 +37,9 @@ mod legacy_scheduler_absence_tests { } } pub mod serving_hooks; +mod stage_performance; pub mod telemetry; +pub use stage_performance::{StageDecodeTimingHint, stage_decode_timing_hints}; pub mod tokenizer; // Re-export key types for consumers diff --git a/crates/skippy-server/src/stage_performance.rs b/crates/skippy-server/src/stage_performance.rs new file mode 100644 index 0000000000..373383481f --- /dev/null +++ b/crates/skippy-server/src/stage_performance.rs @@ -0,0 +1,228 @@ +//! Process-local observations of steady decode work performed by staged runtimes. +//! +//! Embedded stages share a process with the host, so retaining a bounded, +//! model-keyed timing hint here lets the host advertise real stage behavior +//! without changing the stage execution wire protocol. + +use std::{ + collections::HashMap, + sync::{Mutex, OnceLock}, + time::{Duration, Instant}, +}; + +use skippy_protocol::{StageConfig, binary::StageWireMessage}; + +const MAX_OBSERVATION_AGE: Duration = Duration::from_secs(30 * 60); +const MAX_EFFECTIVE_SAMPLES: u64 = 256; +const MAX_TRACKED_STAGE_TIMINGS: usize = 128; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct StageDecodeTimingHint { + pub model_id: String, + /// Mean steady-decode runtime work, normalized by the loaded layer count. + pub observed_us_per_layer: u64, + pub sample_count: u64, + pub sample_age_ms: u64, +} + +#[derive(Clone, Debug)] +struct StageDecodeTimingObservation { + observed_us_per_layer: u64, + sample_count: u64, + observed_at: Instant, + layer_start: u32, + layer_end: u32, +} + +static STAGE_DECODE_TIMINGS: OnceLock>> = + OnceLock::new(); + +pub(crate) fn record_stage_decode_timing( + config: &StageConfig, + message: &StageWireMessage, + compute_ms: f64, +) { + if !matches!( + message.kind, + skippy_protocol::binary::WireMessageKind::DecodeEmbd + ) || message.state.decode_step < 8 + || !compute_ms.is_finite() + || compute_ms <= 0.0 + { + return; + } + let layer_count = u64::from(config.layer_end.saturating_sub(config.layer_start)); + let Some(executed_tokens) = u64::try_from(message.token_count) + .ok() + .filter(|count| *count > 0) + else { + return; + }; + if layer_count == 0 { + return; + } + let compute_us = (compute_ms * 1_000.0).round().max(1.0) as u64; + let sample = compute_us.div_ceil(layer_count.saturating_mul(executed_tokens)); + let mut timings = STAGE_DECODE_TIMINGS + .get_or_init(|| Mutex::new(HashMap::new())) + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let now = Instant::now(); + timings.retain(|_, observation| { + now.duration_since(observation.observed_at) <= MAX_OBSERVATION_AGE + }); + let incompatible = timings.get(&config.model_id).is_some_and(|observation| { + observation.layer_start != config.layer_start || observation.layer_end != config.layer_end + }); + if incompatible { + timings.remove(&config.model_id); + } + if !timings.contains_key(&config.model_id) + && timings.len() >= MAX_TRACKED_STAGE_TIMINGS + && let Some(oldest) = timings + .iter() + .min_by_key(|(_, observation)| observation.observed_at) + .map(|(model_id, _)| model_id.clone()) + { + timings.remove(&oldest); + } + let observation = + timings + .entry(config.model_id.clone()) + .or_insert(StageDecodeTimingObservation { + observed_us_per_layer: sample, + sample_count: 0, + observed_at: now, + layer_start: config.layer_start, + layer_end: config.layer_end, + }); + if observation.sample_count < MAX_EFFECTIVE_SAMPLES { + let next_count = observation.sample_count + 1; + observation.observed_us_per_layer = observation + .observed_us_per_layer + .saturating_mul(observation.sample_count) + .saturating_add(sample) + / next_count; + observation.sample_count = next_count; + } else { + // Retain a bounded EWMA after the initial arithmetic-mean window. + observation.observed_us_per_layer = observation + .observed_us_per_layer + .saturating_mul(7) + .saturating_add(sample) + / 8; + } + observation.observed_at = now; +} + +pub fn stage_decode_timing_hints() -> Vec { + let Some(timings) = STAGE_DECODE_TIMINGS.get() else { + return Vec::new(); + }; + let mut timings = timings + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + timings.retain(|_, observation| observation.observed_at.elapsed() <= MAX_OBSERVATION_AGE); + let mut hints = timings + .iter() + .filter_map(|(model_id, observation)| { + let age = observation.observed_at.elapsed(); + (age <= MAX_OBSERVATION_AGE).then(|| StageDecodeTimingHint { + model_id: model_id.clone(), + observed_us_per_layer: observation.observed_us_per_layer, + sample_count: observation.sample_count, + sample_age_ms: u64::try_from(age.as_millis()).unwrap_or(u64::MAX), + }) + }) + .collect::>(); + hints.sort_by(|left, right| left.model_id.cmp(&right.model_id)); + hints +} + +#[cfg(test)] +mod tests { + use super::*; + use skippy_protocol::binary::{StageStateHeader, WireMessageKind}; + + fn config(model_id: &str) -> StageConfig { + StageConfig { + model_id: model_id.to_string(), + layer_start: 10, + layer_end: 20, + ..StageConfig::default() + } + } + + fn message(decode_step: i32) -> StageWireMessage { + StageWireMessage { + kind: WireMessageKind::DecodeEmbd, + pos_start: 0, + token_count: 1, + state: StageStateHeader { + decode_step, + ..StageStateHeader::new(WireMessageKind::DecodeEmbd) + }, + request_id: 1, + session_id: 1, + sampling: None, + chat_sampling_metadata: None, + tokens: vec![1], + positions: vec![0], + activation: Vec::new(), + raw_bytes: Vec::new(), + } + } + + #[test] + fn records_only_steady_decode_and_normalizes_by_layers() { + let model_id = format!("timing-test-{}", std::process::id()); + let config = config(&model_id); + record_stage_decode_timing(&config, &message(7), 10.0); + assert!( + stage_decode_timing_hints() + .iter() + .all(|hint| hint.model_id != model_id) + ); + + record_stage_decode_timing(&config, &message(8), 10.0); + let hint = stage_decode_timing_hints() + .into_iter() + .find(|hint| hint.model_id == model_id) + .expect("steady timing hint"); + assert_eq!(hint.observed_us_per_layer, 1_000); + assert_eq!(hint.sample_count, 1); + } + + #[test] + fn normalizes_batched_decode_by_executed_token_count() { + let model_id = format!("timing-batch-test-{}", std::process::id()); + let config = config(&model_id); + let mut batched = message(8); + batched.token_count = 4; + + record_stage_decode_timing(&config, &batched, 40.0); + let hint = stage_decode_timing_hints() + .into_iter() + .find(|hint| hint.model_id == model_id) + .expect("batched timing hint"); + assert_eq!(hint.observed_us_per_layer, 1_000); + } + + #[test] + fn changing_stage_range_resets_the_observation_window() { + let model_id = format!("timing-range-test-{}", std::process::id()); + let first = config(&model_id); + record_stage_decode_timing(&first, &message(8), 10.0); + + let mut changed = first.clone(); + changed.layer_end = 15; + record_stage_decode_timing(&changed, &message(8), 20.0); + + let hint = stage_decode_timing_hints() + .into_iter() + .find(|hint| hint.model_id == model_id) + .expect("changed-range timing hint"); + assert_eq!(hint.observed_us_per_layer, 4_000); + assert_eq!(hint.sample_count, 1); + } +} diff --git a/crates/skippy-topology-sim/Cargo.toml b/crates/skippy-topology-sim/Cargo.toml new file mode 100644 index 0000000000..a9e4dd5ce6 --- /dev/null +++ b/crates/skippy-topology-sim/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "skippy-topology-sim" +edition.workspace = true +license.workspace = true +version.workspace = true +description = "Scenario-driven placement simulator for the performance-aware topology planner" +repository = "https://github.com/Mesh-LLM/mesh-llm" +homepage = "https://github.com/Mesh-LLM/mesh-llm" + +[dependencies] +thiserror = "2" +skippy-coordinator = { path = "../skippy-coordinator" } +serde.workspace = true +toml = "1.1" + +[dev-dependencies] +serde_json.workspace = true diff --git a/crates/skippy-topology-sim/scenarios/benchmarks_anchor_pair.toml b/crates/skippy-topology-sim/scenarios/benchmarks_anchor_pair.toml new file mode 100644 index 0000000000..20996293c9 --- /dev/null +++ b/crates/skippy-topology-sim/scenarios/benchmarks_anchor_pair.toml @@ -0,0 +1,67 @@ +# Calibration anchor from docs/BENCHMARKS.md: GLM-4.7-Flash-Q4_K_M (17 GB) +# on an M4 Max (68 tok/s solo) + Mac mini M4 over Wi-Fi. +# Measured anchors: solo 68, 2-way split (85/15) 21, 3-way split (62/31/8) +# 12-13 tok/s. BENCHMARKS.md calls these "a quick reality check". +# +# Calibration method (recorded so future measurements tighten it): +# - active_weight_fraction 0.34: GLM-4.7-Flash is MoE; the solo anchor +# (68 tok/s => 14.7 ms/token) fixes streamed bytes at ~5.8 GB/token +# given the M4 Max's ~394 GB/s effective Q4 decode bandwidth. +# - per_hop_overhead_ms 13.0: back-solved from the 2-way anchor. This is +# per-token wire time beyond RTT+transfer: QUIC stream setup, copies, +# scheduling. The BENCHMARKS.md note "overhead dominated by per-token +# RPC latency" is this coefficient. Passive edge measurements +# (LargeFrameObservation) will refine it with real per-edge data. +# - 3-way uses a second mini-class node; residual error <= ~10% on all +# three anchors. +[nodes.m4max] +vram_bytes = 68719476736 # 64 GiB unified +sustained_mem_bandwidth_mib_per_s = 417000 +sustained_compute_gflop_per_s = 34000 + +[nodes.mini] +vram_bytes = 34359738368 # 32 GiB unified +sustained_mem_bandwidth_mib_per_s = 160000 +sustained_compute_gflop_per_s = 10200 + +[nodes.mini2] +vram_bytes = 34359738368 +sustained_mem_bandwidth_mib_per_s = 160000 +sustained_compute_gflop_per_s = 10200 + +[links."m4max -> mini"] +rtt_ms = 3 +large_frame_mib_per_s = 30 + +[links."mini -> m4max"] +rtt_ms = 3 +large_frame_mib_per_s = 30 + +[links."m4max -> mini2"] +rtt_ms = 3 +large_frame_mib_per_s = 30 + +[links."mini2 -> m4max"] +rtt_ms = 3 +large_frame_mib_per_s = 30 + +[links."mini -> mini2"] +rtt_ms = 4 +large_frame_mib_per_s = 30 + +[links."mini2 -> mini"] +rtt_ms = 4 +large_frame_mib_per_s = 30 + +[model] +layer_count = 40 +weight_bytes_per_layer = 425000000 # ~17 GB / 40 layers +kv_bytes_per_token = 4096 +native_context_length = 65536 +activation_frame_bytes = 8192 +active_weight_fraction = 0.34 +per_stage_overhead_ms = 1.3 +per_hop_overhead_ms = 13.0 + +[workload] +minimum_nodes = 1 diff --git a/crates/skippy-topology-sim/scenarios/cross_continent_chain.toml b/crates/skippy-topology-sim/scenarios/cross_continent_chain.toml new file mode 100644 index 0000000000..34915e5c4e --- /dev/null +++ b/crates/skippy-topology-sim/scenarios/cross_continent_chain.toml @@ -0,0 +1,53 @@ +# Scenario 4 from the design corpus: three nodes chained across long links. +# The planner must either reject the 33 ms TPOT target honestly (estimate +# above target) or find the fewest-hop layout; it must not claim the target +# is met when per-hop latency alone exceeds it. +[nodes.west] +vram_bytes = 42949672960 # 40 GB +sustained_mem_bandwidth_mib_per_s = 1858000 +sustained_compute_gflop_per_s = 312000 + +[nodes.central] +vram_bytes = 42949672960 +sustained_mem_bandwidth_mib_per_s = 1858000 +sustained_compute_gflop_per_s = 312000 + +[nodes.east] +vram_bytes = 42949672960 +sustained_mem_bandwidth_mib_per_s = 1858000 +sustained_compute_gflop_per_s = 312000 + +[links."west -> central"] +rtt_ms = 70 +large_frame_mib_per_s = 10 + +[links."central -> west"] +rtt_ms = 70 +large_frame_mib_per_s = 10 + +[links."central -> east"] +rtt_ms = 80 +large_frame_mib_per_s = 8 + +[links."east -> central"] +rtt_ms = 80 +large_frame_mib_per_s = 8 + +[links."west -> east"] +rtt_ms = 140 +large_frame_mib_per_s = 4 + +[links."east -> west"] +rtt_ms = 140 +large_frame_mib_per_s = 4 + +[model] +layer_count = 48 +weight_bytes_per_layer = 1610612736 # 72 GiB model +kv_bytes_per_token = 4096 +native_context_length = 65536 +activation_frame_bytes = 8192 + +[workload] +minimum_nodes = 3 +target_decode_tpot_ms = 33 diff --git a/crates/skippy-topology-sim/scenarios/heterogeneous_pair.toml b/crates/skippy-topology-sim/scenarios/heterogeneous_pair.toml new file mode 100644 index 0000000000..3fcafbe058 --- /dev/null +++ b/crates/skippy-topology-sim/scenarios/heterogeneous_pair.toml @@ -0,0 +1,31 @@ +# Scenario 2 from the design doc's corpus: the M4 Max + Mac mini Wi-Fi pair +# behind docs/BENCHMARKS.md's 68 -> 21 tok/s 2-way anchor (calibration target). +# Values are spec-class priors until gpu-bench measurements replace them. +[nodes.alpha] +vram_bytes = 68719476736 # 64 GiB unified +sustained_mem_bandwidth_mib_per_s = 546000 +sustained_compute_gflop_per_s = 34000 + +[nodes.beta] +vram_bytes = 34359738368 # 32 GiB unified +sustained_mem_bandwidth_mib_per_s = 204000 +sustained_compute_gflop_per_s = 10200 + +[links."alpha -> beta"] +rtt_ms = 3 +large_frame_mib_per_s = 30 # Wi-Fi 6 large-frame prior + +[links."beta -> alpha"] +rtt_ms = 3 +large_frame_mib_per_s = 30 + +[model] +layer_count = 40 +weight_bytes_per_layer = 1610612736 # 1.5 GiB (60 GiB model) +kv_bytes_per_token = 4096 +native_context_length = 65536 +activation_frame_bytes = 8192 # 4096-wide f16 frame + +[workload] +minimum_nodes = 2 +target_decode_tpot_ms = 33 diff --git a/crates/skippy-topology-sim/scenarios/straggler_triplet.toml b/crates/skippy-topology-sim/scenarios/straggler_triplet.toml new file mode 100644 index 0000000000..e0b4299728 --- /dev/null +++ b/crates/skippy-topology-sim/scenarios/straggler_triplet.toml @@ -0,0 +1,44 @@ +# Scenario 3 from the design corpus: A100 + RTX 4090 + laptop CPU. The laptop +# is VRAM-rich relative to its bandwidth; perf-aware planning must give it few +# layers (or effectively exclude it via span size) despite its memory. +[nodes.datacenter] +vram_bytes = 85899345920 # A100 80 GB +sustained_mem_bandwidth_mib_per_s = 1858000 # ~1.9 TB/s, GB->MiB converted +sustained_compute_gflop_per_s = 312000 + +[nodes.prosumer] +vram_bytes = 25769803776 # 4090 24 GB +sustained_mem_bandwidth_mib_per_s = 962000 +sustained_compute_gflop_per_s = 83000 + +[nodes.laptop] +vram_bytes = 34359738368 # 32 GB unified, CPU-only inference +sustained_mem_bandwidth_mib_per_s = 76000 # ~80 GB/s LPDDR5 prior +sustained_compute_gflop_per_s = 300 + +[links."datacenter -> prosumer"] +rtt_ms = 1 +large_frame_mib_per_s = 900 # 10 GbE prior + +[links."prosumer -> datacenter"] +rtt_ms = 1 +large_frame_mib_per_s = 900 + +[links."prosumer -> laptop"] +rtt_ms = 4 +large_frame_mib_per_s = 30 # Wi-Fi prior + +[links."laptop -> prosumer"] +rtt_ms = 4 +large_frame_mib_per_s = 30 + +[model] +layer_count = 62 +weight_bytes_per_layer = 1610612736 # 1.5 GiB (~93 GiB model) +kv_bytes_per_token = 8192 +native_context_length = 65536 +activation_frame_bytes = 4096 + +[workload] +minimum_nodes = 2 +target_decode_tpot_ms = 33 diff --git a/crates/skippy-topology-sim/src/execution.rs b/crates/skippy-topology-sim/src/execution.rs new file mode 100644 index 0000000000..e137ebbbdc --- /dev/null +++ b/crates/skippy-topology-sim/src/execution.rs @@ -0,0 +1,148 @@ +//! Discrete pipeline execution model over a planned topology. +//! +//! This layer answers "what tok/s will this plan actually deliver" — +//! complementing the placement layer (which decides the plan) with an +//! execution estimate per plan, calibrated against the measured anchors in +//! `docs/BENCHMARKS.md`. +//! +//! Two decode regimes: +//! +//! - **Serial** (single stream / `parallel_lanes == 1`): autoregressive +//! decode depends on the previous token's logits, so every token +//! traverses every stage and returns. TPOT = Σ stage service times + +//! Σ edge times (including the prediction-return hop). This is why the +//! measured anchors drop 68 → 21 → 12-13 tok/s across 1/2/3-way splits. +//! - **Pipelined** (`parallel_lanes > 1`): stages process consecutive +//! tokens concurrently; throughput is bounded by the slowest stage plus +//! its egress edge. TPOT per lane ≈ max stage+edge time. +//! +//! Per-stage service time is weight-streaming: the bytes a stage must read +//! from memory per token, divided by that node's sustained bandwidth. For +//! dense models that is the stage's parameter bytes; for MoE models only +//! the active expert bytes are touched per token (the calibration anchor +//! GLM-4.7-Flash streams ~7.2 GB/token of its ~17 GB). + +use crate::{Scenario, ScenarioLink}; + +/// Modeled tok/s and per-token breakdown for one executed plan. +#[derive(Clone, Debug, PartialEq)] +pub struct ExecutionEstimate { + /// Steady-state tokens per second for a single decoding stream. + pub serial_tok_s: f64, + /// Steady-state tokens per second per lane when lanes run concurrently + /// (pipelined regime). `None` when the plan lacks the bandwidth signals + /// to model pipelining. + pub pipelined_tok_s_per_lane: Option, + /// Total serial per-token time in microseconds (all stages + all hops). + pub serial_token_us: u64, + /// Per-stage service time in microseconds, in stage order. + pub stage_service_us: Vec, + /// Per-hop time in microseconds (activation transfer + RTT), in order, + /// including the final prediction-return hop. + pub hop_us: Vec, +} + +impl Scenario { + /// Bytes a node streams from memory per decoded token for a stage of + /// `layer_count` layers. Dense models touch every parameter; MoE models + /// touch only the active fraction, expressed as `active_weight_fraction` + /// (0.0-1.0; 1.0 = dense). + fn stage_streamed_bytes(&self, layer_count: u32) -> Option { + let fraction = self.model.active_weight_fraction.unwrap_or(1.0); + if !(0.0..=1.0).contains(&fraction) { + return None; + } + Some((self.model.weight_bytes_per_layer as f64 * f64::from(layer_count) * fraction) as u64) + } + + /// Directed link lookup between two nodes (exact direction, then the + /// reverse as a symmetric fallback). + fn link(&self, source: &str, target: &str) -> Option<&ScenarioLink> { + self.links + .get(&format!("{source} -> {target}")) + .or_else(|| self.links.get(&format!("{target} -> {source}"))) + } + + /// Estimate execution of a planned topology (stages as + /// `(node_id, layer_count)` in pipeline order). + pub fn estimate_execution(&self, stages: &[(&str, u32)]) -> Result { + if stages.is_empty() { + return Err("no stages".to_string()); + } + let mut stage_service_us = Vec::with_capacity(stages.len()); + let stage_overhead_us = (self.model.per_stage_overhead_ms * 1_000.0) as u64; + for (node_id, layers) in stages { + let node = self + .nodes + .get(*node_id) + .ok_or_else(|| format!("unknown node {node_id}"))?; + let bandwidth = node + .sustained_mem_bandwidth_mib_per_s + .ok_or_else(|| format!("node {node_id} lacks bandwidth signal"))?; + if bandwidth == 0 { + return Err(format!("node {node_id} reports zero bandwidth")); + } + let streamed = self + .stage_streamed_bytes(*layers) + .ok_or("invalid active_weight_fraction")?; + // bytes / (MiB/s) in microseconds, plus calibrated per-stage + // software overhead. + stage_service_us.push( + (streamed as f64 * 1_000_000.0 / (f64::from(bandwidth) * 1_048_576.0)) as u64 + + stage_overhead_us, + ); + } + + let hop_overhead_us = (self.model.per_hop_overhead_ms * 1_000.0) as u64; + let mut hop_us = Vec::new(); + for window in stages.windows(2) { + let link = self + .link(window[0].0, window[1].0) + .ok_or_else(|| format!("missing link {} -> {}", window[0].0, window[1].0))?; + let mut us = u64::from(link.rtt_ms) * 1_000 + hop_overhead_us; + if let Some(bandwidth) = link.large_frame_mib_per_s.filter(|b| *b > 0) { + let frame = self.model.activation_frame_bytes; + if frame > 0 { + us += + (frame as f64 * 1_000_000.0 / (f64::from(bandwidth) * 1_048_576.0)) as u64; + } + } + hop_us.push(us); + } + // Prediction-return hop: final stage back to stage 0. + if stages.len() > 1 { + let (last, first) = (stages[stages.len() - 1].0, stages[0].0); + let link = self + .link(last, first) + .ok_or_else(|| format!("missing return link {last} -> {first}"))?; + let mut us = u64::from(link.rtt_ms) * 1_000 + hop_overhead_us; + if let Some(bandwidth) = link.large_frame_mib_per_s.filter(|b| *b > 0) { + let frame = self.model.activation_frame_bytes; + if frame > 0 { + us += + (frame as f64 * 1_000_000.0 / (f64::from(bandwidth) * 1_048_576.0)) as u64; + } + } + hop_us.push(us); + } + + let serial_us: u64 = stage_service_us.iter().sum::() + hop_us.iter().sum::(); + let serial_tok_s = 1_000_000.0 / serial_us as f64; + // Pipelined regime: throughput bounded by the slowest stage+egress + // pair. With no hops (single stage) it is just the stage time. + let pipelined_us = stage_service_us + .iter() + .enumerate() + .map(|(index, stage)| stage + hop_us.get(index).copied().unwrap_or(0)) + .max() + .ok_or("no stages")?; + let pipelined_tok_s_per_lane = 1_000_000.0 / pipelined_us as f64; + Ok(ExecutionEstimate { + serial_tok_s, + pipelined_tok_s_per_lane: Some(pipelined_tok_s_per_lane), + serial_token_us: serial_us, + stage_service_us, + hop_us, + }) + } +} diff --git a/crates/skippy-topology-sim/src/lib.rs b/crates/skippy-topology-sim/src/lib.rs new file mode 100644 index 0000000000..1f23987628 --- /dev/null +++ b/crates/skippy-topology-sim/src/lib.rs @@ -0,0 +1,370 @@ +//! Scenario-driven placement simulator for the performance-aware topology +//! planner (`docs/design/PERFORMANCE_AWARE_TOPOLOGY_PLANNER.md`). +//! +//! A scenario is a TOML file describing nodes, directed links, a model +//! package, and workload intent. The simulator feeds the scenario into +//! [`skippy_coordinator::topology::plan_topology`] and scores the resulting +//! plan with the same cost model the planner uses, so planner decisions can +//! be asserted against expectations ("a faster node receives as much work as +//! its capacity allows", "a slow link rejects the TPOT target") in CI without +//! a cluster. +//! +//! The [`execution`](execution) layer adds a discrete pipeline model over a +//! chosen plan: per-stage service times from streamed weight bytes and +//! measured bandwidth, per-hop latency + activation transfer, serial vs +//! pipelined decode regimes, calibrated against `docs/BENCHMARKS.md`. + +use serde::Deserialize; +use skippy_coordinator::topology::{ + TopologyEdge, TopologyNode, TopologyPlanningInput, plan_topology, +}; + +pub mod execution; + +/// One candidate node in a scenario. +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ScenarioNode { + pub vram_bytes: u64, + /// Sustained memory bandwidth in MiB/s (`None` = unreported signal). + #[serde(default)] + pub sustained_mem_bandwidth_mib_per_s: Option, + /// Sustained fp16 compute in GFLOP/s (`None` = unreported signal). + #[serde(default)] + pub sustained_compute_gflop_per_s: Option, + /// Observed steady-decode runtime work in microseconds per loaded layer. + #[serde(default)] + pub observed_decode_us_per_layer: Option, +} + +/// One directed link between scenario nodes. Keys are `" -> "`. +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ScenarioLink { + pub rtt_ms: u32, + #[serde(default)] + pub large_frame_mib_per_s: Option, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ScenarioModel { + pub layer_count: u32, + pub weight_bytes_per_layer: u64, + pub kv_bytes_per_token: u64, + pub native_context_length: u32, + #[serde(default)] + pub activation_frame_bytes: u64, + /// Fraction of weight bytes actually streamed per token (MoE active + /// experts / dense). 1.0 (default) = dense; the GLM-4.7-Flash anchor + /// implies ~0.34. Only used by the execution layer, not placement. + #[serde(default)] + pub active_weight_fraction: Option, + /// Calibration knob: fixed per-stage per-token software overhead + /// (dispatch, kernel launch, sync) in milliseconds. Execution layer only. + #[serde(default)] + pub per_stage_overhead_ms: f64, + /// Calibration knob: fixed per-hop per-token software overhead (QUIC + /// stream, copies, scheduling) in milliseconds, on top of RTT and + /// activation transfer. Execution layer only. + #[serde(default)] + pub per_hop_overhead_ms: f64, +} + +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ScenarioWorkload { + #[serde(default)] + pub minimum_nodes: Option, + #[serde(default)] + pub target_decode_tpot_ms: Option, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Scenario { + pub nodes: std::collections::BTreeMap, + #[serde(default)] + pub links: std::collections::BTreeMap, + pub model: ScenarioModel, + #[serde(default)] + pub workload: ScenarioWorkload, +} + +#[derive(Debug, thiserror::Error)] +pub enum ScenarioError { + #[error("scenario parse error: {0}")] + Parse(#[from] toml::de::Error), + #[error( + "unknown top-level scenario key `{key}` — links must be declared as `[links.\"a -> b\"]` tables, not top-level keys" + )] + UnknownTopLevelKey { key: String }, + #[error("malformed link key `{key}` — expected exactly one ` -> ` separating two node ids")] + MalformedLinkKey { key: String }, +} + +impl Scenario { + pub fn from_toml(input: &str) -> Result { + let value: toml::Value = toml::from_str(input)?; + for key in value.as_table().into_iter().flat_map(|table| table.keys()) { + if !matches!(key.as_str(), "nodes" | "links" | "model" | "workload") { + return Err(ScenarioError::UnknownTopLevelKey { key: key.clone() }); + } + } + let scenario: Scenario = toml::from_str(input)?; + // Validate link keys now so malformed edges fail at parse time, + // not silently at planning time. + for key in scenario.links.keys() { + parse_link_key(key)?; + } + Ok(scenario) + } + + /// Build the coordinator planning input for this scenario. + pub fn planning_input(&self) -> TopologyPlanningInput { + let nodes = self + .nodes + .iter() + .map(|(id, node)| TopologyNode { + node_id: id.clone(), + detected_vram_bytes: node.vram_bytes, + max_vram_bytes: None, + runtime_headroom_bytes: node.vram_bytes / 10, + stage_transfer_latency_ms: self.node_latency_ms(id), + sustained_mem_bandwidth_mib_per_s: node.sustained_mem_bandwidth_mib_per_s, + sustained_compute_gflop_per_s: node.sustained_compute_gflop_per_s, + observed_decode_us_per_layer: node.observed_decode_us_per_layer, + }) + .collect::>(); + let edges = self + .links + .iter() + .map(|(key, link)| { + // Keys are validated at parse time; a malformed key here is + // a programming error, not scenario content. + let (source, target) = parse_link_key(key).expect("validated link key"); + TopologyEdge { + source_node_id: source, + target_node_id: target, + rtt_ms: link.rtt_ms, + large_frame_mib_per_s: link.large_frame_mib_per_s, + } + }) + .collect::>(); + TopologyPlanningInput { + native_context_length: self.model.native_context_length, + layer_count: self.model.layer_count, + model_weight_bytes: self.model.weight_bytes_per_layer + * u64::from(self.model.layer_count), + layer_weight_bytes: Vec::new(), + kv_bytes_per_token: self.model.kv_bytes_per_token, + recurrent_bytes_per_sequence_by_layer: Vec::new(), + reserved_sequence_ids: 16, + minimum_nodes: self.workload.minimum_nodes.unwrap_or(1), + nodes, + context_length_override: None, + parallel_lanes_override: None, + target_decode_tpot_ms: self.workload.target_decode_tpot_ms, + active_weight_fraction_permil: ((self.model.active_weight_fraction.unwrap_or(1.0) + * 1000.0) + .round() as u32) + .clamp(1, 1000), + edges, + activation_frame_bytes: self.model.activation_frame_bytes, + } + } + + /// Minimum observed RTT involving this node, used as the node's + /// coordinator-RTT stand-in when links are present. + fn node_latency_ms(&self, node_id: &str) -> Option { + let mut best: Option = None; + for (key, link) in &self.links { + let (source, target) = parse_link_key(key).expect("validated link key"); + if source == node_id || target == node_id { + best = Some(best.map_or(link.rtt_ms, |current| current.min(link.rtt_ms))); + } + } + best + } + + /// Plan and score the scenario, returning the chosen plan plus modeled + /// per-stage service times for assertions. + pub fn plan(&self) -> Result { + plan_topology(&self.planning_input()).map_err(|error| error.to_string()) + } +} + +fn parse_link_key(key: &str) -> Result<(String, String), ScenarioError> { + // Accept exactly one "->" separating two non-empty node ids; anything + // else is a malformed key that would silently produce unusable edges. + let parts: Vec<&str> = key.split("->").collect(); + if parts.len() == 2 { + let source = parts[0].trim(); + let target = parts[1].trim(); + if !source.is_empty() + && !target.is_empty() + && !source.contains(' ') + && !target.contains(' ') + { + return Ok((source.to_string(), target.to_string())); + } + } + Err(ScenarioError::MalformedLinkKey { + key: key.to_string(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + const HETEROGENEOUS_PAIR: &str = r#" +[nodes.alpha] +vram_bytes = 68719476736 # 64 GiB +sustained_mem_bandwidth_mib_per_s = 546000 +sustained_compute_gflop_per_s = 34000 + +[nodes.beta] +vram_bytes = 51539607552 +sustained_mem_bandwidth_mib_per_s = 273000 +sustained_compute_gflop_per_s = 17000 + +[links."alpha -> beta"] +rtt_ms = 2 + +[links."beta -> alpha"] +rtt_ms = 2 + +[model] +layer_count = 40 +weight_bytes_per_layer = 1610612736 # 1.5 GiB +kv_bytes_per_token = 4096 +native_context_length = 65536 + +[workload] +minimum_nodes = 2 +target_decode_tpot_ms = 33 +"#; + + #[test] + fn link_tables_outside_links_fail_loudly() { + // `["a -> b"]` at top level parses as a quoted *key* named + // "a -> b", not a links entry — historically this silently + // dropped the link. The schema now rejects unknown top-level + // keys so misdeclared links cannot hide. + let scenario = r#" +[nodes.alpha] +vram_bytes = 68719476736 +sustained_mem_bandwidth_mib_per_s = 546000 + +[nodes.beta] +vram_bytes = 51539607552 +sustained_mem_bandwidth_mib_per_s = 273000 + +["alpha -> beta"] +rtt_ms = 2 + +[model] +layer_count = 40 +weight_bytes_per_layer = 1610612736 +kv_bytes_per_token = 4096 +native_context_length = 65536 + +[workload] +minimum_nodes = 2 +"#; + let error = + Scenario::from_toml(scenario).expect_err("misdeclared top-level link must be rejected"); + assert!(error.to_string().contains("unknown top-level scenario key")); + } + + #[test] + fn malformed_link_keys_fail_loudly() { + for key in ["alpha", "alpha -> beta -> gamma", " -> beta", "alpha -> "] { + let scenario = format!( + "[nodes.alpha]\nvram_bytes = 68719476736\n\ + [nodes.beta]\nvram_bytes = 51539607552\n\ + [links.\"{key}\"]\nrtt_ms = 2\n\ + [model]\nlayer_count = 40\nweight_bytes_per_layer = 1610612736\n\ + kv_bytes_per_token = 4096\nnative_context_length = 65536\n\ + [workload]\nminimum_nodes = 2\n" + ); + let error = Scenario::from_toml(&scenario) + .err() + .unwrap_or_else(|| panic!("malformed link key `{key}` must be rejected")); + assert!( + error.to_string().contains("malformed link key"), + "key `{key}`: {error}" + ); + } + } + + #[test] + fn unknown_nested_keys_fail_loudly() { + let scenario = HETEROGENEOUS_PAIR.replace( + "sustained_mem_bandwidth_mib_per_s = 546000", + "sustained_mem_bandwith_mib_per_s = 546000", + ); + let error = Scenario::from_toml(&scenario) + .expect_err("a misspelled nested performance field must be rejected"); + assert!( + error.to_string().contains("unknown field"), + "unexpected parse error: {error}" + ); + } + + #[test] + fn single_stream_objective_prefers_the_faster_node() { + let scenario = Scenario::from_toml(HETEROGENEOUS_PAIR).expect("scenario"); + let plan = scenario.plan().expect("plan"); + assert_eq!(plan.stages.len(), 2); + let alpha = plan + .stages + .iter() + .find(|stage| stage.node_id == "alpha") + .expect("alpha stage"); + let beta = plan + .stages + .iter() + .find(|stage| stage.node_id == "beta") + .expect("beta stage"); + let alpha_layers = alpha.layer_end - alpha.layer_start; + let beta_layers = beta.layer_end - beta.layer_start; + assert_eq!(alpha_layers + beta_layers, 40, "all layers placed"); + assert!( + alpha_layers > beta_layers, + "faster alpha should carry more work" + ); + assert!( + beta_layers > 0, + "every selected stage must remain non-empty" + ); + } + + #[test] + fn missing_signal_node_keeps_capacity_only_placement() { + let scenario = Scenario::from_toml(HETEROGENEOUS_PAIR).expect("scenario"); + let mut input = scenario.planning_input(); + input.nodes[0].sustained_mem_bandwidth_mib_per_s = None; + let plan = plan_topology(&input).expect("plan"); + let alpha = plan + .stages + .iter() + .find(|stage| stage.node_id == "alpha") + .expect("alpha stage"); + let beta = plan + .stages + .iter() + .find(|stage| stage.node_id == "beta") + .expect("beta stage"); + // Without complete signals the exact capacity-greedy fallback fills + // the first node to its memory ceiling and hands the remainder to the + // second. The fallback contract is locked more precisely in the + // coordinator package; this scenario guards full, non-empty coverage. + let alpha_layers = alpha.layer_end - alpha.layer_start; + let beta_layers = beta.layer_end - beta.layer_start; + assert_eq!(alpha_layers + beta_layers, 40, "all layers placed"); + assert!(alpha_layers > 0 && beta_layers > 0); + assert!(alpha_layers > beta_layers); + } +} diff --git a/crates/skippy-topology-sim/tests/calibration.rs b/crates/skippy-topology-sim/tests/calibration.rs new file mode 100644 index 0000000000..35758dbc1a --- /dev/null +++ b/crates/skippy-topology-sim/tests/calibration.rs @@ -0,0 +1,109 @@ +//! Execution-model calibration against the measured anchors in +//! `docs/BENCHMARKS.md` (GLM-4.7-Flash-Q4_K_M on M4 Max + Mac mini over +//! Wi-Fi: solo 68 tok/s, 2-way split 21, 3-way split 12-13). +//! +//! Tolerance is +/-15% per anchor: BENCHMARKS.md itself calls the numbers +//! "a quick reality check". If these tests drift, either the model or the +//! calibration coefficients in the anchor scenario need updating — and a +//! model that cannot reproduce the anchors must not drive placement +//! decisions. +//! +//! `planner_model_matches_execution_sim` additionally locks the +//! *coordinator planner's* modeled decode TPOT to the calibrated execution +//! sim on the same scenario: the two cost models must agree, or the +//! planner will rank candidates against numbers nobody has validated. + +use skippy_topology_sim::Scenario; + +fn load(name: &str) -> Scenario { + let path = format!("{}/scenarios/{name}", env!("CARGO_MANIFEST_DIR")); + let raw = std::fs::read_to_string(&path).unwrap_or_else(|error| panic!("read {path}: {error}")); + Scenario::from_toml(&raw).unwrap_or_else(|error| panic!("parse {path}: {error}")) +} + +fn assert_within(anchor: f64, modeled: f64, label: &str) { + let tolerance = anchor * 0.15; + assert!( + (modeled - anchor).abs() <= tolerance, + "{label}: modeled {modeled:.1} tok/s vs anchor {anchor} (±{tolerance:.1})" + ); +} + +#[test] +fn reproduces_solo_anchor() { + let scenario = load("benchmarks_anchor_pair.toml"); + let estimate = scenario + .estimate_execution(&[("m4max", 40)]) + .expect("execution estimate"); + assert_within(68.0, estimate.serial_tok_s, "solo M4 Max"); +} + +#[test] +fn reproduces_two_way_split_anchor() { + let scenario = load("benchmarks_anchor_pair.toml"); + // BENCHMARKS.md 2-way split is 85/15: 34/6 of 40 layers. + let estimate = scenario + .estimate_execution(&[("m4max", 34), ("mini", 6)]) + .expect("execution estimate"); + assert_within(21.0, estimate.serial_tok_s, "2-way split"); + // The serial regime must be strictly slower than the pipelined + // estimate — that difference is why splits cost single-stream decode. + assert!(estimate.serial_tok_s < estimate.pipelined_tok_s_per_lane.unwrap()); +} + +#[test] +fn reproduces_three_way_split_anchor() { + let scenario = load("benchmarks_anchor_pair.toml"); + // BENCHMARKS.md 3-way split is 62/31/8: 25/12/3 of 40 layers. + let estimate = scenario + .estimate_execution(&[("m4max", 25), ("mini", 12), ("mini2", 3)]) + .expect("execution estimate"); + assert_within(13.0, estimate.serial_tok_s, "3-way split (12-13 anchor)"); +} + +#[test] +fn monotonically_worse_with_more_hops() { + // Sanity property: adding hops must never improve single-stream decode. + let scenario = load("benchmarks_anchor_pair.toml"); + let solo = scenario.estimate_execution(&[("m4max", 40)]).unwrap(); + let two = scenario + .estimate_execution(&[("m4max", 34), ("mini", 6)]) + .unwrap(); + let three = scenario + .estimate_execution(&[("m4max", 25), ("mini", 12), ("mini2", 3)]) + .unwrap(); + assert!(solo.serial_tok_s > two.serial_tok_s); + assert!(two.serial_tok_s > three.serial_tok_s); +} + +#[test] +fn planner_model_matches_execution_sim() { + // The coordinator planner's modeled decode TPOT must equal the + // calibrated execution sim's serial estimate for the same stage + // assignment. The planner previously used a different formula + // (bottleneck-stage + network) with no calibrated overhead terms and + // no calibration test at all — this locks the two cost models together + // on the anchor scenario so a future divergence fails CI instead of + // silently mis-ranking candidates. + let scenario = load("benchmarks_anchor_pair.toml"); + let plan = scenario.plan().expect("planner plan"); + let chosen: Vec<(&str, u32)> = plan + .stages + .iter() + .map(|stage| (stage.node_id.as_str(), stage.layer_end - stage.layer_start)) + .collect(); + let sim = scenario + .estimate_execution(&chosen) + .expect("sim estimate for the planner-chosen assignment"); + let planner_us = plan + .modeled_decode_tpot_us + .expect("planner models TPOT for the anchor scenario (all nodes signaled)"); + let sim_us = u128::from(sim.serial_token_us); + // Both models must agree within 1% — they use the same terms; any + // larger gap is a formula divergence, not calibration noise. + let tolerance = sim_us / 100; + assert!( + planner_us.abs_diff(sim_us) <= tolerance, + "planner TPOT {planner_us} µs vs sim {sim_us} µs for {chosen:?} (divergence beyond 1%)" + ); +} diff --git a/crates/skippy-topology-sim/tests/scenarios.rs b/crates/skippy-topology-sim/tests/scenarios.rs new file mode 100644 index 0000000000..c4daca5fc3 --- /dev/null +++ b/crates/skippy-topology-sim/tests/scenarios.rs @@ -0,0 +1,66 @@ +//! Corpus scenarios run end-to-end through the real planner. Each asserts +//! the behavioral property the scenario exists to guard (see the design +//! doc's scenario corpus section). + +use skippy_topology_sim::Scenario; + +fn load(name: &str) -> Scenario { + let path = format!("{}/scenarios/{name}", env!("CARGO_MANIFEST_DIR")); + let raw = std::fs::read_to_string(&path).unwrap_or_else(|error| panic!("read {path}: {error}")); + Scenario::from_toml(&raw).unwrap_or_else(|error| panic!("parse {path}: {error}")) +} + +#[test] +fn heterogeneous_pair_favors_the_faster_node() { + let scenario = load("heterogeneous_pair.toml"); + let plan = scenario.plan().expect("plan"); + assert_eq!(plan.stages.len(), 2); + let span = |id: &str| { + plan.stages + .iter() + .find(|stage| stage.node_id == id) + .map(|stage| stage.layer_end - stage.layer_start) + .unwrap_or(0) + }; + let (alpha, beta) = (span("alpha"), span("beta")); + assert_eq!(alpha + beta, 40); + assert!( + alpha > beta, + "faster alpha should carry more work: {alpha}/{beta}" + ); + assert!(beta > 0, "every selected stage must remain non-empty"); +} + +#[test] +fn straggler_triplet_limits_the_laptop() { + let scenario = load("straggler_triplet.toml"); + let plan = scenario.plan().expect("plan"); + let laptop = plan + .stages + .iter() + .find(|stage| stage.node_id == "laptop") + .expect("laptop participates"); + let laptop_layers = laptop.layer_end - laptop.layer_start; + assert!( + laptop_layers <= 4, + "laptop (80 GB/s) must receive at most a few layers: {laptop_layers}" + ); +} + +#[test] +fn cross_continent_chain_fails_tpot_target_honestly() { + let scenario = load("cross_continent_chain.toml"); + let plan = scenario.plan().expect("plan"); + let estimate = plan + .estimated_decode_network_ms_per_token + .expect("latency-aware plan carries an estimate"); + assert!( + estimate > 33, + "70-140 ms hops must yield an estimate above the 33 ms target: {estimate}" + ); + assert_eq!( + plan.decode_tpot_target_met, + Some(false), + "the plan must not claim the TPOT target is met" + ); +} diff --git a/docs/design/PERFORMANCE_AWARE_TOPOLOGY_PLANNER.md b/docs/design/PERFORMANCE_AWARE_TOPOLOGY_PLANNER.md new file mode 100644 index 0000000000..3d86ce1e7a --- /dev/null +++ b/docs/design/PERFORMANCE_AWARE_TOPOLOGY_PLANNER.md @@ -0,0 +1,495 @@ +# Performance-Aware Topology Planner and Placement Simulator + +## Status: Phases 0-3 implemented; 4-5 planned + +- Date: 2026-08-26 +- Owner: TBD +- Origin: skippy-topology channel discussion (2026-08-26); requested by James. +- Implementation: PR #1454 (branch `docs/perf-aware-topology-planner`). + Phases 0-3 (metric plumbing, perf-aware span assignment, directed-edge + network model, modeled-TPOT candidate selection, placement simulator + + scenario corpus, execution sim calibration, passive link measurement, + live per-stage timing feedback, and RTT-floor confidence) are implemented + and tested there. As-built notes are inline below. Phases 4-5 (reference- + hardware A/B and adaptive replanning) remain planned. + +## Problem + +Automatic split placement is decided by a capacity fitter that ignores +performance. The planner in `crates/skippy-coordinator/src/topology.rs` +distributes contiguous layer ranges using memory arithmetic only — +per-layer weights, KV bytes/token, recurrent state, a fixed compute reserve — +and its only network term is `stage_count × max(coordinator RTT)` +(`estimate_decode_network_ms_per_token`). Two consequences: + +1. **Node performance is invisible.** A node with half the sustained memory + bandwidth receives the same layer span as an equal-VRAM node and becomes + the pipeline straggler. Quantized decode is usually memory-bandwidth-bound, + so this is the dominant term, not FLOPs. +2. **Link performance is invisible.** Activation wire time is ignored + entirely; latency enters only as a scalar hop count times the worst + coordinator-measured RTT. Directed per-hop differences (e.g. an + asymmetric Wi-Fi hop, a cross-continent edge) do not influence stage + ordering or span sizes. + +Meanwhile the fleet already measures the missing signals: peers gossip +`gpu_mem_bandwidth_gbps` and `gpu_compute_tflops_fp16/fp32` +(`crates/mesh-llm-host-runtime/src/mesh/gossip.rs`), and the policy crate has +a directed edge model (`StageEdgeSignal { rtt_ms, large_frame_bytes_per_sec }` +in `crates/skippy-topology/src/edge_order.rs`) that automatic placement never +consults. The data exists; the planner does not receive it. + +Measured cost of getting this wrong (`docs/BENCHMARKS.md`): GLM-4.7-Flash +Q4_K_M on M4 Max + Mac mini over Wi-Fi runs 68 tok/s solo, 21 tok/s at a +2-way split, 12-13 tok/s at 3-way. Splitting always costs; splitting without +performance awareness costs more than it must. + +## Goal + +One production planner that takes a complete input set — memory, node +performance, directed link performance, model-side legality, workload +intent, and operational stability — and chooses the topology that best meets +the stated objective. Plus a simulator that evaluates planner decisions +under synthetic conditions without a cluster, so the cost model is testable, +calibratable, and regression-guarded. + +## Non-goals + +- No change to stage execution semantics, the activation wire protocol, or + llama.cpp integration. Runtime work is limited to passive timing + instrumentation; this work still re-plans *placement*. +- No speculative/exotic parallelism (tensor/pipeline-parallel hybrid graphs). + Contiguous layer pipelines only, as today. +- No live adaptive replanning in the first phases (see rollout — hysteresis + and migration come last, after the model is calibrated). + +## Current state + +| Capability | Where | Used by automatic placement? | +|---|---|---| +| Capacity fitting (exact per-layer weights, KV/token, recurrent/lane, 100/85 KV compute reserve, 10% runtime headroom) | `skippy-coordinator/src/topology.rs` | Yes | +| Candidate search (context ↓, node count ↑, lanes ↓, all subsets), stage-0 binding, 33 ms decode TPOT target, 64K shared-context floor | `skippy-coordinator/src/topology.rs`, `mesh-llm-host-runtime/src/runtime/split_planning.rs` | Yes | +| Latency estimate `stage_count × max RTT` | `estimate_decode_network_ms_per_token` | Superseded when edge data is present (modeled per-hop estimate); legacy estimate otherwise | +| GPU benchmarking (mem bw, fp16/fp32 TFLOPS) | `mesh-llm-gpu-bench`, `mesh-llm-system/src/benchmark.rs` | Metrics gossiped; **flow into the planner as of PR #1454** (auto-runs at node startup on non-client nodes) | +| Directed edge signals (RTT + large-frame bandwidth per edge, prediction-return support) | `skippy-topology/src/edge_order.rs` (exhaustive ordering ≤ 8 stages, greedy beyond) | The automatic planner consumes `TopologyEdge` RTT/bandwidth for scoring as of PR #1454. Production currently synthesizes symmetric pair estimates from coordinator-to-peer observations; directed node ordering and prediction-return capability remain confined to the explicit `skippy-topology` planner | +| Perf-aware span assignment (DP over layer boundaries minimizing serial modeled stage time) | `skippy-coordinator/src/topology.rs` (`serial_optimized_spans`) | Explicit opt-in; active when every node in a subset reports sustained bandwidth; exact legacy greedy otherwise | +| Modeled single-stream decode TPOT (Σ stages + Σ hops) for candidate selection | `skippy-coordinator/src/topology.rs` (`modeled_decode_tpot_us`) | Explicit opt-in; full TPOT outranks network-only/unknown estimates, which never imply target success | +| Observed steady-decode timing (µs/layer, sample count, age) | `skippy-server/src/stage_performance.rs`, additive gossip fields in `AdvertisedModelThroughput` | Explicit opt-in; batched decode is normalized by executed tokens, a changed stage range resets the window, and placement requires ≥8 samples no older than 2 minutes | +| RTT-floor confidence (sample count + first/latest sample age) | `mesh/peer_state.rs`, `runtime/local_package.rs` | Yes; remote perf signals are withheld until the minimum RTT is corroborated across the 5-second settle window | +| Placement simulator + scenario corpus | `skippy-topology-sim` crate | CI surface for planner behavior; corpus in `crates/skippy-topology-sim/scenarios/` | +| Model-family cut rules, state affinity, shared-KV cut bans, wire dtype, sidebands | `skippy-topology/src/planning.rs`, `validation.rs` | **No** (explicit-split validation only) — folding legality inputs into automatic planning is future work | + +The table above reflects the tree as of PR #1454 head; the original +`9feef0c1` survey that motivated the design is preserved in the PR's +first commit. + +## Input contract + +### 1. Node performance (per node) + +| Field | Source | Notes | +|---|---|---| +| `usable_vram_bytes` | existing `TopologyNode` | after 10% runtime headroom | +| `sustained_mem_bw_gbps` | gossip (`gpu_mem_bandwidth_gbps`), gpu-bench | measured, not spec | +| `sustained_compute_tflops` | gossip (`gpu_compute_tflops_fp16` preferred) | fallback signal only; decode is usually memory-bound | +| `host_ram_bytes` | node status | workspace/scratch headroom | +| `observed_decode_us_per_layer`, `sample_count`, `sample_age_ms` | staged runtime observation, additive gossip hint | steady decode after warmup; observations older than 30 minutes are omitted | +| `load_ewma`, `metric_age_ms` | future probe | stale load signals decay to neutral | + +### 2. Directed link performance (per ordered node pair) + +| Field | Source | +|---|---| +| `min_rtt_ms` | existing direct peer RTT floor | +| `large_frame_bytes_per_sec` | existing `StageEdgeSignal` field | +| `sample_count`, `first_sample_age_ms`, `last_sample_age_ms` | RTT observation window behind the minimum | +| `direct_prediction_return_supported` | existing `StageEdgeSignal` field; not yet part of the automatic planner contract | + +The explicit `skippy-topology` planner prices unknown edges with its pessimistic +`UNKNOWN_EDGE_RTT_MS` default. The automatic coordinator planner instead tries +the directed edge, then its reverse, then an endpoint's coordinator RTT; if no +RTT signal exists anywhere for a required hop, modeled TPOT is unavailable. +The planner deliberately does not estimate distribution tails or variance here: +iroh owns path selection, while placement only needs to distinguish a one-off +early minimum from a floor corroborated after the direct-path settle recheck. + +`direct_prediction_return_supported` is intentionally not consumed by +automatic placement in this PR. The stage runtime continues to enforce its +existing direct-return/fallback contract. Folding this capability into the +automatic planner requires an explicit mode: either reject an unsupported +return edge when direct return is mandatory, or price the downstream fallback +path when it is allowed. + +### 3. Model-side legality (per family) + +Existing policy inputs from `skippy-topology`: legal cut points, state +affinity, forbidden shared-KV cuts, activation wire dtype and bytes/frame, +required sidebands, backend/kernel support. Hard filters — a plan that +violates them is invalid regardless of score. + +### 4. Workload intent + +Objective selector: `interactive` (TTFT + decode TPOT targets; today's 33 ms +target generalizes) vs `throughput` (aggregate tok/s at concurrency). +Includes prompt/decode mix, context distribution, and lane count. + +### 5. Operational stability + +Artifact/package locality (existing eligibility filter), cold-load time, +node reliability, KV/state migration cost, and a minimum-improvement +threshold + hysteresis window so topologies do not churn. + +## Cost model + +Per stage `i` with layer span `L_i` on node `n(i)` and egress edge `e(i)`: + +```text +stage_time_ms(L_i) = max( Σ_{l∈L_i} weight_bytes(l) / mem_bw(n(i)), # weight streaming + flops(L_i) / sustained_compute(n(i)), # compute-bound regimes + observed_us_per_layer(n(i)) × |L_i| ) # live measured floor + + kv_touch_ms(L_i) # resident KV scan per token +edge_time_ms(i) = act_bytes(L_i) / large_frame_bw(e(i)) + min_rtt(e(i)) +single_stream_tpot_ms = Σ_i ( stage_time_ms(i) + edge_time_ms(i) ) # measured decode regime +pipelined_period_ms = max_i ( stage_time_ms(i) + edge_time_ms(i) ) # lanes > 1 +prefill_ms = Σ_i ( stage_time_ms(i) + edge_time_ms(i) ) # sequential fill +``` + +The BENCHMARKS.md anchors established that today's single-stream decode is +serial through the complete pipeline, while multiple in-flight lanes can +approach the pipelined period. The planner scores the measured single-stream +regime against the existing 33 ms target. + +**As-built (PR #1454):** decode is modeled as weight-streaming +(`streamed weight bytes / sustained_mem_bw`, integer microseconds, scaled +by `active_weight_fraction_permil` for MoE models) **plus both calibrated +overhead terms** — `per_stage_overhead` (1.3 ms) and `per_hop_overhead` +(13 ms) — inherited from the execution sim's BENCHMARKS.md calibration +(`CALIBRATED_PER_STAGE_OVERHEAD_US` / `CALIBRATED_PER_HOP_OVERHEAD_US` in +`skippy-coordinator`). The coordinator's modeled decode TPOT is the +**serial form**: Σ stage service times + Σ hop times across all hops +including the prediction return — every token traverses every stage, the +regime the BENCHMARKS.md anchors prove for single-stream decode. The +planner's number is locked to the calibrated execution sim by the +`planner_model_matches_execution_sim` test (≤1% divergence on the anchor +scenario). Live steady-decode observations are normalized to µs/layer after +warmup, bounded to an initial 256-sample arithmetic mean and then a 1/8 EWMA, +gossiped with count and age, and applied as a measured floor in both the span +DP and serial TPOT score. The compute term and KV-touch term from the formula +above remain plumbed-but-unused pending calibration against broader measured +data. +Missing node bandwidth is **all-or-nothing per candidate**: +a subset missing any node's bandwidth keeps the exact capacity-greedy span +assignment and carries `modeled_decode_tpot_us = None` — note the network +estimate still uses edge-aware per-hop estimation whenever edge data +exists, and only falls back to the legacy `hop_count × max-RTT` estimate +when edges are empty or disabled. A missing edge bandwidth contributes +zero transfer time (latency-only hop); an unmatched hop falls back to node +RTT, and a hop with no RTT signal anywhere declines to model TPOT +(`None`) rather than treating the hop as free. Canonical units: sustained +bandwidth MiB/s (1 MiB = 1_048_576 bytes), edge bandwidth MiB/s, all +modeled times integer microseconds; conversions happen once at parse +(GB/s → MiB/s, TFLOP/s → GFLOP/s). Stage observations older than 30 minutes +are evicted from the process-local recorder; placement applies a stricter +2-minute age and 8-sample floor. For remote candidates, the RTT/edge signal and all node- +performance signals are withheld until at least two valid RTT observations +span 5 seconds and the latest is no older than 30 seconds; this reuses the +capacity-only fallback instead of trusting an early post-connect minimum. + +**Scope of the fallback guarantee:** the all-or-nothing signal check and +the fallback span assignment are **per candidate subset**, not fleet-wide. +In a mixed fleet (some nodes reporting bandwidth, some not), fully-signaled +subsets get serial-optimized spans while subsets containing a signal-less node +keep the capacity-greedy walk — so which subsets win candidate selection +can differ from a signal-less fleet. Additionally, any non-empty edge data +switches the network estimate to edge-aware per-hop accounting for *every* +candidate, including capacity-greedy plans from signal-less subsets. The +bit-identical guarantee holds only when the fleet reports **no node +bandwidth signals and no edge data at all**; it is exercised by +`missing_perf_signals_keep_capacity_only_placement`, +`signalless_subset_placement_unchanged_by_other_nodes_signals`, and +`edge_data_changes_capacity_greedy_candidate_ordering`. + +## Search algorithm + +Preserve the existing candidate enumeration (it is correct and tested); add +performance to scoring and ordering. This is the target algorithm; the +as-built exceptions immediately below distinguish the implementation landed in +PR #1454: + +1. **Hard feasibility filter** (unchanged): memory fit with reserves, + family cut legality, stage-0 binding, sidebands, artifact access. +2. **Enumerate candidates** (unchanged): context lengths highest→lowest, + node counts fewest→most, lanes highest→lowest, node subsets. +3. **Order nodes** on the directed link graph (adopt + `order_pipeline_nodes`: exhaustive ≤ 8 stages, greedy beyond) instead of + VRAM-descending order. +4. **Span assignment**: replace greedy largest-fit with DP over contiguous + layer boundaries that minimizes the serial sum of modeled stage service + time, matching the single-stream TPOT evaluator, subject to per-node + memory ceilings. The recurrence compares every prior boundary, so a + candidate costs `O(layers² × nodes)` — at current scales (≤ ~100 layers, + ≤ ~8 nodes) that is ≤ ~80K comparisons per candidate, trivially cheap; + Knuth-style optimization could reduce it to `O(layers × nodes)` if + fleets grow. +5. **Score lexicographically**: correctness → SLO met → objective-specific + performance (TPOT or throughput) → context/lane utility → confidence and + headroom → deterministic tie-breaks (existing `latency_candidate_ordering` + shape). + +**As-built exceptions (PR #1454):** automatic stages remain ordered by usable +VRAM descending with a node-id tie-break; the coordinator does not yet call +`order_pipeline_nodes`. Automatic planning also does not yet consume the +model-family legality/sideband policy or prediction-return support from +`skippy-topology` (see the current-state table). The span DP minimizes serial +stage service time in that fixed order, while directed edge data affects +candidate scoring only. Edge-aware node ordering and policy integration remain +explicit follow-up work. + +## Simulator + +Two layers, sharing one scenario format (`toml`) — see the as-built corpus in +`crates/skippy-topology-sim/scenarios/`: + +```toml +[nodes.m4max] +vram_bytes = 68719476736 # 64 GiB +sustained_mem_bandwidth_mib_per_s = 546000 # measured +sustained_compute_gflop_per_s = 34000 +observed_decode_us_per_layer = 2400 # optional live measured floor + +[nodes.mini] +vram_bytes = 17179869184 # 16 GiB +sustained_mem_bandwidth_mib_per_s = 120000 +sustained_compute_gflop_per_s = 2000 + +[links."m4max -> mini"] # directed edge, spaces in key +rtt_ms = 3 +large_frame_mib_per_s = 30 # Wi-Fi large-frame prior + +[model] +layer_count = 40 +weight_bytes_per_layer = 1610612736 +kv_bytes_per_token = 4096 +native_context_length = 65536 +activation_frame_bytes = 8192 + +[workload] +minimum_nodes = 2 +``` + +1. **Placement sim** (deterministic, fast, in-crate): scenario → planner → + assert chosen topology and score. Runs in CI as the planner's unit-test + surface: property tests over synthetic grids ("10× slower link must move + the boundary", "half-bandwidth node must receive fewer layers", + "absent signals must reproduce current placement exactly"). +2. **Execution sim** (discrete-event): pipeline of stages with service times + from the cost model + edge models; replays synthetic workload traces; + emits TTFT/TPOT/throughput curves per candidate topology. Covers + degenerate conditions: straggler node, degrading link, cold-start after + failure, mixed prompt lengths at concurrency. + +**Calibration bar:** the execution sim must reproduce the measured ratios in +`docs/BENCHMARKS.md` (68 → 21 → 12-13 tok/s across 1/2/3-way splits on the +documented hardware; 10-25 tok/s at ~20 ms RTT, RPC-latency-dominated) from +the documented inputs, within tolerance. If it cannot, the cost model is +wrong and gets fixed before any production behavior depends on it. + +**As-built (PR #1454):** the execution layer (`skippy-topology-sim::execution`) +models two regimes — **serial** decode (single stream: every token traverses +every stage and returns; TPOT = Σ stages + Σ hops — this is the regime the +BENCHMARKS.md anchors measured) and **pipelined** (lanes > 1: bounded by the +slowest stage+egress pair). Stage service time is streamed-bytes/bandwidth +with `active_weight_fraction` capturing MoE active-expert bytes; two +calibration knobs record what the pure model cannot see: per-stage software +overhead and per-hop RPC overhead (the "per-token RPC latency" BENCHMARKS.md +names as dominant). Calibration scenario `benchmarks_anchor_pair.toml` + +tests reproduce all three anchors within ~10% (tolerance ±15%). Coefficients +are recorded in the scenario with their derivation so real measurements +(passive edge observations in particular) can tighten them. + +## Scenario corpus: realistic hardware, links, and backends + +The simulator is only as honest as its inputs. The corpus spans the hardware +and transports mesh actually runs on, from consumer laptops to datacenter +nodes, with per-backend variation. Numbers below are *prior* starting points +(spec-class), to be replaced by `mesh-llm-gpu-bench` measurements as they are +collected — the corpus format records `source = "spec" | "measured"` per +field and the calibration phase upgrades specs to measurements. + +### Node tiers + +| Tier | Example | VRAM | Sustained mem bw (prior) | Sustained fp16 (prior) | Backends | +|---|---|---|---|---|---| +| Consumer laptop, CPU | 16-32 GB LPDDR5 | shared | 60-100 GB/s | 0.1-0.5 TFLOPS | CPU (AVX2/NEON) | +| Consumer laptop, iGPU | 16-64 GB unified | shared | 100-546 GB/s | 1-34 TFLOPS | Metal (Apple), Vulkan | +| Consumer desktop GPU | RTX 3060/4070, 8-12 GB | 8-12 GB | 360-504 GB/s | 15-30 TFLOPS | CUDA, Vulkan | +| Prosumer GPU | RTX 3090/4090, 24 GB | 24 GB | 936-1008 GB/s | 40-83 TFLOPS | CUDA | +| Prosumer multi-GPU | 2-4× above | 48-96 GB | per-GPU, NVLink absent | per-GPU | CUDA | +| Datacenter GPU | A100 80 GB | 80 GB | 1.9-2.0 TB/s | 78-312 TFLOPS | CUDA | +| Datacenter GPU | H100 80 GB | 80 GB | 3.3-3.4 TB/s | 197-990 TFLOPS | CUDA | +| Datacenter GPU | MI300X 192 GB | 192 GB | 5.3 TB/s | 163-1307 TFLOPS | ROCm | + +Backend matters independently of the chip: the same GPU on CUDA vs Vulkan can +differ materially in sustained throughput, and llama.cpp quant kernels vary by +backend and quant (Q4_K_M, Q8_0, f16). Corpus entries therefore carry +`(hardware, backend, quant)` triples, not hardware alone. + +### Link tiers + +| Tier | Example | Typical RTT | Large-frame throughput (prior) | +|---|---|---|---| +| Loopback / same host | localhost | 0.05-0.2 ms | 20-60 GB/s | +| Direct cable | Thunderbolt/2.5-10GbE point-to-point | 0.1-0.5 ms | 1-20 GB/s | +| LAN wired | 1-10 GbE switched | 0.3-2 ms | 100 MB/s-1 GB/s | +| LAN Wi-Fi | Wi-Fi 5/6/6e | 2-10 ms | 10-60 MB/s | +| Metro WAN | same-city fiber | 5-15 ms | 10-100 MB/s | +| Continental WAN | Sydney↔QLD class | 15-40 ms | 5-50 MB/s | +| Intercontinental | US↔EU/US↔APAC | 60-250 ms | 1-20 MB/s | + +Asymmetry is first-class: edges are directed, so scenarios include pairs +where A→B and B→A differ (asymmetric Wi-Fi, rate-limited cloud egress). + +### Corpus scenarios (initial set) + +Landed in `crates/skippy-topology-sim/scenarios/` as of PR #1454: +`heterogeneous_pair.toml` (2), `straggler_triplet.toml` (3), +`cross_continent_chain.toml` (4). Remaining from the initial set — +homogeneous pair (1), mixed-quant fleet (5), load/staleness sweep (6), +failure cold-start (7) — are open corpus work tracked in issue #1455. + +1. **Homogeneous pair** (2× M4 Max, Thunderbolt): baseline sanity. *(pending)* +2. **Heterogeneous pair** (M4 Max + Mac mini, Wi-Fi): reproduces the + `docs/BENCHMARKS.md` 68 → 21 tok/s anchor. **landed** +3. **Straggler triplet** (A100 + 4090 + laptop-CPU): the laptop must get + few layers or be excluded; tests performance-aware span assignment + against capacity-only. **landed** +4. **Cross-continent chain** (3 nodes, 60-150 ms edges): tests that + edge-aware ordering minimizes high-latency hops and rejects infeasible + TPOT targets rather than accepting them. **landed** +5. **Mixed-quant fleet** (same model, Q4/Q8/f16 on different nodes): + activation wire dtype interacts with per-node bytes/layer. *(pending)* +6. **Load and staleness sweep** (one node busy/stale): confidence decay + must fall back toward capacity-only placement. *(pending)* +7. **Failure cold-start** (node rejoins empty): migration/dwell-time + accounting under phase 5 policies. *(pending)* + +### Where the data comes from + +- **Priors:** vendor spec sheets (bandwidth classes, not marketing peaks), + recorded in the corpus with `source = "spec"`. +- **Measurements:** `mesh-llm-gpu-bench` runs on real fleet nodes over time + (`source = "measured"`, timestamped, superseding specs), plus edge probes + already producing `StageEdgeSignal` data. +- **Anchors:** the measured results in `docs/BENCHMARKS.md` are regression + anchors the execution sim must reproduce within tolerance. + +The corpus lives in-repo as scenario TOML files so CI, the planner tests, and +the execution sim all consume the same data. + +## Changing network conditions + +Bandwidth is not static: Wi-Fi fades, links get congested, VPNs re-route. +The planner's job under drift is **detect → re-estimate → decide**, with +anti-churn protection so a transient dip does not cause a topology stampede. + +**What exists today (as of PR #1454):** +- Node perf metrics (mem bw, compute) and per-participant RTT are part of the + split-participant signature (`split_participant_signature`), so a measured + change re-triggers planning automatically. +- **Edge bandwidth is measured passively**: every artifact transfer (either + direction) records bytes/second on that peer link (`LargeFrameObservation`), + age-gated to 30 minutes, conservatively min-merged into plan edges. As + conditions change, the next transfer re-measures the link — drift detection + rides the traffic the mesh already generates. Active probing (synthetic + frames between idle stage peers) remains future work. +- The planner edge type is directed, and simulator scenarios can supply truly + asymmetric A→B/B→A values. Production does not yet measure remote pair + directions independently: `participant_edges` synthesizes both directions + with the same conservative max RTT and min bandwidth from coordinator-to-peer + observations. +- The best-seen RTT remains a minimum, but its observation count and first/ + latest sample ages are retained. Remote performance-aware placement waits + for two samples spanning the 5-second direct-path recheck; a lone 200 ms-old + sample falls back to capacity-only placement. Distribution tails are not + planner inputs. +- Embedded stages record steady-decode compute time after warmup, normalize it + per loaded layer, gossip the bounded timing hint, and use it as a measured + floor on analytical stage service time. The participant signature includes + the observation so a fresh planning round cannot silently reuse a stale + claim. +- Performance-aware placement is deliberately default-off while the remaining + legality, measurement-trust, and adoption gates are incomplete. Set + `MESH_TOPOLOGY_PERF_AWARE=1/true/on/yes` to opt in. Unset, disable spellings, + and unknown values strip perf signals + edges and reproduce capacity-only + placement. Disabled-mode participant identity also omits perf fields; opt-in + identity quantizes them so sub-bucket noise does not force plan churn. + +**The three detection windows and their design:** + +| Window | Signal | Response | +|---|---|---| +| Per-token (immediate) | In-flight decode misses TPOT target | Runtime concern, not planner's — no topology change; the plan already priced this link into its estimate | +| Per-probe (minutes) | Re-measured edge/node metrics shift | Signature change flows into the coordinator claim's `participant_set_hash`, invalidating the current generation's identity and forcing a fresh planning round. **Today the fresh round replaces the incumbent unconditionally** — the minimum-improvement threshold below is phase-5 design, not yet implemented | +| Per-epoch (hours/days) | Slow drift, new nodes, day/night load | Same replan trigger; hysteresis (phase 5) dampens noise | + +**Why not react instantly:** re-sharding a live mesh costs KV migration + +pipeline stall. A Wi-Fi blip that halves bandwidth for 20 seconds should not +evict a topology that took minutes to load. The planner therefore treats +edge measurements as *estimates with age and confidence*, not instantaneous +truth — the same design as metric-age decay in the input contract. + +**What phase 5 adds:** adaptive replanning with explicit hysteresis and +migration budgets — re-estimating when an edge's sustained (not transient) +bandwidth drops materially below what the plan assumed, and migrating only +when the modeled improvement exceeds the migration cost. Until then the +system degrades to today's behavior: the plan made at startup holds until +membership or a signature change forces a re-plan. + +**Open question (phase 5):** how to age/degrade edge bandwidth measurements +between probes. Candidates are an EWMA of sustained transfer samples or a +conservative age-based floor. The execution sim's calibration against +BENCHMARKS.md anchors is the testbed for choosing between these; planner-side +distribution-tail estimation is intentionally out of scope. + +## Phased rollout + +| Phase | Deliverable | Gate | Status | +|---|---|---|---| +| 0 | Thread gossiped perf metrics through `SplitTopologyPlanInput → TopologyNode` | no behavior change (signals recorded, unused) | **Done** (PR #1454) — metrics flowed through and joined the replan signature | +| 1 | Cost model + merged scoring in `skippy-coordinator`; absent-signal fallback = exact current behavior | placement-parity tests vs old planner on signal-less inputs | **Done** (PR #1454 hardening) — objective-consistent `serial_optimized_spans` DP, tri-state estimate ordering, and parity tests | +| 2 | Placement sim in CI; scenario corpus incl. BENCHMARKS.md anchors | property tests green; parity suite green | **Done** (PR #1454) — `skippy-topology-sim` + 3 corpus scenarios | +| 3 | Passive edge measurement; execution sim calibration; observed stage-timing feedback; settle-time RTT confidence | calibration tolerance met; uncorroborated remote signals fall back safely | **Done** (PR #1454) — passive edge bandwidth from real artifact transfers (both directions, age-gated 30 min, conservative min-merge, replan signature); execution sim + BENCHMARKS.md calibration tests (±15% tolerance, currently within ~10% on all three anchors); live steady-decode µs/layer feeds span DP and serial TPOT as a measured floor; min RTT carries sample count + first/latest age and requires corroboration across the settle window. Active synthetic probing remains optional future corpus work, not a phase-4 prerequisite | +| 4 | Performance-aware placement live | A/B on staging meshes vs capacity-only | **Code path explicit opt-in; default-on is blocked on reference-hardware A/B plus phase 5 and legality integration** | +| 5 | Adaptive replanning with hysteresis + migration budgets | dwell-time threshold; no churn under synthetic perturbations | Planned | + +Phase 1's fallback property is the safety story: with no signals *and no +edge data anywhere in the fleet*, the merged planner is bit-identical to +today's (per-subset scope above). Each phase is independently mergeable. + +## Alternatives considered + +- **Third planner, purpose-built.** Rejected: duplicates feasibility logic + that already exists and is tested in two places; the merge is smaller than + a rewrite and keeps the policy crate's validation as the legality + authority. +- **Pure simulation-first (build sim, decide later).** Rejected: the sim + needs the cost model, the cost model needs the input plumbing; phase 0/1 + deliver both and the sim then has something honest to simulate. +- **vLLM/SGLang-style profile-guided autotuning.** Deferred: calibration + from observed stage timings (phase 5) gets most of the value without a + profile store and offline tuning loop. + +## Risks + +- **Cost model error → worse placements.** Mitigated by the fallback + property, calibration gates, and phase 4 A/B before default-on. +- **Stale/lying gossip.** Mitigated by default-off selection, bounded GPU CSV + cardinality/plausibility, fresh multi-sample stage timing, RTT-floor + corroboration, per-candidate absent-signal fallback, and pessimistic + unknown-edge defaults. Static gpu-bench claims remain soft hints and are not + a trust boundary; production enablement still requires locally corroborated + measurements. +- **Search blowup on large fleets.** Node subsets are already bounded; + DP span assignment is `O(layers² × nodes)` per candidate. Automatic edge + ordering is not wired in today; once adopted, the existing policy planner's + exhaustive ≤8-stage / greedy >8-stage split bounds that additional search. diff --git a/scripts/affected-crates.sh b/scripts/affected-crates.sh index 4d4c4e18ac..af48ca7d59 100755 --- a/scripts/affected-crates.sh +++ b/scripts/affected-crates.sh @@ -54,6 +54,7 @@ WORKSPACE_MEMBERS=( "skippy-tokenizer" "skippy-coordinator" "skippy-topology" + "skippy-topology-sim" "skippy-cache" "skippy-metrics" "openai-frontend" diff --git a/tools/xtask/data/console_print_allowlist.json b/tools/xtask/data/console_print_allowlist.json index a2489892f3..90790e852b 100644 --- a/tools/xtask/data/console_print_allowlist.json +++ b/tools/xtask/data/console_print_allowlist.json @@ -1929,19 +1929,19 @@ ], "crates/mesh-llm-host-runtime/src/runtime/local_split/tests.rs": [ { - "line": 2103, + "line": 2166, "macro_name": "eprintln!" }, { - "line": 2108, + "line": 2171, "macro_name": "eprintln!" }, { - "line": 2120, + "line": 2183, "macro_name": "eprintln!" }, { - "line": 2141, + "line": 2204, "macro_name": "eprintln!" } ],