diff --git a/crates/ika-sui-client/src/grpc.rs b/crates/ika-sui-client/src/grpc.rs index 9a5d566bd3..2167caef7c 100644 --- a/crates/ika-sui-client/src/grpc.rs +++ b/crates/ika-sui-client/src/grpc.rs @@ -50,7 +50,7 @@ use tonic::metadata::{Ascii, MetadataKey, MetadataValue}; use crate::rate_limit::RateLimitGate; use crate::transport::{ CheckpointSummaryStream, DynamicFieldEntry, DynamicFieldPage, ExecutedTransaction, - SubmittedTransaction, SuiFundsBreakdown, SuiTransport, SuiWriter, TransportError, + SubmittedTransaction, SuiFundsBreakdown, SuiNodeInfo, SuiTransport, SuiWriter, TransportError, }; /// Sui rejects a transaction whose gas payment names more than @@ -788,6 +788,27 @@ impl SuiTransport for SuiGrpcClient { .map(|chain_identifier| chain_identifier.to_string()) } + async fn get_sui_node_info(&self) -> Result, TransportError> { + // Same `GetServiceInfo` call `get_latest_checkpoint_sequence` and + // `get_sui_chain_identifier` already use; we read two different + // fields off it. It takes no arguments and touches no store beyond + // the node's own watermarks, so it is the cheapest call the fullnode + // serves — which is what makes it safe to poll on a timer. + let mut rpc = self.rpc.clone(); + let response = self + .gated_network(async move { + rpc.ledger_client() + .get_service_info(proto::GetServiceInfoRequest::default()) + .await + }) + .await? + .into_inner(); + Ok(Some(SuiNodeInfo { + server_version: response.server_opt().map(str::to_owned), + chain_identifier: response.chain_id_opt().map(str::to_owned), + })) + } + async fn get_current_epoch(&self) -> Result { let mut rpc = self.rpc.clone(); let mut request = proto::GetEpochRequest::default(); diff --git a/crates/ika-sui-client/src/grpc_backend.rs b/crates/ika-sui-client/src/grpc_backend.rs index aa2a3d979c..43747d84d6 100644 --- a/crates/ika-sui-client/src/grpc_backend.rs +++ b/crates/ika-sui-client/src/grpc_backend.rs @@ -106,6 +106,15 @@ impl GrpcSuiClient { } } + /// The read transport this backend was built over. Handed to the + /// `ika_sui_client_sui_node_info` refresher, which needs a `'static` + /// handle to the same uplink the client reads through — so that the + /// version it reports is the version of the node actually serving this + /// client, not of some separately-opened connection. + pub fn transport(&self) -> std::sync::Arc { + self.transport.clone() + } + /// The writer uplink, or a clear error on a read-only node. Transaction /// building/submission (and the migration-sweep reads that serve it) are /// notifier-gated to a direct fullnode connection. diff --git a/crates/ika-sui-client/src/lib.rs b/crates/ika-sui-client/src/lib.rs index fb6812b7d6..5b759ec40d 100644 --- a/crates/ika-sui-client/src/lib.rs +++ b/crates/ika-sui-client/src/lib.rs @@ -46,6 +46,7 @@ pub mod ika_dwallet_transactions; pub mod ika_protocol_transactions; pub mod ika_validator_transactions; pub mod metrics; +pub mod node_info; pub mod rate_limit; pub mod transaction_builder; pub mod transaction_context; @@ -102,6 +103,11 @@ pub struct SuiClient

{ system_arg_cache: OnceCell, clock_arg_cache: OnceCell, dwallet_coordinator_arg_cache: OnceCell, + /// Keeps `ika_sui_client_sui_node_info` current for as long as this client + /// exists — the handle aborts the refresh task on drop, so a short-lived + /// CLI client does not leave a timer behind. `None` on clients built for + /// tests, which have no registry anyone scrapes. + _node_info_refresh: Option, } pub type SuiBackend = grpc_backend::GrpcSuiClient; @@ -144,6 +150,11 @@ impl SuiConnectorClient { let inner = grpc_backend::GrpcSuiClient::new_with_headers(grpc_url, headers, rate_limit_gate) .await?; + // Spawned, never awaited: identifying the fullnode must not add a + // round trip to node boot, least of all on the wedged-uplink nodes + // this metric exists to describe. + let node_info_refresh = + node_info::spawn_sui_node_info_refresh(inner.transport(), sui_client_metrics.clone()); let self_ = Self { inner, sui_client_metrics, @@ -151,6 +162,7 @@ impl SuiConnectorClient { system_arg_cache: OnceCell::new(), clock_arg_cache: OnceCell::new(), dwallet_coordinator_arg_cache: OnceCell::new(), + _node_info_refresh: Some(node_info_refresh), }; self_.describe().await?; Ok(self_) @@ -166,6 +178,11 @@ impl SuiConnectorClient { ika_network_config: IkaNetworkConfig, ) -> anyhow::Result { let inner = grpc_backend::GrpcSuiClient::with_transport(transport); + // The relay transport has no service-info RPC, so this settles on + // `server_version="unsupported"` after one probe and stops — which is + // the honest answer for a node with no fullnode of its own. + let node_info_refresh = + node_info::spawn_sui_node_info_refresh(inner.transport(), sui_client_metrics.clone()); let self_ = Self { inner, sui_client_metrics, @@ -173,6 +190,7 @@ impl SuiConnectorClient { system_arg_cache: OnceCell::new(), clock_arg_cache: OnceCell::new(), dwallet_coordinator_arg_cache: OnceCell::new(), + _node_info_refresh: Some(node_info_refresh), }; self_.describe().await?; Ok(self_) @@ -218,6 +236,10 @@ where system_arg_cache: OnceCell::new(), clock_arg_cache: OnceCell::new(), dwallet_coordinator_arg_cache: OnceCell::new(), + // Not spawned for tests: `new_for_testing` is called from + // synchronous contexts with no runtime, and nothing scrapes the + // throwaway registry it builds. + _node_info_refresh: None, } } diff --git a/crates/ika-sui-client/src/metrics.rs b/crates/ika-sui-client/src/metrics.rs index 75ebca191d..271ad6006a 100644 --- a/crates/ika-sui-client/src/metrics.rs +++ b/crates/ika-sui-client/src/metrics.rs @@ -1,9 +1,29 @@ // Copyright (c) Mysten Labs, Inc. // SPDX-License-Identifier: BSD-3-Clause-Clear -use prometheus::{IntCounterVec, Registry, register_int_counter_vec_with_registry}; -use std::sync::Arc; +use prometheus::{ + IntCounterVec, IntGauge, IntGaugeVec, Registry, register_int_counter_vec_with_registry, + register_int_gauge_vec_with_registry, register_int_gauge_with_registry, +}; use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; + +/// Label value published on [`SuiClientMetrics::sui_node_info`] from process +/// start until the node's `GetServiceInfo` first answers. A validator whose +/// series is *still* on this label is the signal: its Sui uplink has never +/// once told us what it is. +pub const SUI_NODE_INFO_UNKNOWN: &str = "unknown"; + +/// Label value for a client whose transport has no service-info RPC at all — +/// a peer-only validator reading Sui through the Ika p2p mirror relay. There +/// is no operator fullnode on the other end to have a version, so this is a +/// *healthy* terminal state and must not be read as `unknown`. +pub const SUI_NODE_INFO_UNSUPPORTED: &str = "unsupported"; + +/// Label value for a node that answered `GetServiceInfo` but left the field +/// empty (both fields are `optional` in the proto, and a proxy in front of a +/// fullnode can strip them). Distinct from `unknown`: the RPC works. +pub const SUI_NODE_INFO_UNREPORTED: &str = "unreported"; /// Process-wide counter for chain-side calls to /// `get_network_encryption_key_with_full_data_by_epoch`. Test @@ -50,6 +70,60 @@ pub struct SuiClientMetrics { /// keeps the endpoint refusing. Fleet-wide this should be flat at zero; /// growth on a single node points at that node's endpoint, not at ika. pub sui_rate_limited_errors: IntCounterVec, + /// Info-style gauge (always `1` on exactly one child) identifying the Sui + /// fullnode this validator's uplink is pointed at: `server_version` is the + /// node's own software version string, `chain_identifier` its genesis + /// digest. + /// + /// # Why + /// + /// Two mainnet validators wedged in early boot at consecutive epoch + /// boundaries (2026-08-28 and 08-29). Both had their own Sui fullnode's + /// RPC failing for hours beforehand — one at ~440 errors/hour, the other + /// ~316, against a fleet baseline of 0–6. The leading hypothesis for the + /// first is simply that the operator never upgraded their Sui node across + /// a Sui release rollout. We could not confirm or refute that, because + /// nothing anywhere in the fleet's telemetry says which Sui version a + /// validator is talking to: `sui_rpc_errors` tells us an uplink is + /// unhealthy, never *what* is on the other end of it. This gauge closes + /// that gap — one `group by (server_version)` over the fleet shows the + /// version spread and singles out the operator who is behind. + /// + /// # Semantics + /// + /// Registered eagerly with `server_version="unknown"` at `1`, so a node + /// that never gets an answer still publishes a series — a validator stuck + /// on `unknown` is itself the signal that its Sui RPC never replied. + /// Refreshes never *remove* a child, they flip the old one to `0` and the + /// new one to `1`, so a version change reads as a clean 1→0 / 0→1 + /// transition across a scrape gap instead of a disappearing series. + /// + /// # Cardinality + /// + /// One child at `1` per node at a time, plus one retired child at `0` per + /// version the node has ever reported — in practice one or two per + /// process lifetime, since an operator's Sui version changes only when + /// they upgrade and the process restarts on ours. Negligible. + pub sui_node_info: IntGaugeVec, + /// Unix seconds of the last successful `GetServiceInfo`; `0` until the + /// first one lands. + /// + /// Freshness companion to [`Self::sui_node_info`], which on its own cannot + /// distinguish "this really is the version, confirmed minutes ago" from + /// "this was the version hours ago and the RPC has been dead since" — the + /// info gauge holds its last known value by design. `0` additionally + /// separates "never answered" from "answered once and went stale", which + /// matters because a boot-time wedge produces the former and a mid-life + /// failure the latter. + /// + /// Left at `0` forever on a peer-only node (see + /// [`SUI_NODE_INFO_UNSUPPORTED`]) — there is nothing to refresh, so pair + /// any staleness query with `server_version != "unsupported"`. + pub sui_node_info_last_success_unixtime: IntGauge, + /// The `(server_version, chain_identifier)` pair currently published at + /// `1`. Held so a refresh can flip exactly the previous child to `0` + /// without enumerating (or resetting) the vec. + active_sui_node_info: Arc>, } impl SuiClientMetrics { @@ -76,6 +150,23 @@ impl SuiClientMetrics { registry, ) .unwrap(), + sui_node_info: register_int_gauge_vec_with_registry!( + "ika_sui_client_sui_node_info", + "Info gauge (always 1) identifying the Sui fullnode this validator's uplink is connected to, by its reported server version and chain id", + &["server_version", "chain_identifier"], + registry, + ) + .unwrap(), + sui_node_info_last_success_unixtime: register_int_gauge_with_registry!( + "ika_sui_client_sui_node_info_last_success_unixtime", + "Unix seconds of the last successful Sui GetServiceInfo call; 0 until the first one succeeds", + registry, + ) + .unwrap(), + active_sui_node_info: Arc::new(Mutex::new([ + SUI_NODE_INFO_UNKNOWN.to_string(), + SUI_NODE_INFO_UNKNOWN.to_string(), + ])), }; // Publish both series from process start so a dashboard/alert can tell // "no rate limiting" apart from "this node never reported". @@ -87,6 +178,14 @@ impl SuiClientMetrics { .with_label_values(&[signal.label()]) .inc_by(0); } + // Same reasoning, one step stronger: the `unknown` child is not just a + // zero placeholder, it is the value that stays at 1 for a node whose + // Sui RPC never answers. It has to exist before the first refresh — + // which for a wedged node is a refresh that never completes. + this.sui_node_info + .with_label_values(&[SUI_NODE_INFO_UNKNOWN, SUI_NODE_INFO_UNKNOWN]) + .set(1); + this.sui_node_info_last_success_unixtime.set(0); Arc::new(this) } @@ -94,4 +193,56 @@ impl SuiClientMetrics { let registry = Registry::new(); Self::new(®istry) } + + /// Publish a fresh `GetServiceInfo` answer: flip the previously-active + /// child of [`Self::sui_node_info`] to `0`, the new one to `1`, and stamp + /// [`Self::sui_node_info_last_success_unixtime`]. + /// + /// Re-reporting the same pair is the common case (the version only changes + /// when the operator upgrades) and is cheap: it re-asserts `1` and + /// re-stamps the freshness gauge without touching any other child. + pub fn set_sui_node_info(&self, server_version: Option<&str>, chain_identifier: Option<&str>) { + self.flip_sui_node_info( + server_version.unwrap_or(SUI_NODE_INFO_UNREPORTED), + chain_identifier.unwrap_or(SUI_NODE_INFO_UNREPORTED), + ); + self.sui_node_info_last_success_unixtime + .set(unix_seconds_now()); + } + + /// Mark this client's uplink as having no service-info RPC — a peer-only + /// validator reading through the p2p mirror relay. + /// + /// Deliberately does *not* stamp the freshness gauge: there is no fullnode + /// to be fresh about, and leaving it at `0` keeps "never answered" and + /// "cannot answer" apart in queries by the label alone. + pub fn set_sui_node_info_unsupported(&self) { + self.flip_sui_node_info(SUI_NODE_INFO_UNSUPPORTED, SUI_NODE_INFO_UNSUPPORTED); + } + + fn flip_sui_node_info(&self, server_version: &str, chain_identifier: &str) { + // A poisoned lock here carries no invariant worth propagating — the + // guarded value is two label strings — and this runs on a background + // refresh whose whole contract is to never disturb the client. + let mut active = self + .active_sui_node_info + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + if active[0] != server_version || active[1] != chain_identifier { + self.sui_node_info + .with_label_values(&[active[0].as_str(), active[1].as_str()]) + .set(0); + *active = [server_version.to_string(), chain_identifier.to_string()]; + } + self.sui_node_info + .with_label_values(&[server_version, chain_identifier]) + .set(1); + } +} + +fn unix_seconds_now() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs() as i64) + .unwrap_or(0) } diff --git a/crates/ika-sui-client/src/node_info.rs b/crates/ika-sui-client/src/node_info.rs new file mode 100644 index 0000000000..3d13c0dffd --- /dev/null +++ b/crates/ika-sui-client/src/node_info.rs @@ -0,0 +1,406 @@ +// Copyright (c) dWallet Labs, Ltd. +// SPDX-License-Identifier: BSD-3-Clause-Clear + +//! Background refresher for `ika_sui_client_sui_node_info`. +//! +//! # Why this is a background task +//! +//! The metric exists because an operator's *outdated or failing* Sui fullnode +//! is invisible to the fleet (see +//! [`crate::metrics::SuiClientMetrics::sui_node_info`]). The nodes it most +//! needs to describe are therefore precisely the nodes whose RPC is slow, +//! hanging, or dead — so identifying them must never sit on a code path any +//! real client operation waits for. Nothing here is awaited by a caller: the +//! task is spawned at client construction, every call is deadlined, and every +//! failure is a no-op that leaves the last known labels in place. +//! +//! # Why failures are not counted in `sui_rpc_errors` +//! +//! Deliberate. `ika_sui_client_sui_rpc_errors` is the fleet's Sui-uplink +//! health signal, and its healthy baseline is 0–6 errors/hour — a band narrow +//! enough that a passive probe contributing its own floor to it would blunt +//! exactly the signal the probe exists to explain. A probe failing while the +//! node's real reads succeed is also not an incident, and would be a false +//! positive in that counter. The probe's failures are already fully +//! observable in its own two metrics: the `server_version` label stays at +//! `unknown` (or goes stale) and +//! `ika_sui_client_sui_node_info_last_success_unixtime` stops advancing. +//! +//! One counter the refresh *does* still feed, unavoidably and correctly: +//! calls go through the node's shared [`crate::rate_limit::RateLimitGate`], +//! so a rate-limited response is classified like any other and increments +//! `ika_sui_client_rate_limited_errors_total`. At a maximum of five calls an +//! hour that is negligible, and if the endpoint really is refusing, saying so +//! is right. + +use std::future::Future; +use std::sync::Arc; +use std::time::Duration; + +use tracing::{debug, info}; + +use crate::metrics::SuiClientMetrics; +use crate::transport::{SuiNodeInfo, SuiTransport, TransportError}; + +/// Steady-state refresh cadence. An operator's Sui version changes only when +/// they deploy an upgrade, so minutes of staleness cost nothing; the interval +/// is set by wanting a fleet-wide version census to converge within a Grafana +/// panel's typical lookback, not by any need for freshness. +const REFRESH_INTERVAL: Duration = Duration::from_secs(12 * 60); + +/// Delay before the second attempt while the node has *never* been +/// identified, doubling up to [`REFRESH_INTERVAL`]. A node that answers on +/// the first try never uses this; one whose uplink is flaky at boot gets +/// identified in tens of seconds instead of waiting out a full interval, +/// without turning a sustained outage into a polling loop. +const INITIAL_RETRY_INTERVAL: Duration = Duration::from_secs(30); + +/// Per-call deadline. A hung fullnode is one of the failure modes this metric +/// was built to expose, so the probe cannot be allowed to hang with it: a +/// task blocked forever in `GetServiceInfo` would leave the labels frozen and +/// never retry, which looks identical to a healthy node that stopped changing. +const CALL_TIMEOUT: Duration = Duration::from_secs(20); + +/// Handle to the spawned refresh task; aborts it on drop so the task's +/// lifetime is exactly the owning client's. +#[derive(Debug)] +pub struct SuiNodeInfoRefresh(tokio::task::JoinHandle<()>); + +impl Drop for SuiNodeInfoRefresh { + fn drop(&mut self) { + self.0.abort(); + } +} + +/// Start refreshing `ika_sui_client_sui_node_info` from `transport`. +/// +/// Must be called from inside a Tokio runtime (every construction path is +/// already `async`). Returns immediately; the first probe runs on the spawned +/// task, not here. +pub fn spawn_sui_node_info_refresh( + transport: Arc, + metrics: Arc, +) -> SuiNodeInfoRefresh { + SuiNodeInfoRefresh(tokio::spawn(refresh_loop( + move || { + let transport = transport.clone(); + async move { transport.get_sui_node_info().await } + }, + metrics, + ))) +} + +/// Result of one probe, in the terms the loop schedules on. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Tick { + /// The node answered; labels now describe it. + Identified, + /// This transport structurally cannot answer. Terminal — nothing about it + /// will change on a later attempt. + Unsupported, + /// The call failed or timed out. Previous labels are left untouched. + Failed, +} + +async fn refresh_loop(fetch: F, metrics: Arc) +where + F: Fn() -> Fut + Send + 'static, + Fut: Future, TransportError>> + Send, +{ + let mut acquisition_delay = INITIAL_RETRY_INTERVAL; + let mut ever_identified = false; + loop { + let sleep_for = match refresh_once(&fetch, &metrics).await { + Tick::Identified => { + if !ever_identified { + ever_identified = true; + info!("identified the connected Sui node; publishing sui_node_info"); + } + REFRESH_INTERVAL + } + // Nothing to poll for. Leaving the task alive would burn a timer + // forever to re-derive a constant. + Tick::Unsupported => return, + // Once we know the version, a failed refresh is not urgent — the + // last known labels stay published and the freshness gauge already + // says how old they are. The fast ramp is only for *acquiring* a + // version we have never had. + Tick::Failed if ever_identified => REFRESH_INTERVAL, + Tick::Failed => { + let now = acquisition_delay; + acquisition_delay = (acquisition_delay * 2).min(REFRESH_INTERVAL); + now + } + }; + tokio::time::sleep(sleep_for).await; + } +} + +async fn refresh_once(fetch: &F, metrics: &SuiClientMetrics) -> Tick +where + F: Fn() -> Fut, + Fut: Future, TransportError>>, +{ + match tokio::time::timeout(CALL_TIMEOUT, fetch()).await { + Ok(Ok(Some(info))) => { + metrics.set_sui_node_info( + info.server_version.as_deref(), + info.chain_identifier.as_deref(), + ); + Tick::Identified + } + Ok(Ok(None)) => { + metrics.set_sui_node_info_unsupported(); + Tick::Unsupported + } + // Debug, not warn: on the wedged nodes this metric describes, the Sui + // uplink fails continuously for hours, and a probe that narrates that + // once per interval adds nothing the metric does not already say. + Ok(Err(error)) => { + debug!( + ?error, + "Sui GetServiceInfo probe failed; sui_node_info unchanged" + ); + Tick::Failed + } + Err(_) => { + debug!( + timeout = ?CALL_TIMEOUT, + "Sui GetServiceInfo probe timed out; sui_node_info unchanged" + ); + Tick::Failed + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::metrics::{ + SUI_NODE_INFO_UNKNOWN, SUI_NODE_INFO_UNREPORTED, SUI_NODE_INFO_UNSUPPORTED, + }; + + fn info(server: &str) -> SuiNodeInfo { + SuiNodeInfo { + server_version: Some(server.to_string()), + chain_identifier: Some("35834a8a".to_string()), + } + } + + fn gauge(metrics: &SuiClientMetrics, server: &str, chain: &str) -> i64 { + metrics + .sui_node_info + .with_label_values(&[server, chain]) + .get() + } + + /// The eager registration is the whole reason a never-answering node is + /// visible at all: before any probe runs the client must already export + /// `unknown = 1`, with the freshness gauge at 0 to say "never". + #[test] + fn unknown_is_published_before_any_refresh() { + let metrics = SuiClientMetrics::new_for_testing(); + assert_eq!( + gauge(&metrics, SUI_NODE_INFO_UNKNOWN, SUI_NODE_INFO_UNKNOWN), + 1 + ); + assert_eq!(metrics.sui_node_info_last_success_unixtime.get(), 0); + } + + /// A successful probe flips `unknown` to 0 and the real version to 1 — + /// and `unknown` must still EXIST at 0 rather than vanish, so a query + /// spanning the transition sees a value on both sides of it. + #[tokio::test] + async fn success_flips_unknown_to_the_reported_version() { + let metrics = SuiClientMetrics::new_for_testing(); + + let tick = refresh_once(&|| async { Ok(Some(info("sui-node/1.78.1"))) }, &metrics).await; + + assert_eq!(tick, Tick::Identified); + assert_eq!( + gauge(&metrics, SUI_NODE_INFO_UNKNOWN, SUI_NODE_INFO_UNKNOWN), + 0 + ); + assert_eq!(gauge(&metrics, "sui-node/1.78.1", "35834a8a"), 1); + assert!(metrics.sui_node_info_last_success_unixtime.get() > 0); + } + + /// An operator upgrading their fullnode must read as one series going to + /// 0 and one to 1, never as two series at 1. + #[tokio::test] + async fn a_version_change_leaves_exactly_one_child_at_one() { + let metrics = SuiClientMetrics::new_for_testing(); + + refresh_once(&|| async { Ok(Some(info("sui-node/1.77.2"))) }, &metrics).await; + refresh_once(&|| async { Ok(Some(info("sui-node/1.78.1"))) }, &metrics).await; + + assert_eq!(gauge(&metrics, "sui-node/1.77.2", "35834a8a"), 0); + assert_eq!(gauge(&metrics, "sui-node/1.78.1", "35834a8a"), 1); + assert_eq!( + gauge(&metrics, SUI_NODE_INFO_UNKNOWN, SUI_NODE_INFO_UNKNOWN), + 0 + ); + } + + /// Re-reporting the same version is the steady state; it must stay at 1 + /// and keep advancing the freshness stamp. + #[tokio::test] + async fn repeating_the_same_version_is_idempotent() { + let metrics = SuiClientMetrics::new_for_testing(); + + refresh_once(&|| async { Ok(Some(info("sui-node/1.78.1"))) }, &metrics).await; + metrics.sui_node_info_last_success_unixtime.set(1); + refresh_once(&|| async { Ok(Some(info("sui-node/1.78.1"))) }, &metrics).await; + + assert_eq!(gauge(&metrics, "sui-node/1.78.1", "35834a8a"), 1); + assert!(metrics.sui_node_info_last_success_unixtime.get() > 1); + } + + /// A failing probe must change nothing at all — not the labels, not the + /// freshness stamp. Staying on `unknown` IS the report. + #[tokio::test] + async fn a_failed_probe_leaves_the_previous_labels_alone() { + let metrics = SuiClientMetrics::new_for_testing(); + + let first = refresh_once( + &|| async { Err(TransportError::Network("down".into())) }, + &metrics, + ) + .await; + + assert_eq!(first, Tick::Failed); + assert_eq!( + gauge(&metrics, SUI_NODE_INFO_UNKNOWN, SUI_NODE_INFO_UNKNOWN), + 1 + ); + assert_eq!(metrics.sui_node_info_last_success_unixtime.get(), 0); + + refresh_once(&|| async { Ok(Some(info("sui-node/1.78.1"))) }, &metrics).await; + let stamped = metrics.sui_node_info_last_success_unixtime.get(); + refresh_once( + &|| async { Err(TransportError::Network("down".into())) }, + &metrics, + ) + .await; + + assert_eq!(gauge(&metrics, "sui-node/1.78.1", "35834a8a"), 1); + assert_eq!(metrics.sui_node_info_last_success_unixtime.get(), stamped); + } + + /// A node that answers but leaves `server` unset is a working RPC, so it + /// must not be reported as `unknown` — and its answer is fresh. + #[tokio::test] + async fn an_empty_server_field_reports_unreported_not_unknown() { + let metrics = SuiClientMetrics::new_for_testing(); + + refresh_once(&|| async { Ok(Some(SuiNodeInfo::default())) }, &metrics).await; + + assert_eq!( + gauge(&metrics, SUI_NODE_INFO_UNREPORTED, SUI_NODE_INFO_UNREPORTED), + 1 + ); + assert!(metrics.sui_node_info_last_success_unixtime.get() > 0); + } + + /// The p2p-relay case: terminal, and pointedly not stamped fresh, so a + /// staleness query can exclude it by label instead of by timestamp. + #[tokio::test] + async fn an_unsupporting_transport_is_terminal_and_never_stamped_fresh() { + let metrics = SuiClientMetrics::new_for_testing(); + + let tick = refresh_once(&|| async { Ok(None) }, &metrics).await; + + assert_eq!(tick, Tick::Unsupported); + assert_eq!( + gauge( + &metrics, + SUI_NODE_INFO_UNSUPPORTED, + SUI_NODE_INFO_UNSUPPORTED + ), + 1 + ); + assert_eq!( + gauge(&metrics, SUI_NODE_INFO_UNKNOWN, SUI_NODE_INFO_UNKNOWN), + 0 + ); + assert_eq!(metrics.sui_node_info_last_success_unixtime.get(), 0); + } + + /// A hung fullnode must not hang the probe: the deadline fires and the + /// tick reports failure, leaving the loop free to try again. + #[tokio::test(start_paused = true)] + async fn a_hanging_call_times_out_instead_of_wedging_the_task() { + let metrics = SuiClientMetrics::new_for_testing(); + + let tick = refresh_once( + &|| async { + tokio::time::sleep(CALL_TIMEOUT * 10).await; + Ok(Some(info("never-arrives"))) + }, + &metrics, + ) + .await; + + assert_eq!(tick, Tick::Failed); + assert_eq!( + gauge(&metrics, SUI_NODE_INFO_UNKNOWN, SUI_NODE_INFO_UNKNOWN), + 1 + ); + } + + /// The loop must stop itself on an unsupporting transport rather than + /// re-deriving a constant every interval forever. + #[tokio::test(start_paused = true)] + async fn the_loop_exits_on_an_unsupporting_transport() { + let metrics = SuiClientMetrics::new_for_testing(); + + // Completes only because the loop returns; a poll loop would hang here. + refresh_loop(|| async { Ok(None) }, metrics.clone()).await; + + assert_eq!( + gauge( + &metrics, + SUI_NODE_INFO_UNSUPPORTED, + SUI_NODE_INFO_UNSUPPORTED + ), + 1 + ); + } + + /// Before the first success the loop retries on the short schedule; after + /// it, it settles onto the steady interval. Asserted through elapsed + /// virtual time on a paused clock. + #[tokio::test(start_paused = true)] + async fn retries_are_fast_until_identified_then_settle() { + let metrics = SuiClientMetrics::new_for_testing(); + let attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + + let started = tokio::time::Instant::now(); + let counter = attempts.clone(); + let loop_task = tokio::spawn(refresh_loop( + move || { + let n = counter.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + async move { + if n < 2 { + Err(TransportError::Network("down".into())) + } else { + Ok(Some(info("sui-node/1.78.1"))) + } + } + }, + metrics.clone(), + )); + + // Two failures (30s + 60s of backoff) then the success. + tokio::time::sleep(Duration::from_secs(91)).await; + assert_eq!(gauge(&metrics, "sui-node/1.78.1", "35834a8a"), 1); + assert_eq!(attempts.load(std::sync::atomic::Ordering::SeqCst), 3); + + // Having succeeded, it now waits the full interval rather than 120s. + tokio::time::sleep(REFRESH_INTERVAL - Duration::from_secs(60)).await; + assert_eq!(attempts.load(std::sync::atomic::Ordering::SeqCst), 3); + assert!(started.elapsed() < REFRESH_INTERVAL * 2); + + loop_task.abort(); + } +} diff --git a/crates/ika-sui-client/src/transport.rs b/crates/ika-sui-client/src/transport.rs index f0ce8bb404..8a5e677ffb 100644 --- a/crates/ika-sui-client/src/transport.rs +++ b/crates/ika-sui-client/src/transport.rs @@ -209,11 +209,51 @@ pub struct DynamicFieldPage { pub type CheckpointSummaryStream = BoxStream<'static, Result>; +/// What the Sui fullnode behind a transport reports about *itself* — as +/// opposed to about the chain. Sourced from the `GetServiceInfo` RPC. +/// +/// Both fields are `Option` because the proto marks them optional: a node that +/// answers the call is not obliged to fill them in, and a proxy in front of a +/// fullnode may strip them. `None` is therefore "the node answered but said +/// nothing", which is a different state from "the call failed" (an `Err` on +/// [`SuiTransport::get_sui_node_info`]) and from "this transport has no such +/// RPC" (`Ok(None)` on the same method). +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct SuiNodeInfo { + /// `GetServiceInfoResponse.server` — the fullnode's own software version + /// string, the gRPC analogue of the HTTP `server` header (e.g. + /// `sui-node/1.78.1-abc1234`). This is the field that answers "which Sui + /// release is this operator actually running". + pub server_version: Option, + /// `GetServiceInfoResponse.chain_id` — the genesis checkpoint digest, + /// verbatim as the node spelled it. Kept as the raw string rather than + /// parsed into a [`ChainIdentifier`]: this is a label value, and a parse + /// failure here must not cost us the `server_version` that came back in + /// the same response. + pub chain_identifier: Option, +} + #[async_trait] pub trait SuiTransport: Send + Sync { // -- chain metadata --------------------------------------------------------------------- async fn get_chain_identifier(&self) -> Result; async fn get_current_epoch(&self) -> Result; + /// Identity of the Sui fullnode this transport talks to: its software + /// version and chain id. Feeds `ika_sui_client_sui_node_info` — see + /// [`crate::metrics::SuiClientMetrics::sui_node_info`] for why the fleet + /// needs it. + /// + /// `Ok(None)` means *this transport structurally cannot answer* — the p2p + /// mirror relay speaks Ika's own wire protocol and never sees a Sui + /// fullnode's service info. That is deliberately distinct from `Err`, + /// which means a real fullnode was asked and the call failed. The + /// refresher reports the two differently, and stops polling on the former. + /// + /// Default is `Ok(None)` so that the many test doubles and relay + /// transports implementing this trait need no change. + async fn get_sui_node_info(&self) -> Result, TransportError> { + Ok(None) + } /// Sui [`Committee`] for the given epoch (or current if `None`). Used as /// a fallback when the committee ratchet's BLS proof chain is broken /// by upstream pruning of end-of-epoch checkpoints. diff --git a/dev-docs/conventions/metrics.md b/dev-docs/conventions/metrics.md index 580ffdbf2d..aeea8c0667 100644 --- a/dev-docs/conventions/metrics.md +++ b/dev-docs/conventions/metrics.md @@ -617,6 +617,8 @@ ika_split_brain_system_checkpoint_forks ika_stranded_network_key_missing_from_registry_read_condition_active ika_sui_client_chain_blob_reads ika_sui_client_rate_limited_errors_total +ika_sui_client_sui_node_info +ika_sui_client_sui_node_info_last_success_unixtime ika_sui_client_sui_rpc_errors ika_sui_connector_chain_active_system_sessions_count ika_sui_connector_chain_active_user_sessions_count