From 2461c3b5ae1ce79ce9c0c5c21c8d34f8fe0d8f30 Mon Sep 17 00:00:00 2001 From: Omer Sadika Date: Sat, 29 Aug 2026 22:57:35 +0300 Subject: [PATCH 1/2] feat(sui-client): export the connected Sui node's server version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two mainnet operators' validators wedged in early boot at consecutive epoch boundaries, both with their own Sui fullnode's RPC failing for hours beforehand (~440 and ~316 errors/hour against a fleet baseline of 0-6). The leading hypothesis for the first — an operator who never upgraded their Sui node across a release rollout — could be neither confirmed nor refuted, because nothing in the fleet's telemetry says which Sui version a validator is talking to. `sui_rpc_errors` says an uplink is unhealthy; it never says what is on the other end of it. Add `ika_sui_client_sui_node_info` (info gauge, labels `server_version` and `chain_identifier`, sourced from the fullnode's own GetServiceInfo) and `ika_sui_client_sui_node_info_last_success_unixtime` for freshness. Both live on `SuiClientMetrics`, so they reach Mimir through ika-proxy on the same registry as `sui_rpc_errors`. The info gauge is registered eagerly at `server_version="unknown"`: a validator still on that label is itself the signal that its Sui RPC has never once answered, which is exactly the state both wedged nodes were in. Refreshes flip children rather than removing them, so a version change reads as a clean transition across a scrape gap. `SuiTransport::get_sui_node_info` is a new trait method defaulting to `Ok(None)` — "this transport has no service-info RPC", the honest answer for the p2p mirror relay and one that leaves the existing test doubles untouched. Only `SuiGrpcClient` overrides it. Refreshing runs on a task spawned at client construction and aborted on drop. It is never awaited by a caller: the nodes this metric most needs to describe are the ones whose RPC hangs. Every call is deadlined at 20s, failures leave the last known labels in place, and the cadence is 12 minutes after a short ramp for the first acquisition. Probe failures deliberately do NOT feed `sui_rpc_errors` — that counter's healthy baseline is 0-6/hour, too narrow to absorb a passive probe's floor without blunting the signal the probe exists to explain. Co-Authored-By: Claude Fable 5 --- crates/ika-sui-client/src/grpc.rs | 23 +- crates/ika-sui-client/src/grpc_backend.rs | 9 + crates/ika-sui-client/src/lib.rs | 22 ++ crates/ika-sui-client/src/metrics.rs | 155 ++++++++- crates/ika-sui-client/src/node_info.rs | 406 ++++++++++++++++++++++ crates/ika-sui-client/src/transport.rs | 40 +++ 6 files changed, 652 insertions(+), 3 deletions(-) create mode 100644 crates/ika-sui-client/src/node_info.rs diff --git a/crates/ika-sui-client/src/grpc.rs b/crates/ika-sui-client/src/grpc.rs index 9a5d566bd3..2167caef7c 100644 --- a/crates/ika-sui-client/src/grpc.rs +++ b/crates/ika-sui-client/src/grpc.rs @@ -50,7 +50,7 @@ use tonic::metadata::{Ascii, MetadataKey, MetadataValue}; use crate::rate_limit::RateLimitGate; use crate::transport::{ CheckpointSummaryStream, DynamicFieldEntry, DynamicFieldPage, ExecutedTransaction, - SubmittedTransaction, SuiFundsBreakdown, SuiTransport, SuiWriter, TransportError, + SubmittedTransaction, SuiFundsBreakdown, SuiNodeInfo, SuiTransport, SuiWriter, TransportError, }; /// Sui rejects a transaction whose gas payment names more than @@ -788,6 +788,27 @@ impl SuiTransport for SuiGrpcClient { .map(|chain_identifier| chain_identifier.to_string()) } + async fn get_sui_node_info(&self) -> Result, TransportError> { + // Same `GetServiceInfo` call `get_latest_checkpoint_sequence` and + // `get_sui_chain_identifier` already use; we read two different + // fields off it. It takes no arguments and touches no store beyond + // the node's own watermarks, so it is the cheapest call the fullnode + // serves — which is what makes it safe to poll on a timer. + let mut rpc = self.rpc.clone(); + let response = self + .gated_network(async move { + rpc.ledger_client() + .get_service_info(proto::GetServiceInfoRequest::default()) + .await + }) + .await? + .into_inner(); + Ok(Some(SuiNodeInfo { + server_version: response.server_opt().map(str::to_owned), + chain_identifier: response.chain_id_opt().map(str::to_owned), + })) + } + async fn get_current_epoch(&self) -> Result { let mut rpc = self.rpc.clone(); let mut request = proto::GetEpochRequest::default(); diff --git a/crates/ika-sui-client/src/grpc_backend.rs b/crates/ika-sui-client/src/grpc_backend.rs index aa2a3d979c..43747d84d6 100644 --- a/crates/ika-sui-client/src/grpc_backend.rs +++ b/crates/ika-sui-client/src/grpc_backend.rs @@ -106,6 +106,15 @@ impl GrpcSuiClient { } } + /// The read transport this backend was built over. Handed to the + /// `ika_sui_client_sui_node_info` refresher, which needs a `'static` + /// handle to the same uplink the client reads through — so that the + /// version it reports is the version of the node actually serving this + /// client, not of some separately-opened connection. + pub fn transport(&self) -> std::sync::Arc { + self.transport.clone() + } + /// The writer uplink, or a clear error on a read-only node. Transaction /// building/submission (and the migration-sweep reads that serve it) are /// notifier-gated to a direct fullnode connection. diff --git a/crates/ika-sui-client/src/lib.rs b/crates/ika-sui-client/src/lib.rs index fb6812b7d6..5b759ec40d 100644 --- a/crates/ika-sui-client/src/lib.rs +++ b/crates/ika-sui-client/src/lib.rs @@ -46,6 +46,7 @@ pub mod ika_dwallet_transactions; pub mod ika_protocol_transactions; pub mod ika_validator_transactions; pub mod metrics; +pub mod node_info; pub mod rate_limit; pub mod transaction_builder; pub mod transaction_context; @@ -102,6 +103,11 @@ pub struct SuiClient

{ system_arg_cache: OnceCell, clock_arg_cache: OnceCell, dwallet_coordinator_arg_cache: OnceCell, + /// Keeps `ika_sui_client_sui_node_info` current for as long as this client + /// exists — the handle aborts the refresh task on drop, so a short-lived + /// CLI client does not leave a timer behind. `None` on clients built for + /// tests, which have no registry anyone scrapes. + _node_info_refresh: Option, } pub type SuiBackend = grpc_backend::GrpcSuiClient; @@ -144,6 +150,11 @@ impl SuiConnectorClient { let inner = grpc_backend::GrpcSuiClient::new_with_headers(grpc_url, headers, rate_limit_gate) .await?; + // Spawned, never awaited: identifying the fullnode must not add a + // round trip to node boot, least of all on the wedged-uplink nodes + // this metric exists to describe. + let node_info_refresh = + node_info::spawn_sui_node_info_refresh(inner.transport(), sui_client_metrics.clone()); let self_ = Self { inner, sui_client_metrics, @@ -151,6 +162,7 @@ impl SuiConnectorClient { system_arg_cache: OnceCell::new(), clock_arg_cache: OnceCell::new(), dwallet_coordinator_arg_cache: OnceCell::new(), + _node_info_refresh: Some(node_info_refresh), }; self_.describe().await?; Ok(self_) @@ -166,6 +178,11 @@ impl SuiConnectorClient { ika_network_config: IkaNetworkConfig, ) -> anyhow::Result { let inner = grpc_backend::GrpcSuiClient::with_transport(transport); + // The relay transport has no service-info RPC, so this settles on + // `server_version="unsupported"` after one probe and stops — which is + // the honest answer for a node with no fullnode of its own. + let node_info_refresh = + node_info::spawn_sui_node_info_refresh(inner.transport(), sui_client_metrics.clone()); let self_ = Self { inner, sui_client_metrics, @@ -173,6 +190,7 @@ impl SuiConnectorClient { system_arg_cache: OnceCell::new(), clock_arg_cache: OnceCell::new(), dwallet_coordinator_arg_cache: OnceCell::new(), + _node_info_refresh: Some(node_info_refresh), }; self_.describe().await?; Ok(self_) @@ -218,6 +236,10 @@ where system_arg_cache: OnceCell::new(), clock_arg_cache: OnceCell::new(), dwallet_coordinator_arg_cache: OnceCell::new(), + // Not spawned for tests: `new_for_testing` is called from + // synchronous contexts with no runtime, and nothing scrapes the + // throwaway registry it builds. + _node_info_refresh: None, } } diff --git a/crates/ika-sui-client/src/metrics.rs b/crates/ika-sui-client/src/metrics.rs index 75ebca191d..271ad6006a 100644 --- a/crates/ika-sui-client/src/metrics.rs +++ b/crates/ika-sui-client/src/metrics.rs @@ -1,9 +1,29 @@ // Copyright (c) Mysten Labs, Inc. // SPDX-License-Identifier: BSD-3-Clause-Clear -use prometheus::{IntCounterVec, Registry, register_int_counter_vec_with_registry}; -use std::sync::Arc; +use prometheus::{ + IntCounterVec, IntGauge, IntGaugeVec, Registry, register_int_counter_vec_with_registry, + register_int_gauge_vec_with_registry, register_int_gauge_with_registry, +}; use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; + +/// Label value published on [`SuiClientMetrics::sui_node_info`] from process +/// start until the node's `GetServiceInfo` first answers. A validator whose +/// series is *still* on this label is the signal: its Sui uplink has never +/// once told us what it is. +pub const SUI_NODE_INFO_UNKNOWN: &str = "unknown"; + +/// Label value for a client whose transport has no service-info RPC at all — +/// a peer-only validator reading Sui through the Ika p2p mirror relay. There +/// is no operator fullnode on the other end to have a version, so this is a +/// *healthy* terminal state and must not be read as `unknown`. +pub const SUI_NODE_INFO_UNSUPPORTED: &str = "unsupported"; + +/// Label value for a node that answered `GetServiceInfo` but left the field +/// empty (both fields are `optional` in the proto, and a proxy in front of a +/// fullnode can strip them). Distinct from `unknown`: the RPC works. +pub const SUI_NODE_INFO_UNREPORTED: &str = "unreported"; /// Process-wide counter for chain-side calls to /// `get_network_encryption_key_with_full_data_by_epoch`. Test @@ -50,6 +70,60 @@ pub struct SuiClientMetrics { /// keeps the endpoint refusing. Fleet-wide this should be flat at zero; /// growth on a single node points at that node's endpoint, not at ika. pub sui_rate_limited_errors: IntCounterVec, + /// Info-style gauge (always `1` on exactly one child) identifying the Sui + /// fullnode this validator's uplink is pointed at: `server_version` is the + /// node's own software version string, `chain_identifier` its genesis + /// digest. + /// + /// # Why + /// + /// Two mainnet validators wedged in early boot at consecutive epoch + /// boundaries (2026-08-28 and 08-29). Both had their own Sui fullnode's + /// RPC failing for hours beforehand — one at ~440 errors/hour, the other + /// ~316, against a fleet baseline of 0–6. The leading hypothesis for the + /// first is simply that the operator never upgraded their Sui node across + /// a Sui release rollout. We could not confirm or refute that, because + /// nothing anywhere in the fleet's telemetry says which Sui version a + /// validator is talking to: `sui_rpc_errors` tells us an uplink is + /// unhealthy, never *what* is on the other end of it. This gauge closes + /// that gap — one `group by (server_version)` over the fleet shows the + /// version spread and singles out the operator who is behind. + /// + /// # Semantics + /// + /// Registered eagerly with `server_version="unknown"` at `1`, so a node + /// that never gets an answer still publishes a series — a validator stuck + /// on `unknown` is itself the signal that its Sui RPC never replied. + /// Refreshes never *remove* a child, they flip the old one to `0` and the + /// new one to `1`, so a version change reads as a clean 1→0 / 0→1 + /// transition across a scrape gap instead of a disappearing series. + /// + /// # Cardinality + /// + /// One child at `1` per node at a time, plus one retired child at `0` per + /// version the node has ever reported — in practice one or two per + /// process lifetime, since an operator's Sui version changes only when + /// they upgrade and the process restarts on ours. Negligible. + pub sui_node_info: IntGaugeVec, + /// Unix seconds of the last successful `GetServiceInfo`; `0` until the + /// first one lands. + /// + /// Freshness companion to [`Self::sui_node_info`], which on its own cannot + /// distinguish "this really is the version, confirmed minutes ago" from + /// "this was the version hours ago and the RPC has been dead since" — the + /// info gauge holds its last known value by design. `0` additionally + /// separates "never answered" from "answered once and went stale", which + /// matters because a boot-time wedge produces the former and a mid-life + /// failure the latter. + /// + /// Left at `0` forever on a peer-only node (see + /// [`SUI_NODE_INFO_UNSUPPORTED`]) — there is nothing to refresh, so pair + /// any staleness query with `server_version != "unsupported"`. + pub sui_node_info_last_success_unixtime: IntGauge, + /// The `(server_version, chain_identifier)` pair currently published at + /// `1`. Held so a refresh can flip exactly the previous child to `0` + /// without enumerating (or resetting) the vec. + active_sui_node_info: Arc>, } impl SuiClientMetrics { @@ -76,6 +150,23 @@ impl SuiClientMetrics { registry, ) .unwrap(), + sui_node_info: register_int_gauge_vec_with_registry!( + "ika_sui_client_sui_node_info", + "Info gauge (always 1) identifying the Sui fullnode this validator's uplink is connected to, by its reported server version and chain id", + &["server_version", "chain_identifier"], + registry, + ) + .unwrap(), + sui_node_info_last_success_unixtime: register_int_gauge_with_registry!( + "ika_sui_client_sui_node_info_last_success_unixtime", + "Unix seconds of the last successful Sui GetServiceInfo call; 0 until the first one succeeds", + registry, + ) + .unwrap(), + active_sui_node_info: Arc::new(Mutex::new([ + SUI_NODE_INFO_UNKNOWN.to_string(), + SUI_NODE_INFO_UNKNOWN.to_string(), + ])), }; // Publish both series from process start so a dashboard/alert can tell // "no rate limiting" apart from "this node never reported". @@ -87,6 +178,14 @@ impl SuiClientMetrics { .with_label_values(&[signal.label()]) .inc_by(0); } + // Same reasoning, one step stronger: the `unknown` child is not just a + // zero placeholder, it is the value that stays at 1 for a node whose + // Sui RPC never answers. It has to exist before the first refresh — + // which for a wedged node is a refresh that never completes. + this.sui_node_info + .with_label_values(&[SUI_NODE_INFO_UNKNOWN, SUI_NODE_INFO_UNKNOWN]) + .set(1); + this.sui_node_info_last_success_unixtime.set(0); Arc::new(this) } @@ -94,4 +193,56 @@ impl SuiClientMetrics { let registry = Registry::new(); Self::new(®istry) } + + /// Publish a fresh `GetServiceInfo` answer: flip the previously-active + /// child of [`Self::sui_node_info`] to `0`, the new one to `1`, and stamp + /// [`Self::sui_node_info_last_success_unixtime`]. + /// + /// Re-reporting the same pair is the common case (the version only changes + /// when the operator upgrades) and is cheap: it re-asserts `1` and + /// re-stamps the freshness gauge without touching any other child. + pub fn set_sui_node_info(&self, server_version: Option<&str>, chain_identifier: Option<&str>) { + self.flip_sui_node_info( + server_version.unwrap_or(SUI_NODE_INFO_UNREPORTED), + chain_identifier.unwrap_or(SUI_NODE_INFO_UNREPORTED), + ); + self.sui_node_info_last_success_unixtime + .set(unix_seconds_now()); + } + + /// Mark this client's uplink as having no service-info RPC — a peer-only + /// validator reading through the p2p mirror relay. + /// + /// Deliberately does *not* stamp the freshness gauge: there is no fullnode + /// to be fresh about, and leaving it at `0` keeps "never answered" and + /// "cannot answer" apart in queries by the label alone. + pub fn set_sui_node_info_unsupported(&self) { + self.flip_sui_node_info(SUI_NODE_INFO_UNSUPPORTED, SUI_NODE_INFO_UNSUPPORTED); + } + + fn flip_sui_node_info(&self, server_version: &str, chain_identifier: &str) { + // A poisoned lock here carries no invariant worth propagating — the + // guarded value is two label strings — and this runs on a background + // refresh whose whole contract is to never disturb the client. + let mut active = self + .active_sui_node_info + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + if active[0] != server_version || active[1] != chain_identifier { + self.sui_node_info + .with_label_values(&[active[0].as_str(), active[1].as_str()]) + .set(0); + *active = [server_version.to_string(), chain_identifier.to_string()]; + } + self.sui_node_info + .with_label_values(&[server_version, chain_identifier]) + .set(1); + } +} + +fn unix_seconds_now() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs() as i64) + .unwrap_or(0) } diff --git a/crates/ika-sui-client/src/node_info.rs b/crates/ika-sui-client/src/node_info.rs new file mode 100644 index 0000000000..3d13c0dffd --- /dev/null +++ b/crates/ika-sui-client/src/node_info.rs @@ -0,0 +1,406 @@ +// Copyright (c) dWallet Labs, Ltd. +// SPDX-License-Identifier: BSD-3-Clause-Clear + +//! Background refresher for `ika_sui_client_sui_node_info`. +//! +//! # Why this is a background task +//! +//! The metric exists because an operator's *outdated or failing* Sui fullnode +//! is invisible to the fleet (see +//! [`crate::metrics::SuiClientMetrics::sui_node_info`]). The nodes it most +//! needs to describe are therefore precisely the nodes whose RPC is slow, +//! hanging, or dead — so identifying them must never sit on a code path any +//! real client operation waits for. Nothing here is awaited by a caller: the +//! task is spawned at client construction, every call is deadlined, and every +//! failure is a no-op that leaves the last known labels in place. +//! +//! # Why failures are not counted in `sui_rpc_errors` +//! +//! Deliberate. `ika_sui_client_sui_rpc_errors` is the fleet's Sui-uplink +//! health signal, and its healthy baseline is 0–6 errors/hour — a band narrow +//! enough that a passive probe contributing its own floor to it would blunt +//! exactly the signal the probe exists to explain. A probe failing while the +//! node's real reads succeed is also not an incident, and would be a false +//! positive in that counter. The probe's failures are already fully +//! observable in its own two metrics: the `server_version` label stays at +//! `unknown` (or goes stale) and +//! `ika_sui_client_sui_node_info_last_success_unixtime` stops advancing. +//! +//! One counter the refresh *does* still feed, unavoidably and correctly: +//! calls go through the node's shared [`crate::rate_limit::RateLimitGate`], +//! so a rate-limited response is classified like any other and increments +//! `ika_sui_client_rate_limited_errors_total`. At a maximum of five calls an +//! hour that is negligible, and if the endpoint really is refusing, saying so +//! is right. + +use std::future::Future; +use std::sync::Arc; +use std::time::Duration; + +use tracing::{debug, info}; + +use crate::metrics::SuiClientMetrics; +use crate::transport::{SuiNodeInfo, SuiTransport, TransportError}; + +/// Steady-state refresh cadence. An operator's Sui version changes only when +/// they deploy an upgrade, so minutes of staleness cost nothing; the interval +/// is set by wanting a fleet-wide version census to converge within a Grafana +/// panel's typical lookback, not by any need for freshness. +const REFRESH_INTERVAL: Duration = Duration::from_secs(12 * 60); + +/// Delay before the second attempt while the node has *never* been +/// identified, doubling up to [`REFRESH_INTERVAL`]. A node that answers on +/// the first try never uses this; one whose uplink is flaky at boot gets +/// identified in tens of seconds instead of waiting out a full interval, +/// without turning a sustained outage into a polling loop. +const INITIAL_RETRY_INTERVAL: Duration = Duration::from_secs(30); + +/// Per-call deadline. A hung fullnode is one of the failure modes this metric +/// was built to expose, so the probe cannot be allowed to hang with it: a +/// task blocked forever in `GetServiceInfo` would leave the labels frozen and +/// never retry, which looks identical to a healthy node that stopped changing. +const CALL_TIMEOUT: Duration = Duration::from_secs(20); + +/// Handle to the spawned refresh task; aborts it on drop so the task's +/// lifetime is exactly the owning client's. +#[derive(Debug)] +pub struct SuiNodeInfoRefresh(tokio::task::JoinHandle<()>); + +impl Drop for SuiNodeInfoRefresh { + fn drop(&mut self) { + self.0.abort(); + } +} + +/// Start refreshing `ika_sui_client_sui_node_info` from `transport`. +/// +/// Must be called from inside a Tokio runtime (every construction path is +/// already `async`). Returns immediately; the first probe runs on the spawned +/// task, not here. +pub fn spawn_sui_node_info_refresh( + transport: Arc, + metrics: Arc, +) -> SuiNodeInfoRefresh { + SuiNodeInfoRefresh(tokio::spawn(refresh_loop( + move || { + let transport = transport.clone(); + async move { transport.get_sui_node_info().await } + }, + metrics, + ))) +} + +/// Result of one probe, in the terms the loop schedules on. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Tick { + /// The node answered; labels now describe it. + Identified, + /// This transport structurally cannot answer. Terminal — nothing about it + /// will change on a later attempt. + Unsupported, + /// The call failed or timed out. Previous labels are left untouched. + Failed, +} + +async fn refresh_loop(fetch: F, metrics: Arc) +where + F: Fn() -> Fut + Send + 'static, + Fut: Future, TransportError>> + Send, +{ + let mut acquisition_delay = INITIAL_RETRY_INTERVAL; + let mut ever_identified = false; + loop { + let sleep_for = match refresh_once(&fetch, &metrics).await { + Tick::Identified => { + if !ever_identified { + ever_identified = true; + info!("identified the connected Sui node; publishing sui_node_info"); + } + REFRESH_INTERVAL + } + // Nothing to poll for. Leaving the task alive would burn a timer + // forever to re-derive a constant. + Tick::Unsupported => return, + // Once we know the version, a failed refresh is not urgent — the + // last known labels stay published and the freshness gauge already + // says how old they are. The fast ramp is only for *acquiring* a + // version we have never had. + Tick::Failed if ever_identified => REFRESH_INTERVAL, + Tick::Failed => { + let now = acquisition_delay; + acquisition_delay = (acquisition_delay * 2).min(REFRESH_INTERVAL); + now + } + }; + tokio::time::sleep(sleep_for).await; + } +} + +async fn refresh_once(fetch: &F, metrics: &SuiClientMetrics) -> Tick +where + F: Fn() -> Fut, + Fut: Future, TransportError>>, +{ + match tokio::time::timeout(CALL_TIMEOUT, fetch()).await { + Ok(Ok(Some(info))) => { + metrics.set_sui_node_info( + info.server_version.as_deref(), + info.chain_identifier.as_deref(), + ); + Tick::Identified + } + Ok(Ok(None)) => { + metrics.set_sui_node_info_unsupported(); + Tick::Unsupported + } + // Debug, not warn: on the wedged nodes this metric describes, the Sui + // uplink fails continuously for hours, and a probe that narrates that + // once per interval adds nothing the metric does not already say. + Ok(Err(error)) => { + debug!( + ?error, + "Sui GetServiceInfo probe failed; sui_node_info unchanged" + ); + Tick::Failed + } + Err(_) => { + debug!( + timeout = ?CALL_TIMEOUT, + "Sui GetServiceInfo probe timed out; sui_node_info unchanged" + ); + Tick::Failed + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::metrics::{ + SUI_NODE_INFO_UNKNOWN, SUI_NODE_INFO_UNREPORTED, SUI_NODE_INFO_UNSUPPORTED, + }; + + fn info(server: &str) -> SuiNodeInfo { + SuiNodeInfo { + server_version: Some(server.to_string()), + chain_identifier: Some("35834a8a".to_string()), + } + } + + fn gauge(metrics: &SuiClientMetrics, server: &str, chain: &str) -> i64 { + metrics + .sui_node_info + .with_label_values(&[server, chain]) + .get() + } + + /// The eager registration is the whole reason a never-answering node is + /// visible at all: before any probe runs the client must already export + /// `unknown = 1`, with the freshness gauge at 0 to say "never". + #[test] + fn unknown_is_published_before_any_refresh() { + let metrics = SuiClientMetrics::new_for_testing(); + assert_eq!( + gauge(&metrics, SUI_NODE_INFO_UNKNOWN, SUI_NODE_INFO_UNKNOWN), + 1 + ); + assert_eq!(metrics.sui_node_info_last_success_unixtime.get(), 0); + } + + /// A successful probe flips `unknown` to 0 and the real version to 1 — + /// and `unknown` must still EXIST at 0 rather than vanish, so a query + /// spanning the transition sees a value on both sides of it. + #[tokio::test] + async fn success_flips_unknown_to_the_reported_version() { + let metrics = SuiClientMetrics::new_for_testing(); + + let tick = refresh_once(&|| async { Ok(Some(info("sui-node/1.78.1"))) }, &metrics).await; + + assert_eq!(tick, Tick::Identified); + assert_eq!( + gauge(&metrics, SUI_NODE_INFO_UNKNOWN, SUI_NODE_INFO_UNKNOWN), + 0 + ); + assert_eq!(gauge(&metrics, "sui-node/1.78.1", "35834a8a"), 1); + assert!(metrics.sui_node_info_last_success_unixtime.get() > 0); + } + + /// An operator upgrading their fullnode must read as one series going to + /// 0 and one to 1, never as two series at 1. + #[tokio::test] + async fn a_version_change_leaves_exactly_one_child_at_one() { + let metrics = SuiClientMetrics::new_for_testing(); + + refresh_once(&|| async { Ok(Some(info("sui-node/1.77.2"))) }, &metrics).await; + refresh_once(&|| async { Ok(Some(info("sui-node/1.78.1"))) }, &metrics).await; + + assert_eq!(gauge(&metrics, "sui-node/1.77.2", "35834a8a"), 0); + assert_eq!(gauge(&metrics, "sui-node/1.78.1", "35834a8a"), 1); + assert_eq!( + gauge(&metrics, SUI_NODE_INFO_UNKNOWN, SUI_NODE_INFO_UNKNOWN), + 0 + ); + } + + /// Re-reporting the same version is the steady state; it must stay at 1 + /// and keep advancing the freshness stamp. + #[tokio::test] + async fn repeating_the_same_version_is_idempotent() { + let metrics = SuiClientMetrics::new_for_testing(); + + refresh_once(&|| async { Ok(Some(info("sui-node/1.78.1"))) }, &metrics).await; + metrics.sui_node_info_last_success_unixtime.set(1); + refresh_once(&|| async { Ok(Some(info("sui-node/1.78.1"))) }, &metrics).await; + + assert_eq!(gauge(&metrics, "sui-node/1.78.1", "35834a8a"), 1); + assert!(metrics.sui_node_info_last_success_unixtime.get() > 1); + } + + /// A failing probe must change nothing at all — not the labels, not the + /// freshness stamp. Staying on `unknown` IS the report. + #[tokio::test] + async fn a_failed_probe_leaves_the_previous_labels_alone() { + let metrics = SuiClientMetrics::new_for_testing(); + + let first = refresh_once( + &|| async { Err(TransportError::Network("down".into())) }, + &metrics, + ) + .await; + + assert_eq!(first, Tick::Failed); + assert_eq!( + gauge(&metrics, SUI_NODE_INFO_UNKNOWN, SUI_NODE_INFO_UNKNOWN), + 1 + ); + assert_eq!(metrics.sui_node_info_last_success_unixtime.get(), 0); + + refresh_once(&|| async { Ok(Some(info("sui-node/1.78.1"))) }, &metrics).await; + let stamped = metrics.sui_node_info_last_success_unixtime.get(); + refresh_once( + &|| async { Err(TransportError::Network("down".into())) }, + &metrics, + ) + .await; + + assert_eq!(gauge(&metrics, "sui-node/1.78.1", "35834a8a"), 1); + assert_eq!(metrics.sui_node_info_last_success_unixtime.get(), stamped); + } + + /// A node that answers but leaves `server` unset is a working RPC, so it + /// must not be reported as `unknown` — and its answer is fresh. + #[tokio::test] + async fn an_empty_server_field_reports_unreported_not_unknown() { + let metrics = SuiClientMetrics::new_for_testing(); + + refresh_once(&|| async { Ok(Some(SuiNodeInfo::default())) }, &metrics).await; + + assert_eq!( + gauge(&metrics, SUI_NODE_INFO_UNREPORTED, SUI_NODE_INFO_UNREPORTED), + 1 + ); + assert!(metrics.sui_node_info_last_success_unixtime.get() > 0); + } + + /// The p2p-relay case: terminal, and pointedly not stamped fresh, so a + /// staleness query can exclude it by label instead of by timestamp. + #[tokio::test] + async fn an_unsupporting_transport_is_terminal_and_never_stamped_fresh() { + let metrics = SuiClientMetrics::new_for_testing(); + + let tick = refresh_once(&|| async { Ok(None) }, &metrics).await; + + assert_eq!(tick, Tick::Unsupported); + assert_eq!( + gauge( + &metrics, + SUI_NODE_INFO_UNSUPPORTED, + SUI_NODE_INFO_UNSUPPORTED + ), + 1 + ); + assert_eq!( + gauge(&metrics, SUI_NODE_INFO_UNKNOWN, SUI_NODE_INFO_UNKNOWN), + 0 + ); + assert_eq!(metrics.sui_node_info_last_success_unixtime.get(), 0); + } + + /// A hung fullnode must not hang the probe: the deadline fires and the + /// tick reports failure, leaving the loop free to try again. + #[tokio::test(start_paused = true)] + async fn a_hanging_call_times_out_instead_of_wedging_the_task() { + let metrics = SuiClientMetrics::new_for_testing(); + + let tick = refresh_once( + &|| async { + tokio::time::sleep(CALL_TIMEOUT * 10).await; + Ok(Some(info("never-arrives"))) + }, + &metrics, + ) + .await; + + assert_eq!(tick, Tick::Failed); + assert_eq!( + gauge(&metrics, SUI_NODE_INFO_UNKNOWN, SUI_NODE_INFO_UNKNOWN), + 1 + ); + } + + /// The loop must stop itself on an unsupporting transport rather than + /// re-deriving a constant every interval forever. + #[tokio::test(start_paused = true)] + async fn the_loop_exits_on_an_unsupporting_transport() { + let metrics = SuiClientMetrics::new_for_testing(); + + // Completes only because the loop returns; a poll loop would hang here. + refresh_loop(|| async { Ok(None) }, metrics.clone()).await; + + assert_eq!( + gauge( + &metrics, + SUI_NODE_INFO_UNSUPPORTED, + SUI_NODE_INFO_UNSUPPORTED + ), + 1 + ); + } + + /// Before the first success the loop retries on the short schedule; after + /// it, it settles onto the steady interval. Asserted through elapsed + /// virtual time on a paused clock. + #[tokio::test(start_paused = true)] + async fn retries_are_fast_until_identified_then_settle() { + let metrics = SuiClientMetrics::new_for_testing(); + let attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + + let started = tokio::time::Instant::now(); + let counter = attempts.clone(); + let loop_task = tokio::spawn(refresh_loop( + move || { + let n = counter.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + async move { + if n < 2 { + Err(TransportError::Network("down".into())) + } else { + Ok(Some(info("sui-node/1.78.1"))) + } + } + }, + metrics.clone(), + )); + + // Two failures (30s + 60s of backoff) then the success. + tokio::time::sleep(Duration::from_secs(91)).await; + assert_eq!(gauge(&metrics, "sui-node/1.78.1", "35834a8a"), 1); + assert_eq!(attempts.load(std::sync::atomic::Ordering::SeqCst), 3); + + // Having succeeded, it now waits the full interval rather than 120s. + tokio::time::sleep(REFRESH_INTERVAL - Duration::from_secs(60)).await; + assert_eq!(attempts.load(std::sync::atomic::Ordering::SeqCst), 3); + assert!(started.elapsed() < REFRESH_INTERVAL * 2); + + loop_task.abort(); + } +} diff --git a/crates/ika-sui-client/src/transport.rs b/crates/ika-sui-client/src/transport.rs index f0ce8bb404..8a5e677ffb 100644 --- a/crates/ika-sui-client/src/transport.rs +++ b/crates/ika-sui-client/src/transport.rs @@ -209,11 +209,51 @@ pub struct DynamicFieldPage { pub type CheckpointSummaryStream = BoxStream<'static, Result>; +/// What the Sui fullnode behind a transport reports about *itself* — as +/// opposed to about the chain. Sourced from the `GetServiceInfo` RPC. +/// +/// Both fields are `Option` because the proto marks them optional: a node that +/// answers the call is not obliged to fill them in, and a proxy in front of a +/// fullnode may strip them. `None` is therefore "the node answered but said +/// nothing", which is a different state from "the call failed" (an `Err` on +/// [`SuiTransport::get_sui_node_info`]) and from "this transport has no such +/// RPC" (`Ok(None)` on the same method). +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct SuiNodeInfo { + /// `GetServiceInfoResponse.server` — the fullnode's own software version + /// string, the gRPC analogue of the HTTP `server` header (e.g. + /// `sui-node/1.78.1-abc1234`). This is the field that answers "which Sui + /// release is this operator actually running". + pub server_version: Option, + /// `GetServiceInfoResponse.chain_id` — the genesis checkpoint digest, + /// verbatim as the node spelled it. Kept as the raw string rather than + /// parsed into a [`ChainIdentifier`]: this is a label value, and a parse + /// failure here must not cost us the `server_version` that came back in + /// the same response. + pub chain_identifier: Option, +} + #[async_trait] pub trait SuiTransport: Send + Sync { // -- chain metadata --------------------------------------------------------------------- async fn get_chain_identifier(&self) -> Result; async fn get_current_epoch(&self) -> Result; + /// Identity of the Sui fullnode this transport talks to: its software + /// version and chain id. Feeds `ika_sui_client_sui_node_info` — see + /// [`crate::metrics::SuiClientMetrics::sui_node_info`] for why the fleet + /// needs it. + /// + /// `Ok(None)` means *this transport structurally cannot answer* — the p2p + /// mirror relay speaks Ika's own wire protocol and never sees a Sui + /// fullnode's service info. That is deliberately distinct from `Err`, + /// which means a real fullnode was asked and the call failed. The + /// refresher reports the two differently, and stops polling on the former. + /// + /// Default is `Ok(None)` so that the many test doubles and relay + /// transports implementing this trait need no change. + async fn get_sui_node_info(&self) -> Result, TransportError> { + Ok(None) + } /// Sui [`Committee`] for the given epoch (or current if `None`). Used as /// a fallback when the committee ratchet's BLS proof chain is broken /// by upstream pruning of end-of-epoch checkpoints. From b51c36842bc5ef9be753c11d017da4447301cfba Mon Sep 17 00:00:00 2001 From: Omer Sadika Date: Sun, 30 Aug 2026 18:38:34 +0300 Subject: [PATCH 2/2] docs(metrics): add the sui-node-info pair to the generated inventory ./scripts/check-metric-names.sh --list regeneration for the two metrics this PR introduces; the Format Check convention gate enforces the block. Co-Authored-By: Claude Fable 5 --- dev-docs/conventions/metrics.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev-docs/conventions/metrics.md b/dev-docs/conventions/metrics.md index 580ffdbf2d..aeea8c0667 100644 --- a/dev-docs/conventions/metrics.md +++ b/dev-docs/conventions/metrics.md @@ -617,6 +617,8 @@ ika_split_brain_system_checkpoint_forks ika_stranded_network_key_missing_from_registry_read_condition_active ika_sui_client_chain_blob_reads ika_sui_client_rate_limited_errors_total +ika_sui_client_sui_node_info +ika_sui_client_sui_node_info_last_success_unixtime ika_sui_client_sui_rpc_errors ika_sui_connector_chain_active_system_sessions_count ika_sui_connector_chain_active_user_sessions_count