diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs index 37dc0e00185..38a7ed051d3 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs @@ -1,5 +1,6 @@ use crate::{ - DerivationActorRequest, DerivationEngineClient, NodeActor, actors::derivation::DerivationError, + DerivationActorRequest, DerivationEngineClient, Metrics, NodeActor, + actors::derivation::DerivationError, }; use alloy_eips::BlockNumHash; use alloy_primitives::BlockHash; @@ -8,6 +9,7 @@ use jsonrpsee::core::ClientError; use kona_derive::{ChainProvider, Signal}; use kona_engine::{EngineState, FinalityViolation, FinalizeBlockId}; use kona_protocol::{BlockInfo, L2BlockInfo, SyncStatus}; +use kona_providers_alloy::AlloyChainProviderError; use kona_rpc::RollupNodeApiClient; use thiserror::Error; use tokio::{ @@ -39,6 +41,23 @@ impl SyncStatusSource for C { } } +/// The error of an L1 provider that the delegate's L1 blocks are checked against. +/// +/// It tells a block the provider does not have apart from a failed lookup: a delegate's +/// `current_l1` may be ahead of our L1 provider, and that alone does not reject its heads. +pub trait L1LookupError { + /// Whether the provider has no block at the requested height. + fn is_block_not_found(&self) -> bool; +} + +/// A confirmation-depth miss is also reported as [`AlloyChainProviderError::BlockNotFound`], so +/// the delegate actor's provider must not have a confirmation depth. +impl L1LookupError for AlloyChainProviderError { + fn is_block_not_found(&self) -> bool { + matches!(self, Self::BlockNotFound(_)) + } +} + /// The [`NodeActor`] for the delegate derivation sub-routine. /// /// This actor is responsible for receiving messages from [`NodeActor`]s and polls @@ -54,6 +73,7 @@ where DerivationEngineClient_: DerivationEngineClient, Delegate: SyncStatusSource, L1Provider: ChainProvider, + L1Provider::Error: L1LookupError, { /// The channel on which all inbound requests are received by the [`DelegateDerivationActor`]. inbound_request_rx: mpsc::Receiver, @@ -63,6 +83,9 @@ where /// Source of the sync status of the node that derivation is delegated to. delegate: Delegate, /// L1 provider for validating L1 info for derivation delegation. + /// + /// A block it does not have must be reported as not found, not as a failed lookup: the + /// `current_l1` check tolerates not-found. l1_provider: L1Provider, /// Publishes the delegate's `current_l1`: the L1 block that its derivation has reached. derivation_origin_tx: watch::Sender>, @@ -94,6 +117,7 @@ where DerivationEngineClient_: DerivationEngineClient + 'static, Delegate: SyncStatusSource + 'static, L1Provider: ChainProvider + 'static, + L1Provider::Error: L1LookupError, { /// Creates a new instance of the [`DelegateDerivationActor`]. pub fn new( @@ -129,6 +153,7 @@ where DerivationEngineClient_: DerivationEngineClient + 'static, Delegate: SyncStatusSource + 'static, L1Provider: ChainProvider + Send + 'static, + L1Provider::Error: L1LookupError, { type Error = DerivationError; @@ -160,6 +185,7 @@ where DerivationEngineClient_: DerivationEngineClient + 'static, Delegate: SyncStatusSource + 'static, L1Provider: ChainProvider + 'static, + L1Provider::Error: L1LookupError, { /// Validates a single L1 block height and hash against the canonical L1 chain. async fn validate_l1_block( @@ -168,14 +194,19 @@ where l1_block_number: u64, expected_hash: BlockHash, ) -> Result<(), DerivationDelegationError> { - let block = self - .l1_provider - .block_info_by_number(l1_block_number) - .await - .map_err(|e| DerivationDelegationError::L1Provider(e.to_string()))?; + let block = self.l1_provider.block_info_by_number(l1_block_number).await.map_err(|e| { + if e.is_block_not_found() { + DerivationDelegationError::BlockNotFound { + context: context.to_string(), + number: l1_block_number, + } + } else { + DerivationDelegationError::Provider(e.to_string()) + } + })?; if block.hash != expected_hash { - return Err(DerivationDelegationError::L1ValidationFailed { + return Err(DerivationDelegationError::HashMismatch { context: context.to_string(), number: l1_block_number, expected: expected_hash, @@ -215,16 +246,20 @@ where /// Only a fork at the engine's finalized height is caught here. A chain that forked below it /// and reports a higher finalized block passes, and the execution layer rejects the forkchoice /// update that would move onto it. + /// + /// Returns an error if the sync status must not be applied. If only `current_l1` is not on + /// our L1 chain yet, the heads can still be applied. async fn validate_sync_status( &mut self, v: &SyncStatus, - ) -> Result<(), DerivationDelegationError> { + ) -> Result { Self::validate_label_order(v)?; self.engine_state_rx .borrow() .sync_state .ensure_respects_finality(v.finalized_l2.block_info.id())?; + const CURRENT_L1: &str = "Current L1"; let checks = [ ("L1 Origin of Safe L2", v.safe_l2.l1_origin.number, v.safe_l2.l1_origin.hash), ( @@ -232,7 +267,7 @@ where v.finalized_l2.l1_origin.number, v.finalized_l2.l1_origin.hash, ), - ("Current L1", v.current_l1.number, v.current_l1.hash), + (CURRENT_L1, v.current_l1.number, v.current_l1.hash), ]; let mut validated_l1_blocks = Vec::with_capacity(checks.len()); for (context, number, hash) in checks { @@ -240,10 +275,31 @@ where if validated_l1_blocks.contains(&(number, hash)) { continue; } - self.validate_l1_block(context, number, hash).await?; - validated_l1_blocks.push((number, hash)); + match self.validate_l1_block(context, number, hash).await { + Ok(()) => validated_l1_blocks.push((number, hash)), + // Our L1 provider is behind the delegate's: it does not have this block yet, for + // example while a load-balanced L1 endpoint serves from a lagging backend. The L1 + // origins of the heads are verified, so the heads can be applied. The L1 blocks + // between the safe head's origin and `current_l1` stay unverified until a later + // poll finds a `current_l1` on our chain. If the delegate is on an L1 fork, that + // poll rejects the status with a hash mismatch, but heads already applied are not + // reverted. + Err(err @ DerivationDelegationError::BlockNotFound { .. }) + if context == CURRENT_L1 => + { + info!( + target: "derivation", + error = %err, + "Delegate current L1 not found on our L1 chain yet, not updating derivation origin" + ); + metrics::counter!(Metrics::DERIVATION_DELEGATE_CURRENT_L1_NOT_FOUND) + .increment(1); + return Ok(CurrentL1Check::NotFound); + } + Err(err) => return Err(err), + } } - Ok(()) + Ok(CurrentL1Check::Verified) } /// Logs a rejected delegate status at `warn!` when the reason changed, and at `debug!` while @@ -296,15 +352,20 @@ where } }; - if let Err(err) = self.validate_sync_status(&sync_status).await { - // Validation failures here are expected to be transient, so we skip processing - // this sync status and continue delegating derivation instead of treating it as - // fatal. - self.log_rejected_status(&err); - return Ok(()); + match self.validate_sync_status(&sync_status).await { + Ok(CurrentL1Check::Verified) => { + self.derivation_origin_tx.send_replace(Some(sync_status.current_l1)); + } + Ok(CurrentL1Check::NotFound) => {} + Err(err) => { + // Validation failures here are expected to be transient, so we skip processing + // this sync status and continue delegating derivation instead of treating it as + // fatal. + self.log_rejected_status(&err); + return Ok(()); + } } self.last_rejection = None; - self.derivation_origin_tx.send_replace(Some(sync_status.current_l1)); if self.is_lagging_safe_head_forwarded(&sync_status.safe_l2) { debug!( @@ -370,8 +431,23 @@ where } } +/// Whether a delegate's `current_l1` is on our L1 chain, for a sync status whose heads' L1 origins +/// are. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum CurrentL1Check { + /// `current_l1` is on our L1 chain. + Verified, + /// Our L1 provider does not have `current_l1` yet. The heads can be applied, but the + /// derivation origin must not move. + NotFound, +} + #[derive(Error, Debug)] enum DerivationDelegationError { + /// The L1 provider does not have a block at this height. + #[error("L1 block not found in {context} at block {number}")] + BlockNotFound { context: String, number: u64 }, + /// The delegate's finalized L2 block is ahead of its own safe head. #[error("Finalized L2 {finalized} is ahead of safe L2 {safe}")] FinalizedAheadOfSafe { finalized: u64, safe: u64 }, @@ -381,13 +457,13 @@ enum DerivationDelegationError { #[error("Finalized L2 moves the engine's finality back: {0}")] FinalizedL2BreaksFinality(#[from] FinalityViolation), - /// The L1 provider returned an error (network, RPC, etc.) - #[error("L1 provider error: {0}")] - L1Provider(String), - /// The hash provided by the derivation delegation does not match the canonical chain. #[error("L1 inconsistency in {context} at block {number}: expected {expected}, got {actual}")] - L1ValidationFailed { context: String, number: u64, expected: BlockHash, actual: BlockHash }, + HashMismatch { context: String, number: u64, expected: BlockHash, actual: BlockHash }, + + /// The L1 provider returned an error (network, RPC, etc.) + #[error("L1 provider error: {0}")] + Provider(String), /// The delegate's safe head is ahead of its own local safe head. #[error("Safe L2 {safe} is ahead of local safe L2 {local_safe}")] diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/mod.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/mod.rs index c965fa802f6..b872ddf4fc0 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/mod.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/mod.rs @@ -1,5 +1,5 @@ mod actor; -pub use actor::{DelegateDerivationActor, SyncStatusSource}; +pub use actor::{DelegateDerivationActor, L1LookupError, SyncStatusSource}; #[cfg(test)] mod tests; diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs index b586f878488..0f9c1ec33e9 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs @@ -1,9 +1,12 @@ -use super::{DelegateDerivationActor, SyncStatusSource, actor::DERIVATION_DELEGATE_POLL_INTERVAL}; +use super::{ + DelegateDerivationActor, L1LookupError, SyncStatusSource, + actor::DERIVATION_DELEGATE_POLL_INTERVAL, +}; use crate::{DerivationActorRequest, EngineActorRequest, NodeActor, QueuedDerivationEngineClient}; use alloy_eips::BlockNumHash; use alloy_primitives::B256; use async_trait::async_trait; -use kona_derive::test_utils::TestChainProvider; +use kona_derive::test_utils::{TestChainProvider, TestProviderError}; use kona_engine::{ ConsolidateInput, EngineState, EngineSyncState, EngineSyncStateUpdate, FinalizeBlockId, }; @@ -18,6 +21,14 @@ use tokio::{ const SAFE_ORIGIN: u64 = 1; const FINALIZED_ORIGIN: u64 = 2; const CURRENT_L1: u64 = 3; +/// An L1 block number our L1 provider has no block at. +const NOT_ON_OUR_L1: u64 = 4; + +impl L1LookupError for TestProviderError { + fn is_block_not_found(&self) -> bool { + matches!(self, Self::BlockNotFound) + } +} type SyncStatusReply = Result; @@ -47,7 +58,19 @@ struct Harness { impl Harness { fn new(replies: impl IntoIterator) -> Self { - let mut l1 = TestChainProvider::default(); + Self::with_failing_l1_lookups(replies, []) + } + + /// Like [`Self::new`], but looking up any of the L1 blocks numbered in `failing` fails for a + /// reason other than a missing block. + fn with_failing_l1_lookups( + replies: impl IntoIterator, + failing: impl IntoIterator, + ) -> Self { + let mut l1 = TestChainProvider { + failing_block_numbers: failing.into_iter().collect(), + ..Default::default() + }; for number in [SAFE_ORIGIN, FINALIZED_ORIGIN, CURRENT_L1] { l1.insert_block(number, canonical_l1_block(number)); } @@ -270,7 +293,9 @@ enum DelegateReply { CurrentL1NotCanonical, FetchFailed, FinalizedOriginNotCanonical, + FinalizedOriginNotFound, SafeOriginNotCanonical, + SafeOriginNotFound, Valid, } @@ -287,9 +312,15 @@ impl DelegateReply { Self::FinalizedOriginNotCanonical => { status.finalized_l2.l1_origin = non_canonical(status.finalized_l2.l1_origin) } + Self::FinalizedOriginNotFound => { + status.finalized_l2.l1_origin = canonical_l1_block(NOT_ON_OUR_L1).id() + } Self::SafeOriginNotCanonical => { status.safe_l2.l1_origin = non_canonical(status.safe_l2.l1_origin) } + Self::SafeOriginNotFound => { + status.safe_l2.l1_origin = canonical_l1_block(NOT_ON_OUR_L1).id() + } Self::Valid => {} } Ok(status) @@ -310,13 +341,16 @@ async fn polls_wait_for_engine_sync_completion() { } /// A valid status forwards the safe head and then the hash-pinned finalized block, and publishes -/// the delegate's `current_l1`. A failed fetch, or any L1 reference that is not canonical, skips -/// the poll without forwarding or publishing anything and without stopping the actor. +/// the delegate's `current_l1`. A failed fetch, any L1 reference that is not canonical, or a head's +/// L1 origin that our L1 provider does not have, skips the poll without forwarding or publishing +/// anything and without stopping the actor. #[rstest] #[case::valid(DelegateReply::Valid, true)] #[case::fetch_failed(DelegateReply::FetchFailed, false)] #[case::safe_origin_not_canonical(DelegateReply::SafeOriginNotCanonical, false)] +#[case::safe_origin_not_found(DelegateReply::SafeOriginNotFound, false)] #[case::finalized_origin_not_canonical(DelegateReply::FinalizedOriginNotCanonical, false)] +#[case::finalized_origin_not_found(DelegateReply::FinalizedOriginNotFound, false)] #[case::current_l1_not_canonical(DelegateReply::CurrentL1NotCanonical, false)] #[tokio::test(start_paused = true)] async fn poll_forwards_only_canonical_status(#[case] reply: DelegateReply, #[case] accepted: bool) { @@ -329,6 +363,72 @@ async fn poll_forwards_only_canonical_status(#[case] reply: DelegateReply, #[cas assert_eq!(*harness.current_l1.borrow(), accepted.then(|| delegate_status().current_l1)); } +/// What our L1 provider has at the height of the delegate's `current_l1`. +#[derive(Clone, Copy, Debug)] +enum OurCurrentL1 { + Canonical, + LookupFails, + NotFound, + OtherBlock, +} + +/// Only a `current_l1` that our L1 provider does not have yet leaves the status's heads to be +/// forwarded: their L1 origins are canonical. The published `current_l1` then stays where the last +/// accepted status put it. A `current_l1` on another L1 chain, or a failed lookup, rejects the +/// status whole. +#[rstest] +#[case::canonical(OurCurrentL1::Canonical, true, true)] +#[case::not_found(OurCurrentL1::NotFound, true, false)] +#[case::other_block(OurCurrentL1::OtherBlock, false, false)] +#[case::lookup_fails(OurCurrentL1::LookupFails, false, false)] +#[tokio::test(start_paused = true)] +async fn current_l1_check_only_soft_fails_when_not_found( + #[case] ours: OurCurrentL1, + #[case] heads_forwarded: bool, + #[case] current_l1_published: bool, +) { + // A first status whose `current_l1` is its safe head's L1 origin: one lookup verifies both. + let first_status = + SyncStatus { current_l1: canonical_l1_block(SAFE_ORIGIN), ..delegate_status() }; + let current_l1 = match ours { + OurCurrentL1::Canonical | OurCurrentL1::LookupFails => canonical_l1_block(CURRENT_L1), + OurCurrentL1::NotFound => canonical_l1_block(NOT_ON_OUR_L1), + OurCurrentL1::OtherBlock => BlockInfo { + hash: non_canonical(canonical_l1_block(CURRENT_L1).id()).hash, + ..canonical_l1_block(CURRENT_L1) + }, + }; + let status = SyncStatus { current_l1, ..delegate_status() }; + let failing = matches!(ours, OurCurrentL1::LookupFails).then_some(CURRENT_L1); + let mut harness = + Harness::with_failing_l1_lookups([Ok(first_status.clone()), Ok(status.clone())], failing); + harness.complete_engine_sync().await; + harness.actor.step().await.unwrap(); + assert_eq!(harness.forwarded(), forwarded_for(&first_status)); + assert_eq!(*harness.current_l1.borrow(), Some(first_status.current_l1)); + + harness.actor.step().await.unwrap(); + + assert_eq!(harness.forwarded(), forwarded_if(heads_forwarded, &status)); + let expected_current_l1 = + if current_l1_published { status.current_l1 } else { first_status.current_l1 }; + assert_eq!(*harness.current_l1.borrow(), Some(expected_current_l1)); +} + +/// A `current_l1` that our L1 provider does not have yet does not publish anything on the first +/// poll either, while the status's heads are forwarded. +#[tokio::test(start_paused = true)] +async fn first_poll_with_current_l1_not_found_publishes_no_current_l1() { + let status = SyncStatus { current_l1: canonical_l1_block(NOT_ON_OUR_L1), ..delegate_status() }; + let mut harness = Harness::new([Ok(status.clone())]); + harness.complete_engine_sync().await; + + harness.actor.step().await.unwrap(); + + assert_eq!(harness.forwarded(), forwarded_for(&status)); + assert_eq!(*harness.current_l1.borrow(), None); +} + /// An unchanged status is forwarded again on every poll, one poll interval apart, so a target /// that could not be applied on one poll is offered again on the next. #[tokio::test(start_paused = true)] diff --git a/rust/kona/crates/node/service/src/actors/derivation/mod.rs b/rust/kona/crates/node/service/src/actors/derivation/mod.rs index c3dc57502c0..0ef92f401e2 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/mod.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/mod.rs @@ -2,7 +2,7 @@ mod actor; pub use actor::{DerivationActor, DerivationError}; mod delegated; -pub use delegated::{DelegateDerivationActor, SyncStatusSource}; +pub use delegated::{DelegateDerivationActor, L1LookupError, SyncStatusSource}; mod engine_client; pub use engine_client::{DerivationEngineClient, QueuedDerivationEngineClient}; diff --git a/rust/kona/crates/node/service/src/actors/mod.rs b/rust/kona/crates/node/service/src/actors/mod.rs index cb221342aab..b2299e6e8ea 100644 --- a/rust/kona/crates/node/service/src/actors/mod.rs +++ b/rust/kona/crates/node/service/src/actors/mod.rs @@ -25,7 +25,7 @@ mod derivation; pub use derivation::{ DelegateDerivationActor, DerivationActor, DerivationActorRequest, DerivationClientError, DerivationClientResult, DerivationEngineClient, DerivationError, DerivationState, - DerivationStateMachine, DerivationStateTransitionError, DerivationStateUpdate, + DerivationStateMachine, DerivationStateTransitionError, DerivationStateUpdate, L1LookupError, QueuedDerivationEngineClient, SyncStatusSource, }; diff --git a/rust/kona/crates/node/service/src/lib.rs b/rust/kona/crates/node/service/src/lib.rs index 73a28c6b803..60d4f622ee9 100644 --- a/rust/kona/crates/node/service/src/lib.rs +++ b/rust/kona/crates/node/service/src/lib.rs @@ -19,7 +19,7 @@ pub use actors::{ DerivationActorRequest, DerivationClientError, DerivationClientResult, DerivationEngineClient, DerivationError, DerivationState, DerivationStateMachine, DerivationStateTransitionError, DerivationStateUpdate, EngineActor, EngineActorRequest, EngineClientError, EngineClientResult, - EngineConfig, EngineDerivationClient, EngineError, IpVotes, L1OriginSelector, + EngineConfig, EngineDerivationClient, EngineError, IpVotes, L1LookupError, L1OriginSelector, L1OriginSelectorError, L1OriginSelectorProvider, L1WatcherActor, L1WatcherActorError, L1WatcherChain, L1WatcherDerivationClient, NetworkActor, NetworkActorError, NetworkBuilder, NetworkBuilderError, NetworkConfig, NetworkDriver, NetworkDriverError, NetworkEngineClient, diff --git a/rust/kona/crates/node/service/src/metrics/mod.rs b/rust/kona/crates/node/service/src/metrics/mod.rs index a4fed01bcbf..d83fa9bac18 100644 --- a/rust/kona/crates/node/service/src/metrics/mod.rs +++ b/rust/kona/crates/node/service/src/metrics/mod.rs @@ -27,6 +27,11 @@ impl Metrics { pub const DERIVATION_PENDING_RESET_ATTEMPTS: &str = "kona_node_derivation_pending_reset_attempts"; + /// Delegate sync statuses whose `current_l1` was not found on our L1 chain, so the + /// heads were applied without moving the derivation origin. + pub const DERIVATION_DELEGATE_CURRENT_L1_NOT_FOUND: &str = + "kona_node_derivation_delegate_current_l1_not_found_total"; + /// Identifier for the counter that tracks sequencer state flags. pub const SEQUENCER_STATE: &str = "kona_node_sequencer_state"; @@ -92,6 +97,13 @@ impl Metrics { "Failed attempts of the derivation pipeline reset awaiting a retry, zero when none is pending" ); + // Derivation delegation: current L1 not found on our L1 chain + metrics::describe_counter!( + Self::DERIVATION_DELEGATE_CURRENT_L1_NOT_FOUND, + metrics::Unit::Count, + "Delegate sync statuses whose current L1 was not found on our L1 chain; heads applied without updating the derivation origin" + ); + // Sequencer state metrics::describe_counter!(Self::SEQUENCER_STATE, "Tracks sequencer state flags"); @@ -145,6 +157,9 @@ impl Metrics { // Derivation pending reset attempts metrics::gauge!(Self::DERIVATION_PENDING_RESET_ATTEMPTS).set(0.0); + // Derivation delegation: current L1 not found + metrics::counter!(Self::DERIVATION_DELEGATE_CURRENT_L1_NOT_FOUND).absolute(0); + // Sequencer: reset total transactions sequenced metrics::counter!(Self::SEQUENCER_TOTAL_TRANSACTIONS_SEQUENCED).absolute(0); metrics::counter!(Self::SEQUENCER_BUILD_REJECTIONS).absolute(0); diff --git a/rust/kona/crates/protocol/derive/src/test_utils/chain_providers.rs b/rust/kona/crates/protocol/derive/src/test_utils/chain_providers.rs index 74cb7fdce2a..99b6d696c4c 100644 --- a/rust/kona/crates/protocol/derive/src/test_utils/chain_providers.rs +++ b/rust/kona/crates/protocol/derive/src/test_utils/chain_providers.rs @@ -24,6 +24,8 @@ pub struct TestChainProvider { pub receipts: Vec<(B256, Vec)>, /// Maps block hashes to transactions using a tuple list. pub transactions: Vec<(B256, Vec)>, + /// Block numbers whose lookup by number fails with an error other than not-found. + pub failing_block_numbers: Vec, } impl TestChainProvider { @@ -85,6 +87,9 @@ pub enum TestProviderError { /// The header was not found. #[error("Header not found")] HeaderNotFound, + /// The lookup of the block with this number failed, as a lost connection would. + #[error("Lookup of block {0} failed")] + BlockLookupFailed(u64), /// The receipts were not found. #[error("Receipts not found")] ReceiptsNotFound, @@ -123,6 +128,9 @@ impl ChainProvider for TestChainProvider { } async fn block_info_by_number(&mut self, _number: u64) -> Result { + if self.failing_block_numbers.contains(&_number) { + return Err(TestProviderError::BlockLookupFailed(_number)); + } if let Some((_, block)) = self.blocks.iter().find(|(n, _)| *n == _number) { Ok(*block) } else {