diff --git a/rust/kona/crates/node/engine/src/lib.rs b/rust/kona/crates/node/engine/src/lib.rs index c1f955096bb..e8fd1f9305d 100644 --- a/rust/kona/crates/node/engine/src/lib.rs +++ b/rust/kona/crates/node/engine/src/lib.rs @@ -65,7 +65,7 @@ mod versions; pub use versions::{EngineForkchoiceVersion, EngineGetPayloadVersion, EngineNewPayloadVersion}; mod state; -pub use state::{EngineState, EngineSyncState, EngineSyncStateUpdate}; +pub use state::{EngineState, EngineSyncState, EngineSyncStateUpdate, FinalityViolation}; mod kinds; pub use kinds::EngineKind; diff --git a/rust/kona/crates/node/engine/src/state/core.rs b/rust/kona/crates/node/engine/src/state/core.rs index 7a34ce3f626..67204d7d81f 100644 --- a/rust/kona/crates/node/engine/src/state/core.rs +++ b/rust/kona/crates/node/engine/src/state/core.rs @@ -1,6 +1,7 @@ //! The internal state of the engine controller. use crate::Metrics; +use alloy_eips::BlockNumHash; use alloy_rpc_types_engine::ForkchoiceState; use kona_protocol::L2BlockInfo; use serde::{Deserialize, Serialize}; @@ -54,6 +55,12 @@ impl EngineSyncState { self.finalized_head } + /// Returns the id of the finalized head once finality is established. The default finalized + /// head means it is not yet, so no block pins finality. + pub fn established_finalized_head(&self) -> Option { + (self.finalized_head != L2BlockInfo::default()).then(|| self.finalized_head.block_info.id()) + } + /// Creates a `ForkchoiceState` /// /// - `head_block` = `unsafe_head` @@ -127,8 +134,10 @@ pub struct EngineSyncStateUpdate { /// The chain state viewed by the engine controller. #[derive(Default, Debug, Copy, Clone, PartialEq, Eq)] pub struct EngineState { - /// The sync state of the engine. - pub sync_state: EngineSyncState, + /// Whether the execution layer answered `VALID` to the last forkchoice update the engine sent, + /// so its canonical chain is the engine's. Unlike `el_sync_finished`, a later `SYNCING` answer + /// clears it. + pub el_applied_last_forkchoice: bool, /// Whether or not the EL has finished syncing. pub el_sync_finished: bool, @@ -139,6 +148,9 @@ pub struct EngineState { /// because engine may forgot backupUnsafeHead or backupUnsafeHead is not part /// of the chain. pub need_fcu_call_backup_unsafe_reorg: bool, + + /// The sync state of the engine. + pub sync_state: EngineSyncState, } impl EngineState { diff --git a/rust/kona/crates/node/engine/src/state/finality.rs b/rust/kona/crates/node/engine/src/state/finality.rs new file mode 100644 index 00000000000..94363c549bf --- /dev/null +++ b/rust/kona/crates/node/engine/src/state/finality.rs @@ -0,0 +1,126 @@ +//! Checks that keep a label from moving below finality. + +use crate::EngineSyncState; +use alloy_eips::BlockNumHash; +use thiserror::Error; + +/// A block that a label cannot move to, because it would move the label below finality. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)] +pub enum FinalityViolation { + /// The block is below the finalized head. + #[error("Block {target} is behind the finalized head {finalized:?}")] + BehindFinalizedHead { + /// The finalized head. + finalized: BlockNumHash, + /// The number of the block. + target: u64, + }, + /// The block is a different block at the finalized head's height. + #[error("Block {target:?} conflicts with the finalized head {finalized:?}")] + ConflictsWithFinalizedHead { + /// The finalized head. + finalized: BlockNumHash, + /// The block. + target: BlockNumHash, + }, +} + +impl EngineSyncState { + /// Rejects a block number below the established finalized head. + pub fn ensure_not_behind_finality(&self, target: u64) -> Result<(), FinalityViolation> { + match self.established_finalized_head() { + Some(finalized) if target < finalized.number => { + Err(FinalityViolation::BehindFinalizedHead { finalized, target }) + } + _ => Ok(()), + } + } + + /// Rejects a different block at the established finalized head's height. Before finality is + /// established no block pins it, so any block, including the configured genesis, passes. + pub fn ensure_no_conflict_with_finality( + &self, + target: BlockNumHash, + ) -> Result<(), FinalityViolation> { + match self.established_finalized_head() { + Some(finalized) + if target.number == finalized.number && target.hash != finalized.hash => + { + Err(FinalityViolation::ConflictsWithFinalizedHead { finalized, target }) + } + _ => Ok(()), + } + } + + /// Rejects a block below the established finalized head or conflicting with it. + pub fn ensure_respects_finality(&self, target: BlockNumHash) -> Result<(), FinalityViolation> { + self.ensure_not_behind_finality(target.number)?; + self.ensure_no_conflict_with_finality(target) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::EngineSyncStateUpdate; + use alloy_primitives::B256; + use kona_protocol::{BlockInfo, L2BlockInfo}; + + const FINALIZED_NUMBER: u64 = 5; + + fn block(number: u64, hash: u8) -> L2BlockInfo { + L2BlockInfo { + block_info: BlockInfo { hash: B256::repeat_byte(hash), number, ..Default::default() }, + ..Default::default() + } + } + + fn sync_state_finalized_at(finalized_head: L2BlockInfo) -> EngineSyncState { + EngineSyncState::default().apply_update(EngineSyncStateUpdate { + finalized_head: Some(finalized_head), + ..Default::default() + }) + } + + /// The finalized head at 5 once finality is established, and the default head before. + fn finalized_head(is_finality_established: bool) -> L2BlockInfo { + if is_finality_established { block(FINALIZED_NUMBER, 1) } else { L2BlockInfo::default() } + } + + /// The violation expected for `target` checked against the finalized head at 5. + fn behind(target: L2BlockInfo) -> Option { + Some(FinalityViolation::BehindFinalizedHead { + finalized: block(FINALIZED_NUMBER, 1).block_info.id(), + target: target.block_info.number, + }) + } + + /// The violation expected for `target` conflicting with the finalized head at 5. + fn conflicting(target: L2BlockInfo) -> Option { + Some(FinalityViolation::ConflictsWithFinalizedHead { + finalized: block(FINALIZED_NUMBER, 1).block_info.id(), + target: target.block_info.id(), + }) + } + + /// A block below the established finalized head, or a different block at its height, breaks + /// finality. Before finality is established no block pins it, so a different block at the + /// default head's height, genesis, passes. + #[rstest::rstest] + #[case::above(true, block(FINALIZED_NUMBER + 1, 2), None)] + #[case::at(true, block(FINALIZED_NUMBER, 1), None)] + #[case::below(true, block(FINALIZED_NUMBER - 1, 2), behind(block(FINALIZED_NUMBER - 1, 2)))] + #[case::conflicting(true, block(FINALIZED_NUMBER, 2), conflicting(block(FINALIZED_NUMBER, 2)))] + #[case::not_established(false, block(0, 2), None)] + fn block_is_checked_against_established_finality( + #[case] is_finality_established: bool, + #[case] target: L2BlockInfo, + #[case] expected_violation: Option, + ) { + let sync_state = sync_state_finalized_at(finalized_head(is_finality_established)); + + let result = sync_state.ensure_respects_finality(target.block_info.id()); + + assert_eq!(result.err(), expected_violation); + } +} diff --git a/rust/kona/crates/node/engine/src/state/mod.rs b/rust/kona/crates/node/engine/src/state/mod.rs index b61100917e6..a3b0cfe6f51 100644 --- a/rust/kona/crates/node/engine/src/state/mod.rs +++ b/rust/kona/crates/node/engine/src/state/mod.rs @@ -2,3 +2,6 @@ mod core; pub use core::{EngineState, EngineSyncState, EngineSyncStateUpdate}; + +mod finality; +pub use finality::FinalityViolation; diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/error.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/error.rs index c5bd5dbcb0f..2ce1b187a76 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/error.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/error.rs @@ -1,7 +1,7 @@ //! Contains error types for the [`crate::ConsolidateTask`]. use crate::{ - BuildTaskError, EngineTaskError, SealTaskError, SynchronizeTaskError, + BuildTaskError, EngineTaskError, FinalityViolation, SealTaskError, SynchronizeTaskError, task_queue::tasks::{BuildAndImportError, task::EngineTaskErrorSeverity}, }; use thiserror::Error; @@ -9,6 +9,13 @@ use thiserror::Error; /// An error that occurs when running the [`crate::ConsolidateTask`]. #[derive(Debug, Error)] pub enum ConsolidateTaskError { + /// A delegated safe head would move the safe labels below finality. + #[error("Delegated safe head breaks finality: {0}")] + BreaksFinality(#[from] FinalityViolation), + /// Failed to fetch the execution layer's block at the height of a delegated safe head below the + /// safe head. + #[error("Failed to fetch the block at delegated safe head height {0}")] + FailedToFetchDelegatedSafeBlock(u64), /// The unsafe L2 block is missing. #[error("Unsafe L2 block is missing {0}")] MissingUnsafeL2Block(u64), @@ -38,8 +45,11 @@ impl From for ConsolidateTaskError { impl EngineTaskError for ConsolidateTaskError { fn severity(&self) -> EngineTaskErrorSeverity { match self { + Self::BreaksFinality(_) => EngineTaskErrorSeverity::Skip, Self::MissingUnsafeL2Block(_) => EngineTaskErrorSeverity::Reset, - Self::FailedToFetchUnsafeL2Block => EngineTaskErrorSeverity::Temporary, + Self::FailedToFetchDelegatedSafeBlock(_) | Self::FailedToFetchUnsafeL2Block => { + EngineTaskErrorSeverity::Temporary + } Self::BuildTaskFailed(inner) => inner.severity(), Self::SealTaskFailed(inner) => inner.severity(), Self::ForkchoiceUpdateFailed(inner) => inner.severity(), diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs index d3183cdafa5..00f31457729 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs @@ -100,6 +100,159 @@ impl ConsolidateTask { Ok(()) } + /// This provides symmetric fallback behavior with `build_and_import`. + async fn reconcile_to_safe_head( + &self, + state: &mut EngineState, + safe_l2: &L2BlockInfo, + ) -> Result<(), ConsolidateTaskError> { + warn!( + target: "engine", + safe_l2 = %safe_l2, + "Apply safe head" + ); + + // We intentionally set the unsafe head to safe_l2 to ensure the engine observes a + // self-consistent head state. This is required to correctly handle reorgs (where unsafe + // may be ahead on a non-canonical fork) and to trigger EL sync when the local unsafe head + // lags behind the safe head. + self.apply_delegated_safe_head( + state, + safe_l2, + EngineSyncStateUpdate { + local_safe_head: Some(*safe_l2), + safe_head: Some(*safe_l2), + unsafe_head: Some(*safe_l2), + ..Default::default() + }, + "Updated safe head via follow safe", + ) + .await + } + + /// Moves the safe labels to a delegated safe head that the execution layer does not have yet, + /// leaving the unsafe head in place. + /// + /// A block missing below the unsafe head means the execution layer is still syncing toward the + /// unsafe head. Resetting would restart that sync from the execution layer's old labels, and + /// moving the unsafe head back would retarget it. Like op-node's follow-source mode, only the + /// safe labels follow the delegate, and the forkchoice update carries them to the execution + /// layer. + async fn follow_safe_head_during_el_sync( + &self, + state: &mut EngineState, + safe_l2: &L2BlockInfo, + ) -> Result<(), ConsolidateTaskError> { + warn!( + target: "engine", + safe_l2 = %safe_l2, + unsafe_number = state.sync_state.unsafe_head().block_info.number, + "Execution layer does not have the delegated safe head yet; keeping its sync target" + ); + + self.apply_delegated_safe_head( + state, + safe_l2, + EngineSyncStateUpdate { + local_safe_head: Some(*safe_l2), + safe_head: Some(*safe_l2), + ..Default::default() + }, + "Updated safe head while the execution layer syncs", + ) + .await + } + + /// Applies `update`, whose labels follow the delegated `safe_l2`, and always sends its + /// forkchoice update: the execution layer may have answered `SYNCING` to the same labels + /// before, or lost them in a restart. + async fn apply_delegated_safe_head( + &self, + state: &mut EngineState, + safe_l2: &L2BlockInfo, + update: EngineSyncStateUpdate, + outcome: &'static str, + ) -> Result<(), ConsolidateTaskError> { + synchronize_unconditionally(self.client.as_ref(), state, update) + .await + .map_err(|e| { + warn!(target: "engine", ?e, safe_l2 = %safe_l2, "Applying the delegated safe head failed"); + e + })?; + + info!( + target: "engine", + hash = %safe_l2.block_info.hash, + number = safe_l2.block_info.number, + "{outcome}" + ); + + Ok(()) + } + + /// Follows a delegated safe head below the safe head. Only a different block at its height in + /// a synced execution layer proves the delegate diverged and reorgs the engine onto it; the + /// same block (a lagging delegate) or a missing one leaves the labels. After a delegate L1 + /// reorg whose fork point is above the head it reports, the labels stay on the abandoned + /// blocks until it reports a head above that point. + async fn follow_delegated_safe_head_behind_safe_head( + &self, + state: &mut EngineState, + safe_l2: &L2BlockInfo, + ) -> Result<(), ConsolidateTaskError> { + // Until the execution layer applies the engine's forkchoice, its canonical block at that + // height may predate the engine's own labels, so it proves nothing about the delegate. + if !state.el_applied_last_forkchoice { + debug!( + target: "engine", + safe_l2 = %safe_l2, + "Ignoring a delegated safe head behind the safe head while the execution layer syncs" + ); + return Ok(()); + } + debug!( + target: "engine", + safe_l2 = %safe_l2, + "Looking up the execution layer's block at a delegated safe head behind the safe head" + ); + let el_block = + self.client.get_l2_block(safe_l2.block_info.number.into()).await.map_err(|err| { + warn!( + target: "engine", + ?err, + safe_l2 = %safe_l2, + "Failed to look up the execution layer's block at a delegated safe head" + ); + ConsolidateTaskError::FailedToFetchDelegatedSafeBlock(safe_l2.block_info.number) + })?; + debug!( + target: "engine", + safe_l2 = %safe_l2, + el_hash = ?el_block.as_ref().map(|block| block.header.hash), + "Looked up the execution layer's block at a delegated safe head" + ); + match el_block { + Some(el_block) if el_block.header.hash != safe_l2.block_info.hash => { + debug!( + target: "engine", + safe_l2 = %safe_l2, + el_hash = %el_block.header.hash, + "Delegated safe head behind the safe head diverges from the execution layer" + ); + self.reconcile_to_safe_head(state, safe_l2).await + } + _ => { + debug!( + target: "engine", + safe_l2 = %safe_l2, + safe_number = state.sync_state.safe_head().block_info.number, + "Ignoring a delegated safe head behind the safe head" + ); + Ok(()) + } + } + } + /// Handles the fallback case when the block doesn't match the input or does not exist. async fn reconcile_unsafe_to_safe( &self, @@ -110,37 +263,7 @@ impl ConsolidateTask { self.build_and_import_attributes(state, attributes).await } ConsolidateInput::BlockInfo(safe_l2) => { - warn!(target: "engine", safe_l2 = %safe_l2, "Apply safe head"); - - // We intentionally set the unsafe head to safe_l2 to ensure the engine observes a - // self-consistent head state. This is required to correctly handle reorgs (where - // unsafe may be ahead on a non-canonical fork) and to trigger EL sync when the - // local unsafe head lags behind the safe head. The forkchoice update is always - // sent: the execution layer may have answered `SYNCING` to the same labels before, - // or lost them in a restart. - synchronize_unconditionally( - self.client.as_ref(), - state, - EngineSyncStateUpdate { - local_safe_head: Some(*safe_l2), - safe_head: Some(*safe_l2), - unsafe_head: Some(*safe_l2), - ..Default::default() - }, - ) - .await - .map_err(|e| { - warn!(target: "engine", ?e, safe_l2 = %safe_l2, "Applying the delegated safe head failed"); - e - })?; - - info!( - target: "engine", - hash = %safe_l2.block_info.hash, - number = safe_l2.block_info.number, - "Updated safe head via follow safe" - ); - Ok(()) + self.reconcile_to_safe_head(state, safe_l2).await } } } @@ -160,39 +283,7 @@ impl ConsolidateTask { Ok(Some(block)) => block, Ok(None) => { if let ConsolidateInput::BlockInfo(safe_l2) = &self.input { - // A block missing below the unsafe head means the execution layer is still - // syncing toward the unsafe head. Resetting would restart that sync from the - // execution layer's old labels, and moving the unsafe head back would retarget - // it. Like op-node's follow-source mode, only the safe labels follow the - // delegate, and the forkchoice update, sent even when unchanged, carries them - // to the execution layer. - warn!( - target: "engine", - safe_l2 = %safe_l2, - unsafe_number = state.sync_state.unsafe_head().block_info.number, - "Execution layer does not have the delegated safe head yet; keeping its sync target" - ); - synchronize_unconditionally( - self.client.as_ref(), - state, - EngineSyncStateUpdate { - local_safe_head: Some(*safe_l2), - safe_head: Some(*safe_l2), - ..Default::default() - }, - ) - .await - .map_err(|e| { - warn!(target: "engine", ?e, safe_l2 = %safe_l2, "Applying the delegated safe head failed"); - e - })?; - info!( - target: "engine", - hash = %safe_l2.block_info.hash, - number = safe_l2.block_info.number, - "Updated safe head while the execution layer syncs" - ); - return Ok(()); + return self.follow_safe_head_during_el_sync(state, safe_l2).await; } warn!(target: "engine", "Received `None` block for {}", block_num); return Err(ConsolidateTaskError::MissingUnsafeL2Block(block_num)); @@ -278,6 +369,14 @@ impl ConsolidateTask { } } + // A different block from an execution layer that has not applied the engine's forkchoice + // may be from before its sync, so it does not prove the delegate diverged. + if let ConsolidateInput::BlockInfo(safe_l2) = &self.input && + !state.el_applied_last_forkchoice + { + return self.follow_safe_head_during_el_sync(state, safe_l2).await; + } + debug!( target: "engine", input = ?self.input, @@ -307,6 +406,19 @@ impl ExecutableEngineTask for ConsolidateTask { // injected safe head is ahead of the EngineActor's unsafe head, we reconcile the unsafe chain // up to the safe head instead of consolidating. async fn execute(&self, state: &mut EngineState) -> Result<(), ConsolidateTaskError> { + if let ConsolidateInput::BlockInfo(safe_l2) = &self.input { + state.sync_state.ensure_respects_finality(safe_l2.block_info.id())?; + // Labels with the unsafe head below the safe head take the paths below, which move the + // labels back to a consistent state; ignoring a lagging delegate would keep them broken + // until a finalization above the unsafe head fails. + let are_labels_consistent = state.sync_state.unsafe_head().block_info.number >= + state.sync_state.safe_head().block_info.number; + if are_labels_consistent && + safe_l2.block_info.number < state.sync_state.safe_head().block_info.number + { + return self.follow_delegated_safe_head_behind_safe_head(state, safe_l2).await; + } + } let safe_head_number = match &self.input { ConsolidateInput::Attributes { .. } => state.sync_state.safe_head().block_info.number, ConsolidateInput::BlockInfo(safe_block_info) => safe_block_info.block_info.number, @@ -323,7 +435,8 @@ impl ExecutableEngineTask for ConsolidateTask { mod tests { use super::*; use crate::{ - EngineSyncState, EngineTaskError, ImportedBlockSink, NoopBlockSink, SealTaskError, + EngineSyncState, EngineTaskError, FinalityViolation, ImportedBlockSink, NoopBlockSink, + SealTaskError, test_utils::{ TestAttributesBuilder, TestEngineStateBuilder, l1_info_deposit, test_block_info, test_engine_client, @@ -547,10 +660,9 @@ mod tests { el_has_block.then_some(block) } - /// The safe head before consolidation: either already the delegated target, as a `SYNCING` - /// answer or an earlier poll leaves it, or an older block. - fn initial_safe_head(starts_at_target: bool, target: L2BlockInfo) -> L2BlockInfo { - if starts_at_target { target } else { test_block_info(5) } + /// Either `target` itself or an older block, for a label that may already sit at the target. + fn target_or_older_block(is_at_target: bool, target: L2BlockInfo) -> L2BlockInfo { + if is_at_target { target } else { test_block_info(5) } } /// Below the unsafe head, a delegated safe head always reaches the execution layer: when the @@ -570,8 +682,10 @@ mod tests { ) { let (block, cfg) = genesis_pinned_block(10); let delegated_safe_head = block_info_of(&cfg, &block); - let mut state = - engine_state_with_safe_head(initial_safe_head(starts_at_target, delegated_safe_head)); + let mut state = engine_state_with_safe_head(target_or_older_block( + starts_at_target, + delegated_safe_head, + )); let expected_sync_state = with_safe_head(&state, delegated_safe_head); let (client, l1, l2) = test_engine_client(cfg.clone()); l2.expect_params( @@ -692,4 +806,326 @@ mod tests { l1.assert_finished(); l2.assert_finished(); } + + /// Where a delegated safe head sits against the finalized head. + #[derive(Clone, Copy, Debug)] + enum TargetAgainstFinalized { + Below, + ConflictingAt, + } + + impl TargetAgainstFinalized { + fn delegated_safe_head(self, finalized: L2BlockInfo) -> L2BlockInfo { + let number = finalized.block_info.number; + match self { + Self::Below => test_block_info(number - 1), + Self::ConflictingAt => L2BlockInfo { + block_info: BlockInfo { + hash: !finalized.block_info.hash, + ..finalized.block_info + }, + ..finalized + }, + } + } + } + + /// The outcome of checking a delegated safe head against finality. + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + enum FinalityCheck { + Accepted, + BehindFinalizedHead, + ConflictsWithFinalizedHead, + } + + impl FinalityCheck { + fn of(result: &Result<(), ConsolidateTaskError>) -> Self { + match result { + Ok(()) => Self::Accepted, + Err(ConsolidateTaskError::BreaksFinality( + FinalityViolation::BehindFinalizedHead { .. }, + )) => Self::BehindFinalizedHead, + Err(ConsolidateTaskError::BreaksFinality( + FinalityViolation::ConflictsWithFinalizedHead { .. }, + )) => Self::ConflictsWithFinalizedHead, + Err(err) => panic!("unexpected error {err:?}"), + } + } + } + + /// A delegated safe head that breaks finality, as a stale delegate snapshot reports, is skipped + /// before the execution layer is asked anything, and leaves every label where it was. + #[rstest::rstest] + #[case::below(TargetAgainstFinalized::Below, FinalityCheck::BehindFinalizedHead)] + #[case::conflicting( + TargetAgainstFinalized::ConflictingAt, + FinalityCheck::ConflictsWithFinalizedHead + )] + #[tokio::test] + async fn delegated_safe_head_breaking_finality_is_skipped( + #[case] target: TargetAgainstFinalized, + #[case] expected: FinalityCheck, + ) { + let cfg = Arc::new(RollupConfig::default()); + let mut state = engine_state_with_safe_head(test_block_info(12)); + let state_before = state; + let delegated_safe_head = target.delegated_safe_head(state.sync_state.finalized_head()); + let (client, l1, l2) = test_engine_client(cfg.clone()); + let task = ConsolidateTask::new( + Arc::new(client), + cfg, + ConsolidateInput::BlockInfo(delegated_safe_head), + Arc::new(NoopBlockSink), + ); + + let result = task.execute(&mut state).await; + + assert_eq!(FinalityCheck::of(&result), expected); + assert_eq!(result.unwrap_err().severity(), crate::EngineTaskErrorSeverity::Skip); + assert_eq!(state, state_before); + l1.assert_finished(); + l2.assert_finished(); + } + + /// A delegated safe head from a lagging delegate, between the finalized head, inclusive, and + /// the safe head, does not move the safe labels back, whether or not the execution layer has + /// its block, and sends no forkchoice update. + #[rstest::rstest] + #[case::el_has_block(true, false)] + #[case::el_missing_block(false, false)] + #[case::at_finalized_head(true, true)] + #[tokio::test] + async fn lagging_delegated_safe_head_leaves_labels_unchanged( + #[case] el_has_block: bool, + #[case] finalized_at_target: bool, + ) { + let (block, cfg) = genesis_pinned_block(10); + let delegated_safe_head = block_info_of(&cfg, &block); + let mut state = TestEngineStateBuilder::new() + .with_unsafe_head(test_block_info(20)) + .with_safe_head(test_block_info(12)) + .with_finalized_head(target_or_older_block(finalized_at_target, delegated_safe_head)) + .build(); + let state_before = state; + let (client, l1, l2) = test_engine_client(cfg.clone()); + l2.expect_block_header_by_number(10, el_block(el_has_block, block)); + let sink = Arc::new(RecordingSink::default()); + let task = ConsolidateTask::new( + Arc::new(client), + cfg, + ConsolidateInput::BlockInfo(delegated_safe_head), + sink.clone(), + ); + + task.execute(&mut state).await.unwrap(); + + assert_eq!(state, state_before); + assert!(sink.0.lock().unwrap().is_empty()); + l1.assert_finished(); + l2.assert_finished(); + } + + /// A delegated safe head behind the safe head whose block differs from the execution layer's + /// means the delegate's chain diverged above it, so the engine still reorgs onto it, without + /// moving finalized. + #[tokio::test] + async fn diverged_delegated_safe_head_behind_safe_head_reorgs_the_engine() { + let (block, cfg) = genesis_pinned_block(10); + let delegated_safe_head = block_info_of(&cfg, &block); + let mut state = engine_state_with_safe_head(test_block_info(12)); + let finalized_before = state.sync_state.finalized_head(); + let expected_sync_state = state.sync_state.apply_update(EngineSyncStateUpdate { + local_safe_head: Some(delegated_safe_head), + safe_head: Some(delegated_safe_head), + unsafe_head: Some(delegated_safe_head), + ..Default::default() + }); + let (client, l1, l2) = test_engine_client(cfg.clone()); + l2.expect_block_header_by_number(10, Some(other_chain_block(10))); + l2.expect_forkchoice_updated( + expected_sync_state.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + let task = ConsolidateTask::new( + Arc::new(client), + cfg, + ConsolidateInput::BlockInfo(delegated_safe_head), + Arc::new(NoopBlockSink), + ); + + task.execute(&mut state).await.unwrap(); + + assert_eq!(state.sync_state, expected_sync_state); + assert_eq!(state.sync_state.finalized_head(), finalized_before); + l1.assert_finished(); + l2.assert_finished(); + } + + /// A delegated safe head behind the safe head is not judged against an execution layer that has + /// not applied the engine's last forkchoice, whether it is in its first sync or syncing again + /// after answering `SYNCING`: its canonical block there may predate the engine's labels, so no + /// lookup is made and the labels stay. + #[rstest::rstest] + #[case::initial_sync(false, false)] + #[case::syncing_again(true, false)] + #[tokio::test] + async fn delegated_safe_head_behind_safe_head_is_ignored_while_the_el_syncs( + #[case] el_sync_finished: bool, + #[case] el_applied_last_forkchoice: bool, + ) { + let (block, cfg) = genesis_pinned_block(10); + let delegated_safe_head = block_info_of(&cfg, &block); + let mut state = TestEngineStateBuilder::new() + .with_unsafe_head(test_block_info(20)) + .with_safe_head(test_block_info(12)) + .with_finalized_head(test_block_info(5)) + .with_el_sync_finished(el_sync_finished) + .with_el_applied_last_forkchoice(el_applied_last_forkchoice) + .build(); + let state_before = state; + let (client, l1, l2) = test_engine_client(cfg.clone()); + let task = ConsolidateTask::new( + Arc::new(client), + cfg, + ConsolidateInput::BlockInfo(delegated_safe_head), + Arc::new(NoopBlockSink), + ); + + task.execute(&mut state).await.unwrap(); + + assert_eq!(state, state_before); + l1.assert_finished(); + l2.assert_finished(); + } + + /// An empty block at `number` whose hash differs from [`genesis_pinned_block`]'s at the same + /// height, standing for a block on another chain. + fn other_chain_block(number: u64) -> RpcBlock { + RpcBlock::new( + RpcHeader::new(alloy_consensus::Header { number, timestamp: 1, ..Default::default() }), + BlockTransactions::Full(vec![]), + ) + } + + /// A reorg onto a diverged delegated safe head that the execution layer refuses as an invalid + /// forkchoice state resets the engine, which rebuilds the labels from the execution layer. + #[tokio::test] + async fn rejected_reorg_onto_a_delegated_safe_head_resets_the_engine() { + let (block, cfg) = genesis_pinned_block(10); + let delegated_safe_head = block_info_of(&cfg, &block); + let mut state = engine_state_with_safe_head(test_block_info(12)); + let state_before = state; + let (client, l1, l2) = test_engine_client(cfg.clone()); + l2.expect_block_header_by_number(10, Some(other_chain_block(10))); + l2.expect_error_code( + "engine_forkchoiceUpdatedV3", + i64::from(alloy_rpc_types_engine::INVALID_FORK_CHOICE_STATE_ERROR), + ); + let task = ConsolidateTask::new( + Arc::new(client), + cfg, + ConsolidateInput::BlockInfo(delegated_safe_head), + Arc::new(NoopBlockSink), + ); + + let err = task.execute(&mut state).await.unwrap_err(); + + assert_eq!(err.severity(), crate::EngineTaskErrorSeverity::Reset, "got {err:?}"); + assert_eq!(state, state_before); + l1.assert_finished(); + l2.assert_finished(); + } + + /// Labels with the unsafe head rewound below the safe head are moved back into order by any + /// delegated safe head above finality instead of being kept: at or above the unsafe head the + /// engine reorgs onto it, and below it the matching block consolidates. Keeping them would let + /// a later finalization above the unsafe head fail fatally. + #[rstest::rstest] + #[case::at_or_above_unsafe_head(12, false)] + #[case::below_unsafe_head(8, true)] + #[tokio::test] + async fn delegated_safe_head_repairs_an_unsafe_head_below_the_safe_head( + #[case] delegated_number: u64, + #[case] is_below_unsafe_head: bool, + ) { + let (block, cfg) = genesis_pinned_block(delegated_number); + let delegated_safe_head = block_info_of(&cfg, &block); + let mut state = TestEngineStateBuilder::new() + .with_unsafe_head(test_block_info(10)) + .with_safe_head(test_block_info(15)) + .with_finalized_head(test_block_info(5)) + .build(); + let expected_sync_state = state.sync_state.apply_update(EngineSyncStateUpdate { + local_safe_head: Some(delegated_safe_head), + safe_head: Some(delegated_safe_head), + unsafe_head: (!is_below_unsafe_head).then_some(delegated_safe_head), + ..Default::default() + }); + let (client, l1, l2) = test_engine_client(cfg.clone()); + if is_below_unsafe_head { + l2.expect_params( + "eth_getBlockByNumber", + serde_json::json!([format!("{delegated_number:#x}"), true]), + block, + ); + } + l2.expect_forkchoice_updated( + expected_sync_state.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + let task = ConsolidateTask::new( + Arc::new(client), + cfg, + ConsolidateInput::BlockInfo(delegated_safe_head), + Arc::new(NoopBlockSink), + ); + + task.execute(&mut state).await.unwrap(); + + assert_eq!(state.sync_state, expected_sync_state); + assert!( + state.sync_state.unsafe_head().block_info.number >= + state.sync_state.safe_head().block_info.number + ); + l1.assert_finished(); + l2.assert_finished(); + } + + /// A delegated safe head whose block differs from the execution layer's is not treated as a + /// divergence while the execution layer has not applied the engine's last forkchoice: only the + /// safe labels move to it, and the unsafe head stays the sync target. + #[tokio::test] + async fn mismatched_delegated_safe_head_does_not_reorg_while_the_el_syncs() { + let (block, cfg) = genesis_pinned_block(15); + let delegated_safe_head = block_info_of(&cfg, &block); + let mut state = TestEngineStateBuilder::new() + .with_unsafe_head(test_block_info(20)) + .with_safe_head(test_block_info(12)) + .with_finalized_head(test_block_info(5)) + .with_el_applied_last_forkchoice(false) + .build(); + let expected_sync_state = with_safe_head(&state, delegated_safe_head); + let (client, l1, l2) = test_engine_client(cfg.clone()); + l2.expect_params( + "eth_getBlockByNumber", + serde_json::json!(["0xf", true]), + other_chain_block(15), + ); + l2.expect_forkchoice_updated( + expected_sync_state.create_forkchoice_state(), + PayloadStatusEnum::Syncing, + ); + let task = ConsolidateTask::new( + Arc::new(client), + cfg, + ConsolidateInput::BlockInfo(delegated_safe_head), + Arc::new(NoopBlockSink), + ); + + task.execute(&mut state).await.unwrap(); + + assert_eq!(state.sync_state, expected_sync_state); + l1.assert_finished(); + l2.assert_finished(); + } } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/error.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/error.rs index 92eb39d2e96..3e07f755697 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/error.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/error.rs @@ -1,7 +1,8 @@ //! Contains error types for the [`crate::FinalizeTask`]. use crate::{ - EngineTaskError, SynchronizeTaskError, task_queue::tasks::task::EngineTaskErrorSeverity, + EngineTaskError, FinalityViolation, SynchronizeTaskError, + task_queue::tasks::task::EngineTaskErrorSeverity, }; use alloy_eips::BlockNumHash; use alloy_transport::{RpcError, TransportErrorKind}; @@ -11,14 +12,9 @@ use thiserror::Error; /// An error that occurs when running the [`crate::FinalizeTask`]. #[derive(Debug, Error)] pub enum FinalizeTaskError { - /// The target is below the finalized head, and finality never moves backward. - #[error("Finalization target {target} is behind the finalized head {finalized}")] - BehindFinalizedHead { - /// The number of the finalized head. - finalized: u64, - /// The number of the finalization target. - target: u64, - }, + /// The target would move the finalized head backward, or onto another chain. + #[error("Finalization target breaks finality: {0}")] + BreaksFinality(#[from] FinalityViolation), /// The block to finalize was not found. #[error("The block to finalize was not found: Number {0}")] BlockNotFound(u64), @@ -40,14 +36,6 @@ pub enum FinalizeTaskError { /// The number of the block the engine returned. returned: u64, }, - /// The target is a different block at the finalized head's height. - #[error("Finalization target {target:?} conflicts with the finalized head {finalized:?}")] - ConflictsWithFinalizedHead { - /// The finalized head. - finalized: BlockNumHash, - /// The finalization target. - target: BlockNumHash, - }, /// The forkchoice update call to finalize the block failed. #[error(transparent)] ForkchoiceUpdateFailed(#[from] SynchronizeTaskError), @@ -78,9 +66,8 @@ pub enum FinalizeTaskError { impl EngineTaskError for FinalizeTaskError { fn severity(&self) -> EngineTaskErrorSeverity { match self { - Self::BehindFinalizedHead { .. } | Self::BlockNotSafe { .. } | - Self::ConflictsWithFinalizedHead { .. } | + Self::BreaksFinality(_) | Self::NotCanonical { .. } | Self::NotYetAvailable { .. } => EngineTaskErrorSeverity::Skip, Self::BlockNotFound(_) | Self::BlockNumberMismatch { .. } | Self::FromBlock(_) => { diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs index 30a08a102bd..d89e9c0c0b9 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs @@ -1,8 +1,8 @@ //! A task for finalizing an L2 block. use crate::{ - EngineClient, EngineState, ExecutableEngineTask, FinalizeBlockId, FinalizeTaskError, - state::EngineSyncStateUpdate, task_queue::synchronize, + EngineClient, EngineState, EngineSyncState, ExecutableEngineTask, FinalizeBlockId, + FinalizeTaskError, state::EngineSyncStateUpdate, task_queue::synchronize, }; use alloy_eips::BlockNumHash; use async_trait::async_trait; @@ -42,19 +42,14 @@ impl FinalizeTask { /// Whether `target` is the finalized head itself; a different block at its height is an error. fn is_already_finalized( - finalized_head: &L2BlockInfo, + sync_state: &EngineSyncState, target: BlockNumHash, ) -> Result { - let finalized = finalized_head.block_info.id(); - if target == finalized { + if target == sync_state.finalized_head().block_info.id() { debug!(target: "engine", ?target, "Target is already finalized"); return Ok(true); } - // The default head means finality has not been established yet, so any target, including - // the configured genesis, may initialize it. - if *finalized_head != L2BlockInfo::default() && target.number == finalized.number { - return Err(FinalizeTaskError::ConflictsWithFinalizedHead { finalized, target }); - } + sync_state.ensure_no_conflict_with_finality(target)?; Ok(false) } @@ -101,19 +96,13 @@ impl ExecutableEngineTask for FinalizeTask { type Error = FinalizeTaskError; async fn execute(&self, state: &mut EngineState) -> Result<(), FinalizeTaskError> { - let finalized_head = state.sync_state.finalized_head(); let target_number = self.block_id.number(); // A lagging source can report an older finalized block, or one from another chain. A // pinned hash is enough to detect either without asking the execution layer. - if target_number < finalized_head.block_info.number { - return Err(FinalizeTaskError::BehindFinalizedHead { - finalized: finalized_head.block_info.number, - target: target_number, - }); - } + state.sync_state.ensure_not_behind_finality(target_number)?; if let FinalizeBlockId::ByHash(target) = self.block_id && - Self::is_already_finalized(&finalized_head, target)? + Self::is_already_finalized(&state.sync_state, target)? { return Ok(()); } @@ -132,7 +121,7 @@ impl ExecutableEngineTask for FinalizeTask { // A by-number target only learns its hash from the lookup. if matches!(self.block_id, FinalizeBlockId::ByNumber(_)) && - Self::is_already_finalized(&finalized_head, block_info.block_info.id())? + Self::is_already_finalized(&state.sync_state, block_info.block_info.id())? { return Ok(()); } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs index 45420631095..5708a52c150 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs @@ -1,7 +1,7 @@ use crate::{ EngineState, EngineSyncState, EngineSyncStateUpdate, EngineTaskError, EngineTaskErrorSeverity::{self, Critical, Skip, Temporary}, - ExecutableEngineTask, FinalizeBlockId, FinalizeTask, FinalizeTaskError, + ExecutableEngineTask, FinalityViolation, FinalizeBlockId, FinalizeTask, FinalizeTaskError, test_utils::{ RpcMock, TestEngineStateBuilder, TestRpcBlock as RpcBlock, test_block_info, test_engine_client, test_l2_block, test_rpc_block, @@ -103,11 +103,20 @@ async fn finalize( } const fn is_behind(err: &FinalizeTaskError) -> bool { - matches!(err, FinalizeTaskError::BehindFinalizedHead { finalized: FINALIZED, target: STALE }) + matches!( + err, + FinalizeTaskError::BreaksFinality(FinalityViolation::BehindFinalizedHead { + finalized: BlockNumHash { number: FINALIZED, .. }, + target: STALE, + }) + ) } const fn is_conflict(err: &FinalizeTaskError) -> bool { - matches!(err, FinalizeTaskError::ConflictsWithFinalizedHead { .. }) + matches!( + err, + FinalizeTaskError::BreaksFinality(FinalityViolation::ConflictsWithFinalizedHead { .. }) + ) } const fn is_not_canonical(err: &FinalizeTaskError) -> bool { diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs index a9bc15146ff..fc28ccf0026 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs @@ -100,6 +100,7 @@ async fn send_forkchoice( match &valid_response.payload_status.status { PayloadStatusEnum::Valid => { + state.el_applied_last_forkchoice = true; if !state.el_sync_finished { info!(target: "engine", "Finished execution layer sync."); state.el_sync_finished = true; @@ -107,6 +108,7 @@ async fn send_forkchoice( } // If we're not building a new payload, we're driving EL sync. PayloadStatusEnum::Syncing => { + state.el_applied_last_forkchoice = false; debug!(target: "engine", "Attempting to update forkchoice state while EL syncing") } // Other codes are not expected. diff --git a/rust/kona/crates/node/engine/src/test_utils/engine_state.rs b/rust/kona/crates/node/engine/src/test_utils/engine_state.rs index 60542823b96..1ed3c5b8178 100644 --- a/rust/kona/crates/node/engine/src/test_utils/engine_state.rs +++ b/rust/kona/crates/node/engine/src/test_utils/engine_state.rs @@ -10,6 +10,7 @@ pub struct TestEngineStateBuilder { local_safe_head: Option, safe_head: Option, finalized_head: Option, + el_applied_last_forkchoice: bool, el_sync_finished: bool, } @@ -33,6 +34,7 @@ impl TestEngineStateBuilder { local_safe_head: None, safe_head: None, finalized_head: None, + el_applied_last_forkchoice: true, el_sync_finished: true, } } @@ -55,13 +57,22 @@ impl TestEngineStateBuilder { self } - /// Sets whether EL sync is finished + /// Sets whether EL sync is finished. An execution layer still syncing has not applied the last + /// forkchoice either, so this sets both flags. #[allow(dead_code)] pub const fn with_el_sync_finished(mut self, finished: bool) -> Self { + self.el_applied_last_forkchoice = finished; self.el_sync_finished = finished; self } + /// Sets whether the execution layer applied the last forkchoice update. + #[allow(dead_code)] + pub const fn with_el_applied_last_forkchoice(mut self, applied: bool) -> Self { + self.el_applied_last_forkchoice = applied; + self + } + /// Builds the `EngineState` pub fn build(self) -> EngineState { let mut state = EngineState::default(); @@ -74,6 +85,7 @@ impl TestEngineStateBuilder { finalized_head: Some(self.finalized_head.unwrap_or(self.unsafe_head)), }); + state.el_applied_last_forkchoice = self.el_applied_last_forkchoice; state.el_sync_finished = self.el_sync_finished; state } diff --git a/rust/kona/crates/node/engine/src/test_utils/rpc.rs b/rust/kona/crates/node/engine/src/test_utils/rpc.rs index 0bd0bfef0c6..cda70c224a6 100644 --- a/rust/kona/crates/node/engine/src/test_utils/rpc.rs +++ b/rust/kona/crates/node/engine/src/test_utils/rpc.rs @@ -70,6 +70,16 @@ impl RpcMock { ); } + /// Expects a header-only `eth_getBlockByNumber` call for `number` and answers it with `block`, + /// or with null for `None`. + pub fn expect_block_header_by_number(&self, number: u64, block: Option) { + self.expect_params( + "eth_getBlockByNumber", + serde_json::json!([format!("{number:#x}"), false]), + block, + ); + } + /// Expects an `engine_forkchoiceUpdatedV3` call for `forkchoice` without payload attributes and /// answers it with `status`. pub fn expect_forkchoice_updated( diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs index 480019925a0..37dc0e00185 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs @@ -6,7 +6,7 @@ use alloy_primitives::BlockHash; use async_trait::async_trait; use jsonrpsee::core::ClientError; use kona_derive::{ChainProvider, Signal}; -use kona_engine::{EngineState, FinalizeBlockId}; +use kona_engine::{EngineState, FinalityViolation, FinalizeBlockId}; use kona_protocol::{BlockInfo, L2BlockInfo, SyncStatus}; use kona_rpc::RollupNodeApiClient; use thiserror::Error; @@ -77,6 +77,15 @@ where has_engine_sync_completed: bool, /// Ticker driving periodic polls of the derivation delegate. delegated_derivation_ticker: time::Interval, + + /// A safe head behind the engine's that was forwarded while the execution layer applied the + /// engine's forkchoice, with the engine's safe head at the time. The engine checks such a head + /// against its execution layer once; until either changes, sending it again repeats that + /// lookup to the same answer. + forwarded_lagging_safe_head: Option<(BlockNumHash, BlockNumHash)>, + /// The last rejection of a delegate status, so a delegate stuck on one inconsistency is + /// warned about once rather than on every poll. + last_rejection: Option, } impl @@ -107,6 +116,8 @@ where engine_l2_safe_head: L2BlockInfo::default(), has_engine_sync_completed: false, delegated_derivation_ticker, + forwarded_lagging_safe_head: None, + last_rejection: None, } } } @@ -175,58 +186,44 @@ where Ok(()) } - /// The id of an established finalized head. The default head means finality has not been - /// established yet, so any finalized block from the delegate may initialize it. - fn established_finalized_l2(finalized_head: L2BlockInfo) -> Option { - (finalized_head != L2BlockInfo::default()).then(|| finalized_head.block_info.id()) + /// Rejects a status whose labels are out of order: its finalized block ahead of its safe head, + /// or its safe head ahead of its local safe head. No consistent delegate reports either, and a + /// finalized block ahead of the safe head would let a stale safe head sit below finality. A + /// zero local safe head is a label the delegate has not derived or does not report, so it does + /// not order anything. + fn validate_label_order(status: &SyncStatus) -> Result<(), DerivationDelegationError> { + let finalized = status.finalized_l2.block_info.number; + let safe = status.safe_l2.block_info.number; + let local_safe = if status.local_safe_l2 == L2BlockInfo::default() { + safe + } else { + status.local_safe_l2.block_info.number + }; + if finalized > safe { + return Err(DerivationDelegationError::FinalizedAheadOfSafe { finalized, safe }); + } + if safe > local_safe { + return Err(DerivationDelegationError::SafeAheadOfLocalSafe { local_safe, safe }); + } + Ok(()) } - /// Rejects a finalized L2 block behind, or at the same height as but different from, the - /// engine's. Either means the delegate lags or follows another chain, so none of its status - /// applies: its safe head would move the engine back or onto that chain too. + /// Verifies that the labels reported by the derivation delegate are in order, that its + /// finalized L2 block does not move back from the engine's, and that its L1 info is consistent + /// with the canonical L1 chain. /// /// Only a fork at the engine's finalized height is caught here. A chain that forked below it /// and reports a higher finalized block passes, and the execution layer rejects the forkchoice /// update that would move onto it. - fn validate_finalized_l2_progress( - &self, - reported: BlockNumHash, + async fn validate_sync_status( + &mut self, + v: &SyncStatus, ) -> Result<(), DerivationDelegationError> { - let engine_finalized_head = self.engine_state_rx.borrow().sync_state.finalized_head(); - let Some(engine) = Self::established_finalized_l2(engine_finalized_head) else { - return Ok(()); - }; - if reported.number < engine.number { - return Err(DerivationDelegationError::FinalizedL2Regressed { engine, reported }); - } - if reported.number == engine.number && reported.hash != engine.hash { - return Err(DerivationDelegationError::FinalizedL2Conflict { engine, reported }); - } - Ok(()) - } - - /// Verifies that the derivation delegate's finalized L2 block is not above its safe L2 block - /// and does not move back from the engine's, and that its L1 info is consistent with the - /// canonical L1 chain. - async fn validate_sync_status(&mut self, v: &SyncStatus) -> bool { - // A finalized block above the safe head is an inconsistent snapshot; the engine would only - // refuse to finalize it. - if v.finalized_l2.block_info.number > v.safe_l2.block_info.number { - let err = DerivationDelegationError::FinalizedAheadOfSafe { - finalized: v.finalized_l2.block_info.number, - safe: v.safe_l2.block_info.number, - }; - warn!(target: "derivation", error = %err, "Inconsistent sync status from delegate"); - return false; - } - if let Err(err) = self.validate_finalized_l2_progress(v.finalized_l2.block_info.id()) { - warn!( - target: "derivation", - error = %err, - "Finalized L2 inconsistency detected at sync status from delegate" - ); - return false; - } + Self::validate_label_order(v)?; + self.engine_state_rx + .borrow() + .sync_state + .ensure_respects_finality(v.finalized_l2.block_info.id())?; let checks = [ ("L1 Origin of Safe L2", v.safe_l2.l1_origin.number, v.safe_l2.l1_origin.hash), @@ -237,18 +234,52 @@ where ), ("Current L1", v.current_l1.number, v.current_l1.hash), ]; + let mut validated_l1_blocks = Vec::with_capacity(checks.len()); for (context, number, hash) in checks { - if let Err(err) = self.validate_l1_block(context, number, hash).await { - warn!( - target: "derivation", - context = context, - error = %err, - "L1 inconsistency detected at sync status from delegate" - ); - return false; + // Labels often share an L1 origin, and one lookup proves it canonical for all of them. + if validated_l1_blocks.contains(&(number, hash)) { + continue; } + self.validate_l1_block(context, number, hash).await?; + validated_l1_blocks.push((number, hash)); + } + Ok(()) + } + + /// Logs a rejected delegate status at `warn!` when the reason changed, and at `debug!` while + /// the delegate keeps reporting the same inconsistency. + fn log_rejected_status(&mut self, err: &DerivationDelegationError) { + let reason = err.to_string(); + if self.last_rejection.as_ref() == Some(&reason) { + debug!(target: "derivation", error = %err, "Sync status from delegate rejected again"); + } else { + warn!(target: "derivation", error = %err, "Rejected sync status from delegate"); + self.last_rejection = Some(reason); } - true + } + + /// Whether the delegated safe head is the lagging one already forwarded against the same + /// engine safe head, so the engine would repeat its lookup to the same answer. + fn is_lagging_safe_head_forwarded(&self, safe_l2: &L2BlockInfo) -> bool { + self.lagging_safe_head_to_remember(safe_l2) == self.forwarded_lagging_safe_head && + self.forwarded_lagging_safe_head.is_some() + } + + /// The pair to remember for a delegated safe head behind the engine's consistent safe head + /// while its execution layer applied the engine's forkchoice, or `None` for any other head, + /// which the engine must always see. + fn lagging_safe_head_to_remember( + &self, + safe_l2: &L2BlockInfo, + ) -> Option<(BlockNumHash, BlockNumHash)> { + let engine_state = self.engine_state_rx.borrow(); + let engine_safe_head = engine_state.sync_state.safe_head().block_info; + let is_consistent = + engine_state.sync_state.unsafe_head().block_info.number >= engine_safe_head.number; + (engine_state.el_applied_last_forkchoice && + is_consistent && + safe_l2.block_info.number < engine_safe_head.number) + .then(|| (safe_l2.block_info.id(), engine_safe_head.id())) } /// Fetches, validates, and applies sync status from the derivation delegate. @@ -265,18 +296,30 @@ where } }; - if !self.validate_sync_status(&sync_status).await { + if let Err(err) = self.validate_sync_status(&sync_status).await { // Validation failures here are expected to be transient, so we skip processing // this sync status and continue delegating derivation instead of treating it as // fatal. + self.log_rejected_status(&err); return Ok(()); } + self.last_rejection = None; self.derivation_origin_tx.send_replace(Some(sync_status.current_l1)); - self.engine_client - .send_safe_l2_signal(sync_status.safe_l2.into()) - .await - .map_err(|e| DerivationError::Sender(Box::new(e)))?; + if self.is_lagging_safe_head_forwarded(&sync_status.safe_l2) { + debug!( + target: "derivation", + safe_l2 = ?sync_status.safe_l2, + "Not forwarding a lagging safe head the engine already checked" + ); + } else { + self.forwarded_lagging_safe_head = + self.lagging_safe_head_to_remember(&sync_status.safe_l2); + self.engine_client + .send_safe_l2_signal(sync_status.safe_l2.into()) + .await + .map_err(|e| DerivationError::Sender(Box::new(e)))?; + } // Delegated polling supplies `(number, hash)`. Carry the hash through so the engine // finalizes the specific block we were asked to, not whatever it happens to have at the @@ -329,17 +372,14 @@ where #[derive(Error, Debug)] enum DerivationDelegationError { - /// The delegate's finalized L2 block is above its safe L2 block. + /// The delegate's finalized L2 block is ahead of its own safe head. #[error("Finalized L2 {finalized} is ahead of safe L2 {safe}")] FinalizedAheadOfSafe { finalized: u64, safe: u64 }, - /// The delegate's finalized L2 block is a different block at the engine's finalized height. - #[error("Finalized L2 {reported:?} conflicts with the engine's finalized L2 {engine:?}")] - FinalizedL2Conflict { engine: BlockNumHash, reported: BlockNumHash }, - - /// The delegate's finalized L2 block is behind the engine's. - #[error("Finalized L2 {reported:?} is behind the engine's finalized L2 {engine:?}")] - FinalizedL2Regressed { engine: BlockNumHash, reported: BlockNumHash }, + /// The delegate's finalized L2 block is behind the engine's, or a different block at its + /// height. + #[error("Finalized L2 moves the engine's finality back: {0}")] + FinalizedL2BreaksFinality(#[from] FinalityViolation), /// The L1 provider returned an error (network, RPC, etc.) #[error("L1 provider error: {0}")] @@ -348,4 +388,8 @@ enum DerivationDelegationError { /// The hash provided by the derivation delegation does not match the canonical chain. #[error("L1 inconsistency in {context} at block {number}: expected {expected}, got {actual}")] L1ValidationFailed { context: String, number: u64, expected: BlockHash, actual: BlockHash }, + + /// The delegate's safe head is ahead of its own local safe head. + #[error("Safe L2 {safe} is ahead of local safe L2 {local_safe}")] + SafeAheadOfLocalSafe { local_safe: u64, safe: u64 }, } diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs index fea52b068d1..b586f878488 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs @@ -88,6 +88,15 @@ impl Harness { self.engine_state.send_modify(|state| state.sync_state = sync_state); } + /// Publishes the engine's labels along with whether its execution layer applied the engine's + /// last forkchoice. + fn publish_engine_state(&self, sync_state: EngineSyncState, el_applied_last_forkchoice: bool) { + self.engine_state.send_modify(|state| { + state.sync_state = sync_state; + state.el_applied_last_forkchoice = el_applied_last_forkchoice; + }); + } + /// The engine requests sent since the last call, in order. fn forwarded(&mut self) -> Vec { std::iter::from_fn(|| self.engine_requests.try_recv().ok()).map(Forwarded::from).collect() @@ -367,3 +376,92 @@ async fn poll_rejects_status_whose_finalized_is_ahead_of_its_safe( assert_eq!(harness.forwarded(), forwarded_if(accepted, &status)); } + +/// A status whose labels are out of order is rejected whole: a finalized block ahead of the safe +/// head would let a stale snapshot's safe head sit below finality, and a safe head ahead of the +/// local safe head is not a state any delegate reaches. +#[rstest] +#[case::ordered(15, 20, 20, true)] +#[case::finalized_at_safe(20, 20, 20, true)] +#[case::safe_behind_local_safe(15, 18, 20, true)] +#[case::finalized_ahead_of_safe(16, 15, 20, false)] +#[case::safe_ahead_of_local_safe(15, 21, 20, false)] +#[tokio::test(start_paused = true)] +async fn poll_rejects_status_with_labels_out_of_order( + #[case] finalized: u64, + #[case] safe: u64, + #[case] local_safe: u64, + #[case] accepted: bool, +) { + let status = SyncStatus { + finalized_l2: l2_block(finalized, FINALIZED_ORIGIN), + local_safe_l2: l2_block(local_safe, SAFE_ORIGIN), + safe_l2: l2_block(safe, SAFE_ORIGIN), + ..delegate_status() + }; + let mut harness = Harness::new([Ok(status.clone())]); + harness.complete_engine_sync().await; + + harness.actor.step().await.unwrap(); + + assert_eq!(harness.forwarded(), forwarded_if(accepted, &status)); + assert_eq!(harness.current_l1.borrow().is_some(), accepted); +} + +/// A zero local safe head is a label the delegate has not derived or does not report, so it does +/// not put the safe head out of order and the status is forwarded. +#[tokio::test(start_paused = true)] +async fn unreported_local_safe_head_does_not_order_the_labels() { + let status = SyncStatus { local_safe_l2: L2BlockInfo::default(), ..delegate_status() }; + let mut harness = Harness::new([Ok(status.clone())]); + harness.complete_engine_sync().await; + + harness.actor.step().await.unwrap(); + + assert_eq!(harness.forwarded(), forwarded_for(&status)); +} + +/// An engine sync state with consistent labels: unsafe at 30, safe at `safe`, finalized at 15. +fn engine_safe_at(safe: u64) -> EngineSyncState { + EngineSyncState::default().apply_update(EngineSyncStateUpdate { + finalized_head: Some(l2_block(15, FINALIZED_ORIGIN)), + local_safe_head: Some(l2_block(safe, SAFE_ORIGIN)), + safe_head: Some(l2_block(safe, SAFE_ORIGIN)), + unsafe_head: Some(l2_block(30, SAFE_ORIGIN)), + }) +} + +/// What a poll of `status` forwards when its safe head is held back: the finalized block alone. +fn finalized_only(status: &SyncStatus) -> Vec { + vec![Forwarded::Finalized(FinalizeBlockId::ByHash(status.finalized_l2.block_info.id()))] +} + +/// A safe head behind the engine's is forwarded once while the execution layer applied the +/// engine's forkchoice, since the engine's answer to it cannot change; finalized still goes out +/// on every poll. A moved engine safe head, or an execution layer that did not apply the last +/// forkchoice, sends it again. +#[rstest] +#[case::same_engine_view(true, 25, finalized_only)] +#[case::el_not_applied(false, 25, forwarded_for)] +#[case::engine_safe_moved(true, 26, forwarded_for)] +#[tokio::test(start_paused = true)] +async fn lagging_safe_head_is_forwarded_once_per_engine_view( + #[case] el_applied_last_forkchoice: bool, + #[case] engine_safe_on_second_poll: u64, + #[case] second_poll_forwards: fn(&SyncStatus) -> Vec, +) { + let status = delegate_status(); + let mut harness = Harness::new([Ok(status.clone()), Ok(status.clone())]); + harness.complete_engine_sync_with(engine_safe_at(25)).await; + harness.publish_engine_state(engine_safe_at(25), el_applied_last_forkchoice); + harness.actor.step().await.unwrap(); + assert_eq!(harness.forwarded(), forwarded_for(&status)); + harness.publish_engine_state( + engine_safe_at(engine_safe_on_second_poll), + el_applied_last_forkchoice, + ); + + harness.actor.step().await.unwrap(); + + assert_eq!(harness.forwarded(), second_poll_forwards(&status)); +} diff --git a/rust/kona/crates/node/service/src/actors/engine/actor.rs b/rust/kona/crates/node/service/src/actors/engine/actor.rs index f09047db8a2..b45bfa62238 100644 --- a/rust/kona/crates/node/service/src/actors/engine/actor.rs +++ b/rust/kona/crates/node/service/src/actors/engine/actor.rs @@ -235,7 +235,7 @@ where self.send_derivation_actor_safe_head_if_updated().await?; if !self.el_sync_complete && self.engine.state().el_sync_finished { - if self.engine.state().sync_state.finalized_head() == L2BlockInfo::default() { + if self.engine.state().sync_state.established_finalized_head().is_none() { self.begin_reset(); } else { self.mark_el_sync_complete_and_notify_derivation_actor().await?;