From 7bdee17022acf50261e6592988c0309ab956aa8c Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 11 Nov 2026 18:16:47 +0100 Subject: [PATCH] kona-engine,kona-node-service: keep a stale delegated safe head from moving safe below finality A stale delegate snapshot can report a safe head below kona-node's finalized head, and kona-node applies it, moving safe behind its own finality. Kona-node now drops a delegate status whose labels are out of order (a zero local-safe head counts as unreported) or that breaks the engine's applied finality, and the engine skips a delegated safe head below, or conflicting with, established finality. A delegated safe head below safe is ignored, and forwarded only once per engine view, unless an EL that applied the last forkchoice holds a different block at that height, which proves divergence and reorgs the engine above finality. Labels left with unsafe below safe are repaired by the next delegated head above finality. Repeated rejection warnings are deduplicated. A rejected delegated forkchoice update resets the engine as on develop; recovery is #24065 (#24061). Known window: after a delegate L1 reorg whose fork point is above its new safe head, safe stays on the old fork until the delegate passes it. Closes #23910. BREAKING CHANGE: `ConsolidateTaskError` gains `BreaksFinality` and `FailedToFetchDelegatedSafeBlock`; `FinalizeTaskError::{BehindFinalizedHead, ConflictsWithFinalizedHead}` become `BreaksFinality(FinalityViolation)`; `FinalityViolation` is exported; `EngineState` gains `el_applied_last_forkchoice`. Co-Authored-By: Claude Opus 5.5 --- rust/kona/crates/node/engine/src/lib.rs | 2 +- .../kona/crates/node/engine/src/state/core.rs | 16 +- .../crates/node/engine/src/state/finality.rs | 126 ++++ rust/kona/crates/node/engine/src/state/mod.rs | 3 + .../src/task_queue/tasks/consolidate/error.rs | 14 +- .../src/task_queue/tasks/consolidate/task.rs | 578 +++++++++++++++--- .../src/task_queue/tasks/finalize/error.rs | 25 +- .../src/task_queue/tasks/finalize/task.rs | 27 +- .../task_queue/tasks/finalize/task_test.rs | 15 +- .../task_queue/tasks/synchronize/helper.rs | 2 + .../engine/src/test_utils/engine_state.rs | 14 +- .../crates/node/engine/src/test_utils/rpc.rs | 10 + .../src/actors/derivation/delegated/actor.rs | 180 +++--- .../src/actors/derivation/delegated/tests.rs | 98 +++ .../node/service/src/actors/engine/actor.rs | 2 +- 15 files changed, 925 insertions(+), 187 deletions(-) create mode 100644 rust/kona/crates/node/engine/src/state/finality.rs diff --git a/rust/kona/crates/node/engine/src/lib.rs b/rust/kona/crates/node/engine/src/lib.rs index c1f955096bb..e8fd1f9305d 100644 --- a/rust/kona/crates/node/engine/src/lib.rs +++ b/rust/kona/crates/node/engine/src/lib.rs @@ -65,7 +65,7 @@ mod versions; pub use versions::{EngineForkchoiceVersion, EngineGetPayloadVersion, EngineNewPayloadVersion}; mod state; -pub use state::{EngineState, EngineSyncState, EngineSyncStateUpdate}; +pub use state::{EngineState, EngineSyncState, EngineSyncStateUpdate, FinalityViolation}; mod kinds; pub use kinds::EngineKind; diff --git a/rust/kona/crates/node/engine/src/state/core.rs b/rust/kona/crates/node/engine/src/state/core.rs index 7a34ce3f626..67204d7d81f 100644 --- a/rust/kona/crates/node/engine/src/state/core.rs +++ b/rust/kona/crates/node/engine/src/state/core.rs @@ -1,6 +1,7 @@ //! The internal state of the engine controller. use crate::Metrics; +use alloy_eips::BlockNumHash; use alloy_rpc_types_engine::ForkchoiceState; use kona_protocol::L2BlockInfo; use serde::{Deserialize, Serialize}; @@ -54,6 +55,12 @@ impl EngineSyncState { self.finalized_head } + /// Returns the id of the finalized head once finality is established. The default finalized + /// head means it is not yet, so no block pins finality. + pub fn established_finalized_head(&self) -> Option { + (self.finalized_head != L2BlockInfo::default()).then(|| self.finalized_head.block_info.id()) + } + /// Creates a `ForkchoiceState` /// /// - `head_block` = `unsafe_head` @@ -127,8 +134,10 @@ pub struct EngineSyncStateUpdate { /// The chain state viewed by the engine controller. #[derive(Default, Debug, Copy, Clone, PartialEq, Eq)] pub struct EngineState { - /// The sync state of the engine. - pub sync_state: EngineSyncState, + /// Whether the execution layer answered `VALID` to the last forkchoice update the engine sent, + /// so its canonical chain is the engine's. Unlike `el_sync_finished`, a later `SYNCING` answer + /// clears it. + pub el_applied_last_forkchoice: bool, /// Whether or not the EL has finished syncing. pub el_sync_finished: bool, @@ -139,6 +148,9 @@ pub struct EngineState { /// because engine may forgot backupUnsafeHead or backupUnsafeHead is not part /// of the chain. pub need_fcu_call_backup_unsafe_reorg: bool, + + /// The sync state of the engine. + pub sync_state: EngineSyncState, } impl EngineState { diff --git a/rust/kona/crates/node/engine/src/state/finality.rs b/rust/kona/crates/node/engine/src/state/finality.rs new file mode 100644 index 00000000000..94363c549bf --- /dev/null +++ b/rust/kona/crates/node/engine/src/state/finality.rs @@ -0,0 +1,126 @@ +//! Checks that keep a label from moving below finality. + +use crate::EngineSyncState; +use alloy_eips::BlockNumHash; +use thiserror::Error; + +/// A block that a label cannot move to, because it would move the label below finality. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)] +pub enum FinalityViolation { + /// The block is below the finalized head. + #[error("Block {target} is behind the finalized head {finalized:?}")] + BehindFinalizedHead { + /// The finalized head. + finalized: BlockNumHash, + /// The number of the block. + target: u64, + }, + /// The block is a different block at the finalized head's height. + #[error("Block {target:?} conflicts with the finalized head {finalized:?}")] + ConflictsWithFinalizedHead { + /// The finalized head. + finalized: BlockNumHash, + /// The block. + target: BlockNumHash, + }, +} + +impl EngineSyncState { + /// Rejects a block number below the established finalized head. + pub fn ensure_not_behind_finality(&self, target: u64) -> Result<(), FinalityViolation> { + match self.established_finalized_head() { + Some(finalized) if target < finalized.number => { + Err(FinalityViolation::BehindFinalizedHead { finalized, target }) + } + _ => Ok(()), + } + } + + /// Rejects a different block at the established finalized head's height. Before finality is + /// established no block pins it, so any block, including the configured genesis, passes. + pub fn ensure_no_conflict_with_finality( + &self, + target: BlockNumHash, + ) -> Result<(), FinalityViolation> { + match self.established_finalized_head() { + Some(finalized) + if target.number == finalized.number && target.hash != finalized.hash => + { + Err(FinalityViolation::ConflictsWithFinalizedHead { finalized, target }) + } + _ => Ok(()), + } + } + + /// Rejects a block below the established finalized head or conflicting with it. + pub fn ensure_respects_finality(&self, target: BlockNumHash) -> Result<(), FinalityViolation> { + self.ensure_not_behind_finality(target.number)?; + self.ensure_no_conflict_with_finality(target) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::EngineSyncStateUpdate; + use alloy_primitives::B256; + use kona_protocol::{BlockInfo, L2BlockInfo}; + + const FINALIZED_NUMBER: u64 = 5; + + fn block(number: u64, hash: u8) -> L2BlockInfo { + L2BlockInfo { + block_info: BlockInfo { hash: B256::repeat_byte(hash), number, ..Default::default() }, + ..Default::default() + } + } + + fn sync_state_finalized_at(finalized_head: L2BlockInfo) -> EngineSyncState { + EngineSyncState::default().apply_update(EngineSyncStateUpdate { + finalized_head: Some(finalized_head), + ..Default::default() + }) + } + + /// The finalized head at 5 once finality is established, and the default head before. + fn finalized_head(is_finality_established: bool) -> L2BlockInfo { + if is_finality_established { block(FINALIZED_NUMBER, 1) } else { L2BlockInfo::default() } + } + + /// The violation expected for `target` checked against the finalized head at 5. + fn behind(target: L2BlockInfo) -> Option { + Some(FinalityViolation::BehindFinalizedHead { + finalized: block(FINALIZED_NUMBER, 1).block_info.id(), + target: target.block_info.number, + }) + } + + /// The violation expected for `target` conflicting with the finalized head at 5. + fn conflicting(target: L2BlockInfo) -> Option { + Some(FinalityViolation::ConflictsWithFinalizedHead { + finalized: block(FINALIZED_NUMBER, 1).block_info.id(), + target: target.block_info.id(), + }) + } + + /// A block below the established finalized head, or a different block at its height, breaks + /// finality. Before finality is established no block pins it, so a different block at the + /// default head's height, genesis, passes. + #[rstest::rstest] + #[case::above(true, block(FINALIZED_NUMBER + 1, 2), None)] + #[case::at(true, block(FINALIZED_NUMBER, 1), None)] + #[case::below(true, block(FINALIZED_NUMBER - 1, 2), behind(block(FINALIZED_NUMBER - 1, 2)))] + #[case::conflicting(true, block(FINALIZED_NUMBER, 2), conflicting(block(FINALIZED_NUMBER, 2)))] + #[case::not_established(false, block(0, 2), None)] + fn block_is_checked_against_established_finality( + #[case] is_finality_established: bool, + #[case] target: L2BlockInfo, + #[case] expected_violation: Option, + ) { + let sync_state = sync_state_finalized_at(finalized_head(is_finality_established)); + + let result = sync_state.ensure_respects_finality(target.block_info.id()); + + assert_eq!(result.err(), expected_violation); + } +} diff --git a/rust/kona/crates/node/engine/src/state/mod.rs b/rust/kona/crates/node/engine/src/state/mod.rs index b61100917e6..a3b0cfe6f51 100644 --- a/rust/kona/crates/node/engine/src/state/mod.rs +++ b/rust/kona/crates/node/engine/src/state/mod.rs @@ -2,3 +2,6 @@ mod core; pub use core::{EngineState, EngineSyncState, EngineSyncStateUpdate}; + +mod finality; +pub use finality::FinalityViolation; diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/error.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/error.rs index c5bd5dbcb0f..2ce1b187a76 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/error.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/error.rs @@ -1,7 +1,7 @@ //! Contains error types for the [`crate::ConsolidateTask`]. use crate::{ - BuildTaskError, EngineTaskError, SealTaskError, SynchronizeTaskError, + BuildTaskError, EngineTaskError, FinalityViolation, SealTaskError, SynchronizeTaskError, task_queue::tasks::{BuildAndImportError, task::EngineTaskErrorSeverity}, }; use thiserror::Error; @@ -9,6 +9,13 @@ use thiserror::Error; /// An error that occurs when running the [`crate::ConsolidateTask`]. #[derive(Debug, Error)] pub enum ConsolidateTaskError { + /// A delegated safe head would move the safe labels below finality. + #[error("Delegated safe head breaks finality: {0}")] + BreaksFinality(#[from] FinalityViolation), + /// Failed to fetch the execution layer's block at the height of a delegated safe head below the + /// safe head. + #[error("Failed to fetch the block at delegated safe head height {0}")] + FailedToFetchDelegatedSafeBlock(u64), /// The unsafe L2 block is missing. #[error("Unsafe L2 block is missing {0}")] MissingUnsafeL2Block(u64), @@ -38,8 +45,11 @@ impl From for ConsolidateTaskError { impl EngineTaskError for ConsolidateTaskError { fn severity(&self) -> EngineTaskErrorSeverity { match self { + Self::BreaksFinality(_) => EngineTaskErrorSeverity::Skip, Self::MissingUnsafeL2Block(_) => EngineTaskErrorSeverity::Reset, - Self::FailedToFetchUnsafeL2Block => EngineTaskErrorSeverity::Temporary, + Self::FailedToFetchDelegatedSafeBlock(_) | Self::FailedToFetchUnsafeL2Block => { + EngineTaskErrorSeverity::Temporary + } Self::BuildTaskFailed(inner) => inner.severity(), Self::SealTaskFailed(inner) => inner.severity(), Self::ForkchoiceUpdateFailed(inner) => inner.severity(), diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs index d3183cdafa5..00f31457729 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs @@ -100,6 +100,159 @@ impl ConsolidateTask { Ok(()) } + /// This provides symmetric fallback behavior with `build_and_import`. + async fn reconcile_to_safe_head( + &self, + state: &mut EngineState, + safe_l2: &L2BlockInfo, + ) -> Result<(), ConsolidateTaskError> { + warn!( + target: "engine", + safe_l2 = %safe_l2, + "Apply safe head" + ); + + // We intentionally set the unsafe head to safe_l2 to ensure the engine observes a + // self-consistent head state. This is required to correctly handle reorgs (where unsafe + // may be ahead on a non-canonical fork) and to trigger EL sync when the local unsafe head + // lags behind the safe head. + self.apply_delegated_safe_head( + state, + safe_l2, + EngineSyncStateUpdate { + local_safe_head: Some(*safe_l2), + safe_head: Some(*safe_l2), + unsafe_head: Some(*safe_l2), + ..Default::default() + }, + "Updated safe head via follow safe", + ) + .await + } + + /// Moves the safe labels to a delegated safe head that the execution layer does not have yet, + /// leaving the unsafe head in place. + /// + /// A block missing below the unsafe head means the execution layer is still syncing toward the + /// unsafe head. Resetting would restart that sync from the execution layer's old labels, and + /// moving the unsafe head back would retarget it. Like op-node's follow-source mode, only the + /// safe labels follow the delegate, and the forkchoice update carries them to the execution + /// layer. + async fn follow_safe_head_during_el_sync( + &self, + state: &mut EngineState, + safe_l2: &L2BlockInfo, + ) -> Result<(), ConsolidateTaskError> { + warn!( + target: "engine", + safe_l2 = %safe_l2, + unsafe_number = state.sync_state.unsafe_head().block_info.number, + "Execution layer does not have the delegated safe head yet; keeping its sync target" + ); + + self.apply_delegated_safe_head( + state, + safe_l2, + EngineSyncStateUpdate { + local_safe_head: Some(*safe_l2), + safe_head: Some(*safe_l2), + ..Default::default() + }, + "Updated safe head while the execution layer syncs", + ) + .await + } + + /// Applies `update`, whose labels follow the delegated `safe_l2`, and always sends its + /// forkchoice update: the execution layer may have answered `SYNCING` to the same labels + /// before, or lost them in a restart. + async fn apply_delegated_safe_head( + &self, + state: &mut EngineState, + safe_l2: &L2BlockInfo, + update: EngineSyncStateUpdate, + outcome: &'static str, + ) -> Result<(), ConsolidateTaskError> { + synchronize_unconditionally(self.client.as_ref(), state, update) + .await + .map_err(|e| { + warn!(target: "engine", ?e, safe_l2 = %safe_l2, "Applying the delegated safe head failed"); + e + })?; + + info!( + target: "engine", + hash = %safe_l2.block_info.hash, + number = safe_l2.block_info.number, + "{outcome}" + ); + + Ok(()) + } + + /// Follows a delegated safe head below the safe head. Only a different block at its height in + /// a synced execution layer proves the delegate diverged and reorgs the engine onto it; the + /// same block (a lagging delegate) or a missing one leaves the labels. After a delegate L1 + /// reorg whose fork point is above the head it reports, the labels stay on the abandoned + /// blocks until it reports a head above that point. + async fn follow_delegated_safe_head_behind_safe_head( + &self, + state: &mut EngineState, + safe_l2: &L2BlockInfo, + ) -> Result<(), ConsolidateTaskError> { + // Until the execution layer applies the engine's forkchoice, its canonical block at that + // height may predate the engine's own labels, so it proves nothing about the delegate. + if !state.el_applied_last_forkchoice { + debug!( + target: "engine", + safe_l2 = %safe_l2, + "Ignoring a delegated safe head behind the safe head while the execution layer syncs" + ); + return Ok(()); + } + debug!( + target: "engine", + safe_l2 = %safe_l2, + "Looking up the execution layer's block at a delegated safe head behind the safe head" + ); + let el_block = + self.client.get_l2_block(safe_l2.block_info.number.into()).await.map_err(|err| { + warn!( + target: "engine", + ?err, + safe_l2 = %safe_l2, + "Failed to look up the execution layer's block at a delegated safe head" + ); + ConsolidateTaskError::FailedToFetchDelegatedSafeBlock(safe_l2.block_info.number) + })?; + debug!( + target: "engine", + safe_l2 = %safe_l2, + el_hash = ?el_block.as_ref().map(|block| block.header.hash), + "Looked up the execution layer's block at a delegated safe head" + ); + match el_block { + Some(el_block) if el_block.header.hash != safe_l2.block_info.hash => { + debug!( + target: "engine", + safe_l2 = %safe_l2, + el_hash = %el_block.header.hash, + "Delegated safe head behind the safe head diverges from the execution layer" + ); + self.reconcile_to_safe_head(state, safe_l2).await + } + _ => { + debug!( + target: "engine", + safe_l2 = %safe_l2, + safe_number = state.sync_state.safe_head().block_info.number, + "Ignoring a delegated safe head behind the safe head" + ); + Ok(()) + } + } + } + /// Handles the fallback case when the block doesn't match the input or does not exist. async fn reconcile_unsafe_to_safe( &self, @@ -110,37 +263,7 @@ impl ConsolidateTask { self.build_and_import_attributes(state, attributes).await } ConsolidateInput::BlockInfo(safe_l2) => { - warn!(target: "engine", safe_l2 = %safe_l2, "Apply safe head"); - - // We intentionally set the unsafe head to safe_l2 to ensure the engine observes a - // self-consistent head state. This is required to correctly handle reorgs (where - // unsafe may be ahead on a non-canonical fork) and to trigger EL sync when the - // local unsafe head lags behind the safe head. The forkchoice update is always - // sent: the execution layer may have answered `SYNCING` to the same labels before, - // or lost them in a restart. - synchronize_unconditionally( - self.client.as_ref(), - state, - EngineSyncStateUpdate { - local_safe_head: Some(*safe_l2), - safe_head: Some(*safe_l2), - unsafe_head: Some(*safe_l2), - ..Default::default() - }, - ) - .await - .map_err(|e| { - warn!(target: "engine", ?e, safe_l2 = %safe_l2, "Applying the delegated safe head failed"); - e - })?; - - info!( - target: "engine", - hash = %safe_l2.block_info.hash, - number = safe_l2.block_info.number, - "Updated safe head via follow safe" - ); - Ok(()) + self.reconcile_to_safe_head(state, safe_l2).await } } } @@ -160,39 +283,7 @@ impl ConsolidateTask { Ok(Some(block)) => block, Ok(None) => { if let ConsolidateInput::BlockInfo(safe_l2) = &self.input { - // A block missing below the unsafe head means the execution layer is still - // syncing toward the unsafe head. Resetting would restart that sync from the - // execution layer's old labels, and moving the unsafe head back would retarget - // it. Like op-node's follow-source mode, only the safe labels follow the - // delegate, and the forkchoice update, sent even when unchanged, carries them - // to the execution layer. - warn!( - target: "engine", - safe_l2 = %safe_l2, - unsafe_number = state.sync_state.unsafe_head().block_info.number, - "Execution layer does not have the delegated safe head yet; keeping its sync target" - ); - synchronize_unconditionally( - self.client.as_ref(), - state, - EngineSyncStateUpdate { - local_safe_head: Some(*safe_l2), - safe_head: Some(*safe_l2), - ..Default::default() - }, - ) - .await - .map_err(|e| { - warn!(target: "engine", ?e, safe_l2 = %safe_l2, "Applying the delegated safe head failed"); - e - })?; - info!( - target: "engine", - hash = %safe_l2.block_info.hash, - number = safe_l2.block_info.number, - "Updated safe head while the execution layer syncs" - ); - return Ok(()); + return self.follow_safe_head_during_el_sync(state, safe_l2).await; } warn!(target: "engine", "Received `None` block for {}", block_num); return Err(ConsolidateTaskError::MissingUnsafeL2Block(block_num)); @@ -278,6 +369,14 @@ impl ConsolidateTask { } } + // A different block from an execution layer that has not applied the engine's forkchoice + // may be from before its sync, so it does not prove the delegate diverged. + if let ConsolidateInput::BlockInfo(safe_l2) = &self.input && + !state.el_applied_last_forkchoice + { + return self.follow_safe_head_during_el_sync(state, safe_l2).await; + } + debug!( target: "engine", input = ?self.input, @@ -307,6 +406,19 @@ impl ExecutableEngineTask for ConsolidateTask { // injected safe head is ahead of the EngineActor's unsafe head, we reconcile the unsafe chain // up to the safe head instead of consolidating. async fn execute(&self, state: &mut EngineState) -> Result<(), ConsolidateTaskError> { + if let ConsolidateInput::BlockInfo(safe_l2) = &self.input { + state.sync_state.ensure_respects_finality(safe_l2.block_info.id())?; + // Labels with the unsafe head below the safe head take the paths below, which move the + // labels back to a consistent state; ignoring a lagging delegate would keep them broken + // until a finalization above the unsafe head fails. + let are_labels_consistent = state.sync_state.unsafe_head().block_info.number >= + state.sync_state.safe_head().block_info.number; + if are_labels_consistent && + safe_l2.block_info.number < state.sync_state.safe_head().block_info.number + { + return self.follow_delegated_safe_head_behind_safe_head(state, safe_l2).await; + } + } let safe_head_number = match &self.input { ConsolidateInput::Attributes { .. } => state.sync_state.safe_head().block_info.number, ConsolidateInput::BlockInfo(safe_block_info) => safe_block_info.block_info.number, @@ -323,7 +435,8 @@ impl ExecutableEngineTask for ConsolidateTask { mod tests { use super::*; use crate::{ - EngineSyncState, EngineTaskError, ImportedBlockSink, NoopBlockSink, SealTaskError, + EngineSyncState, EngineTaskError, FinalityViolation, ImportedBlockSink, NoopBlockSink, + SealTaskError, test_utils::{ TestAttributesBuilder, TestEngineStateBuilder, l1_info_deposit, test_block_info, test_engine_client, @@ -547,10 +660,9 @@ mod tests { el_has_block.then_some(block) } - /// The safe head before consolidation: either already the delegated target, as a `SYNCING` - /// answer or an earlier poll leaves it, or an older block. - fn initial_safe_head(starts_at_target: bool, target: L2BlockInfo) -> L2BlockInfo { - if starts_at_target { target } else { test_block_info(5) } + /// Either `target` itself or an older block, for a label that may already sit at the target. + fn target_or_older_block(is_at_target: bool, target: L2BlockInfo) -> L2BlockInfo { + if is_at_target { target } else { test_block_info(5) } } /// Below the unsafe head, a delegated safe head always reaches the execution layer: when the @@ -570,8 +682,10 @@ mod tests { ) { let (block, cfg) = genesis_pinned_block(10); let delegated_safe_head = block_info_of(&cfg, &block); - let mut state = - engine_state_with_safe_head(initial_safe_head(starts_at_target, delegated_safe_head)); + let mut state = engine_state_with_safe_head(target_or_older_block( + starts_at_target, + delegated_safe_head, + )); let expected_sync_state = with_safe_head(&state, delegated_safe_head); let (client, l1, l2) = test_engine_client(cfg.clone()); l2.expect_params( @@ -692,4 +806,326 @@ mod tests { l1.assert_finished(); l2.assert_finished(); } + + /// Where a delegated safe head sits against the finalized head. + #[derive(Clone, Copy, Debug)] + enum TargetAgainstFinalized { + Below, + ConflictingAt, + } + + impl TargetAgainstFinalized { + fn delegated_safe_head(self, finalized: L2BlockInfo) -> L2BlockInfo { + let number = finalized.block_info.number; + match self { + Self::Below => test_block_info(number - 1), + Self::ConflictingAt => L2BlockInfo { + block_info: BlockInfo { + hash: !finalized.block_info.hash, + ..finalized.block_info + }, + ..finalized + }, + } + } + } + + /// The outcome of checking a delegated safe head against finality. + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + enum FinalityCheck { + Accepted, + BehindFinalizedHead, + ConflictsWithFinalizedHead, + } + + impl FinalityCheck { + fn of(result: &Result<(), ConsolidateTaskError>) -> Self { + match result { + Ok(()) => Self::Accepted, + Err(ConsolidateTaskError::BreaksFinality( + FinalityViolation::BehindFinalizedHead { .. }, + )) => Self::BehindFinalizedHead, + Err(ConsolidateTaskError::BreaksFinality( + FinalityViolation::ConflictsWithFinalizedHead { .. }, + )) => Self::ConflictsWithFinalizedHead, + Err(err) => panic!("unexpected error {err:?}"), + } + } + } + + /// A delegated safe head that breaks finality, as a stale delegate snapshot reports, is skipped + /// before the execution layer is asked anything, and leaves every label where it was. + #[rstest::rstest] + #[case::below(TargetAgainstFinalized::Below, FinalityCheck::BehindFinalizedHead)] + #[case::conflicting( + TargetAgainstFinalized::ConflictingAt, + FinalityCheck::ConflictsWithFinalizedHead + )] + #[tokio::test] + async fn delegated_safe_head_breaking_finality_is_skipped( + #[case] target: TargetAgainstFinalized, + #[case] expected: FinalityCheck, + ) { + let cfg = Arc::new(RollupConfig::default()); + let mut state = engine_state_with_safe_head(test_block_info(12)); + let state_before = state; + let delegated_safe_head = target.delegated_safe_head(state.sync_state.finalized_head()); + let (client, l1, l2) = test_engine_client(cfg.clone()); + let task = ConsolidateTask::new( + Arc::new(client), + cfg, + ConsolidateInput::BlockInfo(delegated_safe_head), + Arc::new(NoopBlockSink), + ); + + let result = task.execute(&mut state).await; + + assert_eq!(FinalityCheck::of(&result), expected); + assert_eq!(result.unwrap_err().severity(), crate::EngineTaskErrorSeverity::Skip); + assert_eq!(state, state_before); + l1.assert_finished(); + l2.assert_finished(); + } + + /// A delegated safe head from a lagging delegate, between the finalized head, inclusive, and + /// the safe head, does not move the safe labels back, whether or not the execution layer has + /// its block, and sends no forkchoice update. + #[rstest::rstest] + #[case::el_has_block(true, false)] + #[case::el_missing_block(false, false)] + #[case::at_finalized_head(true, true)] + #[tokio::test] + async fn lagging_delegated_safe_head_leaves_labels_unchanged( + #[case] el_has_block: bool, + #[case] finalized_at_target: bool, + ) { + let (block, cfg) = genesis_pinned_block(10); + let delegated_safe_head = block_info_of(&cfg, &block); + let mut state = TestEngineStateBuilder::new() + .with_unsafe_head(test_block_info(20)) + .with_safe_head(test_block_info(12)) + .with_finalized_head(target_or_older_block(finalized_at_target, delegated_safe_head)) + .build(); + let state_before = state; + let (client, l1, l2) = test_engine_client(cfg.clone()); + l2.expect_block_header_by_number(10, el_block(el_has_block, block)); + let sink = Arc::new(RecordingSink::default()); + let task = ConsolidateTask::new( + Arc::new(client), + cfg, + ConsolidateInput::BlockInfo(delegated_safe_head), + sink.clone(), + ); + + task.execute(&mut state).await.unwrap(); + + assert_eq!(state, state_before); + assert!(sink.0.lock().unwrap().is_empty()); + l1.assert_finished(); + l2.assert_finished(); + } + + /// A delegated safe head behind the safe head whose block differs from the execution layer's + /// means the delegate's chain diverged above it, so the engine still reorgs onto it, without + /// moving finalized. + #[tokio::test] + async fn diverged_delegated_safe_head_behind_safe_head_reorgs_the_engine() { + let (block, cfg) = genesis_pinned_block(10); + let delegated_safe_head = block_info_of(&cfg, &block); + let mut state = engine_state_with_safe_head(test_block_info(12)); + let finalized_before = state.sync_state.finalized_head(); + let expected_sync_state = state.sync_state.apply_update(EngineSyncStateUpdate { + local_safe_head: Some(delegated_safe_head), + safe_head: Some(delegated_safe_head), + unsafe_head: Some(delegated_safe_head), + ..Default::default() + }); + let (client, l1, l2) = test_engine_client(cfg.clone()); + l2.expect_block_header_by_number(10, Some(other_chain_block(10))); + l2.expect_forkchoice_updated( + expected_sync_state.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + let task = ConsolidateTask::new( + Arc::new(client), + cfg, + ConsolidateInput::BlockInfo(delegated_safe_head), + Arc::new(NoopBlockSink), + ); + + task.execute(&mut state).await.unwrap(); + + assert_eq!(state.sync_state, expected_sync_state); + assert_eq!(state.sync_state.finalized_head(), finalized_before); + l1.assert_finished(); + l2.assert_finished(); + } + + /// A delegated safe head behind the safe head is not judged against an execution layer that has + /// not applied the engine's last forkchoice, whether it is in its first sync or syncing again + /// after answering `SYNCING`: its canonical block there may predate the engine's labels, so no + /// lookup is made and the labels stay. + #[rstest::rstest] + #[case::initial_sync(false, false)] + #[case::syncing_again(true, false)] + #[tokio::test] + async fn delegated_safe_head_behind_safe_head_is_ignored_while_the_el_syncs( + #[case] el_sync_finished: bool, + #[case] el_applied_last_forkchoice: bool, + ) { + let (block, cfg) = genesis_pinned_block(10); + let delegated_safe_head = block_info_of(&cfg, &block); + let mut state = TestEngineStateBuilder::new() + .with_unsafe_head(test_block_info(20)) + .with_safe_head(test_block_info(12)) + .with_finalized_head(test_block_info(5)) + .with_el_sync_finished(el_sync_finished) + .with_el_applied_last_forkchoice(el_applied_last_forkchoice) + .build(); + let state_before = state; + let (client, l1, l2) = test_engine_client(cfg.clone()); + let task = ConsolidateTask::new( + Arc::new(client), + cfg, + ConsolidateInput::BlockInfo(delegated_safe_head), + Arc::new(NoopBlockSink), + ); + + task.execute(&mut state).await.unwrap(); + + assert_eq!(state, state_before); + l1.assert_finished(); + l2.assert_finished(); + } + + /// An empty block at `number` whose hash differs from [`genesis_pinned_block`]'s at the same + /// height, standing for a block on another chain. + fn other_chain_block(number: u64) -> RpcBlock { + RpcBlock::new( + RpcHeader::new(alloy_consensus::Header { number, timestamp: 1, ..Default::default() }), + BlockTransactions::Full(vec![]), + ) + } + + /// A reorg onto a diverged delegated safe head that the execution layer refuses as an invalid + /// forkchoice state resets the engine, which rebuilds the labels from the execution layer. + #[tokio::test] + async fn rejected_reorg_onto_a_delegated_safe_head_resets_the_engine() { + let (block, cfg) = genesis_pinned_block(10); + let delegated_safe_head = block_info_of(&cfg, &block); + let mut state = engine_state_with_safe_head(test_block_info(12)); + let state_before = state; + let (client, l1, l2) = test_engine_client(cfg.clone()); + l2.expect_block_header_by_number(10, Some(other_chain_block(10))); + l2.expect_error_code( + "engine_forkchoiceUpdatedV3", + i64::from(alloy_rpc_types_engine::INVALID_FORK_CHOICE_STATE_ERROR), + ); + let task = ConsolidateTask::new( + Arc::new(client), + cfg, + ConsolidateInput::BlockInfo(delegated_safe_head), + Arc::new(NoopBlockSink), + ); + + let err = task.execute(&mut state).await.unwrap_err(); + + assert_eq!(err.severity(), crate::EngineTaskErrorSeverity::Reset, "got {err:?}"); + assert_eq!(state, state_before); + l1.assert_finished(); + l2.assert_finished(); + } + + /// Labels with the unsafe head rewound below the safe head are moved back into order by any + /// delegated safe head above finality instead of being kept: at or above the unsafe head the + /// engine reorgs onto it, and below it the matching block consolidates. Keeping them would let + /// a later finalization above the unsafe head fail fatally. + #[rstest::rstest] + #[case::at_or_above_unsafe_head(12, false)] + #[case::below_unsafe_head(8, true)] + #[tokio::test] + async fn delegated_safe_head_repairs_an_unsafe_head_below_the_safe_head( + #[case] delegated_number: u64, + #[case] is_below_unsafe_head: bool, + ) { + let (block, cfg) = genesis_pinned_block(delegated_number); + let delegated_safe_head = block_info_of(&cfg, &block); + let mut state = TestEngineStateBuilder::new() + .with_unsafe_head(test_block_info(10)) + .with_safe_head(test_block_info(15)) + .with_finalized_head(test_block_info(5)) + .build(); + let expected_sync_state = state.sync_state.apply_update(EngineSyncStateUpdate { + local_safe_head: Some(delegated_safe_head), + safe_head: Some(delegated_safe_head), + unsafe_head: (!is_below_unsafe_head).then_some(delegated_safe_head), + ..Default::default() + }); + let (client, l1, l2) = test_engine_client(cfg.clone()); + if is_below_unsafe_head { + l2.expect_params( + "eth_getBlockByNumber", + serde_json::json!([format!("{delegated_number:#x}"), true]), + block, + ); + } + l2.expect_forkchoice_updated( + expected_sync_state.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + let task = ConsolidateTask::new( + Arc::new(client), + cfg, + ConsolidateInput::BlockInfo(delegated_safe_head), + Arc::new(NoopBlockSink), + ); + + task.execute(&mut state).await.unwrap(); + + assert_eq!(state.sync_state, expected_sync_state); + assert!( + state.sync_state.unsafe_head().block_info.number >= + state.sync_state.safe_head().block_info.number + ); + l1.assert_finished(); + l2.assert_finished(); + } + + /// A delegated safe head whose block differs from the execution layer's is not treated as a + /// divergence while the execution layer has not applied the engine's last forkchoice: only the + /// safe labels move to it, and the unsafe head stays the sync target. + #[tokio::test] + async fn mismatched_delegated_safe_head_does_not_reorg_while_the_el_syncs() { + let (block, cfg) = genesis_pinned_block(15); + let delegated_safe_head = block_info_of(&cfg, &block); + let mut state = TestEngineStateBuilder::new() + .with_unsafe_head(test_block_info(20)) + .with_safe_head(test_block_info(12)) + .with_finalized_head(test_block_info(5)) + .with_el_applied_last_forkchoice(false) + .build(); + let expected_sync_state = with_safe_head(&state, delegated_safe_head); + let (client, l1, l2) = test_engine_client(cfg.clone()); + l2.expect_params( + "eth_getBlockByNumber", + serde_json::json!(["0xf", true]), + other_chain_block(15), + ); + l2.expect_forkchoice_updated( + expected_sync_state.create_forkchoice_state(), + PayloadStatusEnum::Syncing, + ); + let task = ConsolidateTask::new( + Arc::new(client), + cfg, + ConsolidateInput::BlockInfo(delegated_safe_head), + Arc::new(NoopBlockSink), + ); + + task.execute(&mut state).await.unwrap(); + + assert_eq!(state.sync_state, expected_sync_state); + l1.assert_finished(); + l2.assert_finished(); + } } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/error.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/error.rs index 92eb39d2e96..3e07f755697 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/error.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/error.rs @@ -1,7 +1,8 @@ //! Contains error types for the [`crate::FinalizeTask`]. use crate::{ - EngineTaskError, SynchronizeTaskError, task_queue::tasks::task::EngineTaskErrorSeverity, + EngineTaskError, FinalityViolation, SynchronizeTaskError, + task_queue::tasks::task::EngineTaskErrorSeverity, }; use alloy_eips::BlockNumHash; use alloy_transport::{RpcError, TransportErrorKind}; @@ -11,14 +12,9 @@ use thiserror::Error; /// An error that occurs when running the [`crate::FinalizeTask`]. #[derive(Debug, Error)] pub enum FinalizeTaskError { - /// The target is below the finalized head, and finality never moves backward. - #[error("Finalization target {target} is behind the finalized head {finalized}")] - BehindFinalizedHead { - /// The number of the finalized head. - finalized: u64, - /// The number of the finalization target. - target: u64, - }, + /// The target would move the finalized head backward, or onto another chain. + #[error("Finalization target breaks finality: {0}")] + BreaksFinality(#[from] FinalityViolation), /// The block to finalize was not found. #[error("The block to finalize was not found: Number {0}")] BlockNotFound(u64), @@ -40,14 +36,6 @@ pub enum FinalizeTaskError { /// The number of the block the engine returned. returned: u64, }, - /// The target is a different block at the finalized head's height. - #[error("Finalization target {target:?} conflicts with the finalized head {finalized:?}")] - ConflictsWithFinalizedHead { - /// The finalized head. - finalized: BlockNumHash, - /// The finalization target. - target: BlockNumHash, - }, /// The forkchoice update call to finalize the block failed. #[error(transparent)] ForkchoiceUpdateFailed(#[from] SynchronizeTaskError), @@ -78,9 +66,8 @@ pub enum FinalizeTaskError { impl EngineTaskError for FinalizeTaskError { fn severity(&self) -> EngineTaskErrorSeverity { match self { - Self::BehindFinalizedHead { .. } | Self::BlockNotSafe { .. } | - Self::ConflictsWithFinalizedHead { .. } | + Self::BreaksFinality(_) | Self::NotCanonical { .. } | Self::NotYetAvailable { .. } => EngineTaskErrorSeverity::Skip, Self::BlockNotFound(_) | Self::BlockNumberMismatch { .. } | Self::FromBlock(_) => { diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs index 30a08a102bd..d89e9c0c0b9 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs @@ -1,8 +1,8 @@ //! A task for finalizing an L2 block. use crate::{ - EngineClient, EngineState, ExecutableEngineTask, FinalizeBlockId, FinalizeTaskError, - state::EngineSyncStateUpdate, task_queue::synchronize, + EngineClient, EngineState, EngineSyncState, ExecutableEngineTask, FinalizeBlockId, + FinalizeTaskError, state::EngineSyncStateUpdate, task_queue::synchronize, }; use alloy_eips::BlockNumHash; use async_trait::async_trait; @@ -42,19 +42,14 @@ impl FinalizeTask { /// Whether `target` is the finalized head itself; a different block at its height is an error. fn is_already_finalized( - finalized_head: &L2BlockInfo, + sync_state: &EngineSyncState, target: BlockNumHash, ) -> Result { - let finalized = finalized_head.block_info.id(); - if target == finalized { + if target == sync_state.finalized_head().block_info.id() { debug!(target: "engine", ?target, "Target is already finalized"); return Ok(true); } - // The default head means finality has not been established yet, so any target, including - // the configured genesis, may initialize it. - if *finalized_head != L2BlockInfo::default() && target.number == finalized.number { - return Err(FinalizeTaskError::ConflictsWithFinalizedHead { finalized, target }); - } + sync_state.ensure_no_conflict_with_finality(target)?; Ok(false) } @@ -101,19 +96,13 @@ impl ExecutableEngineTask for FinalizeTask { type Error = FinalizeTaskError; async fn execute(&self, state: &mut EngineState) -> Result<(), FinalizeTaskError> { - let finalized_head = state.sync_state.finalized_head(); let target_number = self.block_id.number(); // A lagging source can report an older finalized block, or one from another chain. A // pinned hash is enough to detect either without asking the execution layer. - if target_number < finalized_head.block_info.number { - return Err(FinalizeTaskError::BehindFinalizedHead { - finalized: finalized_head.block_info.number, - target: target_number, - }); - } + state.sync_state.ensure_not_behind_finality(target_number)?; if let FinalizeBlockId::ByHash(target) = self.block_id && - Self::is_already_finalized(&finalized_head, target)? + Self::is_already_finalized(&state.sync_state, target)? { return Ok(()); } @@ -132,7 +121,7 @@ impl ExecutableEngineTask for FinalizeTask { // A by-number target only learns its hash from the lookup. if matches!(self.block_id, FinalizeBlockId::ByNumber(_)) && - Self::is_already_finalized(&finalized_head, block_info.block_info.id())? + Self::is_already_finalized(&state.sync_state, block_info.block_info.id())? { return Ok(()); } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs index 45420631095..5708a52c150 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs @@ -1,7 +1,7 @@ use crate::{ EngineState, EngineSyncState, EngineSyncStateUpdate, EngineTaskError, EngineTaskErrorSeverity::{self, Critical, Skip, Temporary}, - ExecutableEngineTask, FinalizeBlockId, FinalizeTask, FinalizeTaskError, + ExecutableEngineTask, FinalityViolation, FinalizeBlockId, FinalizeTask, FinalizeTaskError, test_utils::{ RpcMock, TestEngineStateBuilder, TestRpcBlock as RpcBlock, test_block_info, test_engine_client, test_l2_block, test_rpc_block, @@ -103,11 +103,20 @@ async fn finalize( } const fn is_behind(err: &FinalizeTaskError) -> bool { - matches!(err, FinalizeTaskError::BehindFinalizedHead { finalized: FINALIZED, target: STALE }) + matches!( + err, + FinalizeTaskError::BreaksFinality(FinalityViolation::BehindFinalizedHead { + finalized: BlockNumHash { number: FINALIZED, .. }, + target: STALE, + }) + ) } const fn is_conflict(err: &FinalizeTaskError) -> bool { - matches!(err, FinalizeTaskError::ConflictsWithFinalizedHead { .. }) + matches!( + err, + FinalizeTaskError::BreaksFinality(FinalityViolation::ConflictsWithFinalizedHead { .. }) + ) } const fn is_not_canonical(err: &FinalizeTaskError) -> bool { diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs index a9bc15146ff..fc28ccf0026 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs @@ -100,6 +100,7 @@ async fn send_forkchoice( match &valid_response.payload_status.status { PayloadStatusEnum::Valid => { + state.el_applied_last_forkchoice = true; if !state.el_sync_finished { info!(target: "engine", "Finished execution layer sync."); state.el_sync_finished = true; @@ -107,6 +108,7 @@ async fn send_forkchoice( } // If we're not building a new payload, we're driving EL sync. PayloadStatusEnum::Syncing => { + state.el_applied_last_forkchoice = false; debug!(target: "engine", "Attempting to update forkchoice state while EL syncing") } // Other codes are not expected. diff --git a/rust/kona/crates/node/engine/src/test_utils/engine_state.rs b/rust/kona/crates/node/engine/src/test_utils/engine_state.rs index 60542823b96..1ed3c5b8178 100644 --- a/rust/kona/crates/node/engine/src/test_utils/engine_state.rs +++ b/rust/kona/crates/node/engine/src/test_utils/engine_state.rs @@ -10,6 +10,7 @@ pub struct TestEngineStateBuilder { local_safe_head: Option, safe_head: Option, finalized_head: Option, + el_applied_last_forkchoice: bool, el_sync_finished: bool, } @@ -33,6 +34,7 @@ impl TestEngineStateBuilder { local_safe_head: None, safe_head: None, finalized_head: None, + el_applied_last_forkchoice: true, el_sync_finished: true, } } @@ -55,13 +57,22 @@ impl TestEngineStateBuilder { self } - /// Sets whether EL sync is finished + /// Sets whether EL sync is finished. An execution layer still syncing has not applied the last + /// forkchoice either, so this sets both flags. #[allow(dead_code)] pub const fn with_el_sync_finished(mut self, finished: bool) -> Self { + self.el_applied_last_forkchoice = finished; self.el_sync_finished = finished; self } + /// Sets whether the execution layer applied the last forkchoice update. + #[allow(dead_code)] + pub const fn with_el_applied_last_forkchoice(mut self, applied: bool) -> Self { + self.el_applied_last_forkchoice = applied; + self + } + /// Builds the `EngineState` pub fn build(self) -> EngineState { let mut state = EngineState::default(); @@ -74,6 +85,7 @@ impl TestEngineStateBuilder { finalized_head: Some(self.finalized_head.unwrap_or(self.unsafe_head)), }); + state.el_applied_last_forkchoice = self.el_applied_last_forkchoice; state.el_sync_finished = self.el_sync_finished; state } diff --git a/rust/kona/crates/node/engine/src/test_utils/rpc.rs b/rust/kona/crates/node/engine/src/test_utils/rpc.rs index 0bd0bfef0c6..cda70c224a6 100644 --- a/rust/kona/crates/node/engine/src/test_utils/rpc.rs +++ b/rust/kona/crates/node/engine/src/test_utils/rpc.rs @@ -70,6 +70,16 @@ impl RpcMock { ); } + /// Expects a header-only `eth_getBlockByNumber` call for `number` and answers it with `block`, + /// or with null for `None`. + pub fn expect_block_header_by_number(&self, number: u64, block: Option) { + self.expect_params( + "eth_getBlockByNumber", + serde_json::json!([format!("{number:#x}"), false]), + block, + ); + } + /// Expects an `engine_forkchoiceUpdatedV3` call for `forkchoice` without payload attributes and /// answers it with `status`. pub fn expect_forkchoice_updated( diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs index 480019925a0..37dc0e00185 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs @@ -6,7 +6,7 @@ use alloy_primitives::BlockHash; use async_trait::async_trait; use jsonrpsee::core::ClientError; use kona_derive::{ChainProvider, Signal}; -use kona_engine::{EngineState, FinalizeBlockId}; +use kona_engine::{EngineState, FinalityViolation, FinalizeBlockId}; use kona_protocol::{BlockInfo, L2BlockInfo, SyncStatus}; use kona_rpc::RollupNodeApiClient; use thiserror::Error; @@ -77,6 +77,15 @@ where has_engine_sync_completed: bool, /// Ticker driving periodic polls of the derivation delegate. delegated_derivation_ticker: time::Interval, + + /// A safe head behind the engine's that was forwarded while the execution layer applied the + /// engine's forkchoice, with the engine's safe head at the time. The engine checks such a head + /// against its execution layer once; until either changes, sending it again repeats that + /// lookup to the same answer. + forwarded_lagging_safe_head: Option<(BlockNumHash, BlockNumHash)>, + /// The last rejection of a delegate status, so a delegate stuck on one inconsistency is + /// warned about once rather than on every poll. + last_rejection: Option, } impl @@ -107,6 +116,8 @@ where engine_l2_safe_head: L2BlockInfo::default(), has_engine_sync_completed: false, delegated_derivation_ticker, + forwarded_lagging_safe_head: None, + last_rejection: None, } } } @@ -175,58 +186,44 @@ where Ok(()) } - /// The id of an established finalized head. The default head means finality has not been - /// established yet, so any finalized block from the delegate may initialize it. - fn established_finalized_l2(finalized_head: L2BlockInfo) -> Option { - (finalized_head != L2BlockInfo::default()).then(|| finalized_head.block_info.id()) + /// Rejects a status whose labels are out of order: its finalized block ahead of its safe head, + /// or its safe head ahead of its local safe head. No consistent delegate reports either, and a + /// finalized block ahead of the safe head would let a stale safe head sit below finality. A + /// zero local safe head is a label the delegate has not derived or does not report, so it does + /// not order anything. + fn validate_label_order(status: &SyncStatus) -> Result<(), DerivationDelegationError> { + let finalized = status.finalized_l2.block_info.number; + let safe = status.safe_l2.block_info.number; + let local_safe = if status.local_safe_l2 == L2BlockInfo::default() { + safe + } else { + status.local_safe_l2.block_info.number + }; + if finalized > safe { + return Err(DerivationDelegationError::FinalizedAheadOfSafe { finalized, safe }); + } + if safe > local_safe { + return Err(DerivationDelegationError::SafeAheadOfLocalSafe { local_safe, safe }); + } + Ok(()) } - /// Rejects a finalized L2 block behind, or at the same height as but different from, the - /// engine's. Either means the delegate lags or follows another chain, so none of its status - /// applies: its safe head would move the engine back or onto that chain too. + /// Verifies that the labels reported by the derivation delegate are in order, that its + /// finalized L2 block does not move back from the engine's, and that its L1 info is consistent + /// with the canonical L1 chain. /// /// Only a fork at the engine's finalized height is caught here. A chain that forked below it /// and reports a higher finalized block passes, and the execution layer rejects the forkchoice /// update that would move onto it. - fn validate_finalized_l2_progress( - &self, - reported: BlockNumHash, + async fn validate_sync_status( + &mut self, + v: &SyncStatus, ) -> Result<(), DerivationDelegationError> { - let engine_finalized_head = self.engine_state_rx.borrow().sync_state.finalized_head(); - let Some(engine) = Self::established_finalized_l2(engine_finalized_head) else { - return Ok(()); - }; - if reported.number < engine.number { - return Err(DerivationDelegationError::FinalizedL2Regressed { engine, reported }); - } - if reported.number == engine.number && reported.hash != engine.hash { - return Err(DerivationDelegationError::FinalizedL2Conflict { engine, reported }); - } - Ok(()) - } - - /// Verifies that the derivation delegate's finalized L2 block is not above its safe L2 block - /// and does not move back from the engine's, and that its L1 info is consistent with the - /// canonical L1 chain. - async fn validate_sync_status(&mut self, v: &SyncStatus) -> bool { - // A finalized block above the safe head is an inconsistent snapshot; the engine would only - // refuse to finalize it. - if v.finalized_l2.block_info.number > v.safe_l2.block_info.number { - let err = DerivationDelegationError::FinalizedAheadOfSafe { - finalized: v.finalized_l2.block_info.number, - safe: v.safe_l2.block_info.number, - }; - warn!(target: "derivation", error = %err, "Inconsistent sync status from delegate"); - return false; - } - if let Err(err) = self.validate_finalized_l2_progress(v.finalized_l2.block_info.id()) { - warn!( - target: "derivation", - error = %err, - "Finalized L2 inconsistency detected at sync status from delegate" - ); - return false; - } + Self::validate_label_order(v)?; + self.engine_state_rx + .borrow() + .sync_state + .ensure_respects_finality(v.finalized_l2.block_info.id())?; let checks = [ ("L1 Origin of Safe L2", v.safe_l2.l1_origin.number, v.safe_l2.l1_origin.hash), @@ -237,18 +234,52 @@ where ), ("Current L1", v.current_l1.number, v.current_l1.hash), ]; + let mut validated_l1_blocks = Vec::with_capacity(checks.len()); for (context, number, hash) in checks { - if let Err(err) = self.validate_l1_block(context, number, hash).await { - warn!( - target: "derivation", - context = context, - error = %err, - "L1 inconsistency detected at sync status from delegate" - ); - return false; + // Labels often share an L1 origin, and one lookup proves it canonical for all of them. + if validated_l1_blocks.contains(&(number, hash)) { + continue; } + self.validate_l1_block(context, number, hash).await?; + validated_l1_blocks.push((number, hash)); + } + Ok(()) + } + + /// Logs a rejected delegate status at `warn!` when the reason changed, and at `debug!` while + /// the delegate keeps reporting the same inconsistency. + fn log_rejected_status(&mut self, err: &DerivationDelegationError) { + let reason = err.to_string(); + if self.last_rejection.as_ref() == Some(&reason) { + debug!(target: "derivation", error = %err, "Sync status from delegate rejected again"); + } else { + warn!(target: "derivation", error = %err, "Rejected sync status from delegate"); + self.last_rejection = Some(reason); } - true + } + + /// Whether the delegated safe head is the lagging one already forwarded against the same + /// engine safe head, so the engine would repeat its lookup to the same answer. + fn is_lagging_safe_head_forwarded(&self, safe_l2: &L2BlockInfo) -> bool { + self.lagging_safe_head_to_remember(safe_l2) == self.forwarded_lagging_safe_head && + self.forwarded_lagging_safe_head.is_some() + } + + /// The pair to remember for a delegated safe head behind the engine's consistent safe head + /// while its execution layer applied the engine's forkchoice, or `None` for any other head, + /// which the engine must always see. + fn lagging_safe_head_to_remember( + &self, + safe_l2: &L2BlockInfo, + ) -> Option<(BlockNumHash, BlockNumHash)> { + let engine_state = self.engine_state_rx.borrow(); + let engine_safe_head = engine_state.sync_state.safe_head().block_info; + let is_consistent = + engine_state.sync_state.unsafe_head().block_info.number >= engine_safe_head.number; + (engine_state.el_applied_last_forkchoice && + is_consistent && + safe_l2.block_info.number < engine_safe_head.number) + .then(|| (safe_l2.block_info.id(), engine_safe_head.id())) } /// Fetches, validates, and applies sync status from the derivation delegate. @@ -265,18 +296,30 @@ where } }; - if !self.validate_sync_status(&sync_status).await { + if let Err(err) = self.validate_sync_status(&sync_status).await { // Validation failures here are expected to be transient, so we skip processing // this sync status and continue delegating derivation instead of treating it as // fatal. + self.log_rejected_status(&err); return Ok(()); } + self.last_rejection = None; self.derivation_origin_tx.send_replace(Some(sync_status.current_l1)); - self.engine_client - .send_safe_l2_signal(sync_status.safe_l2.into()) - .await - .map_err(|e| DerivationError::Sender(Box::new(e)))?; + if self.is_lagging_safe_head_forwarded(&sync_status.safe_l2) { + debug!( + target: "derivation", + safe_l2 = ?sync_status.safe_l2, + "Not forwarding a lagging safe head the engine already checked" + ); + } else { + self.forwarded_lagging_safe_head = + self.lagging_safe_head_to_remember(&sync_status.safe_l2); + self.engine_client + .send_safe_l2_signal(sync_status.safe_l2.into()) + .await + .map_err(|e| DerivationError::Sender(Box::new(e)))?; + } // Delegated polling supplies `(number, hash)`. Carry the hash through so the engine // finalizes the specific block we were asked to, not whatever it happens to have at the @@ -329,17 +372,14 @@ where #[derive(Error, Debug)] enum DerivationDelegationError { - /// The delegate's finalized L2 block is above its safe L2 block. + /// The delegate's finalized L2 block is ahead of its own safe head. #[error("Finalized L2 {finalized} is ahead of safe L2 {safe}")] FinalizedAheadOfSafe { finalized: u64, safe: u64 }, - /// The delegate's finalized L2 block is a different block at the engine's finalized height. - #[error("Finalized L2 {reported:?} conflicts with the engine's finalized L2 {engine:?}")] - FinalizedL2Conflict { engine: BlockNumHash, reported: BlockNumHash }, - - /// The delegate's finalized L2 block is behind the engine's. - #[error("Finalized L2 {reported:?} is behind the engine's finalized L2 {engine:?}")] - FinalizedL2Regressed { engine: BlockNumHash, reported: BlockNumHash }, + /// The delegate's finalized L2 block is behind the engine's, or a different block at its + /// height. + #[error("Finalized L2 moves the engine's finality back: {0}")] + FinalizedL2BreaksFinality(#[from] FinalityViolation), /// The L1 provider returned an error (network, RPC, etc.) #[error("L1 provider error: {0}")] @@ -348,4 +388,8 @@ enum DerivationDelegationError { /// The hash provided by the derivation delegation does not match the canonical chain. #[error("L1 inconsistency in {context} at block {number}: expected {expected}, got {actual}")] L1ValidationFailed { context: String, number: u64, expected: BlockHash, actual: BlockHash }, + + /// The delegate's safe head is ahead of its own local safe head. + #[error("Safe L2 {safe} is ahead of local safe L2 {local_safe}")] + SafeAheadOfLocalSafe { local_safe: u64, safe: u64 }, } diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs index fea52b068d1..b586f878488 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs @@ -88,6 +88,15 @@ impl Harness { self.engine_state.send_modify(|state| state.sync_state = sync_state); } + /// Publishes the engine's labels along with whether its execution layer applied the engine's + /// last forkchoice. + fn publish_engine_state(&self, sync_state: EngineSyncState, el_applied_last_forkchoice: bool) { + self.engine_state.send_modify(|state| { + state.sync_state = sync_state; + state.el_applied_last_forkchoice = el_applied_last_forkchoice; + }); + } + /// The engine requests sent since the last call, in order. fn forwarded(&mut self) -> Vec { std::iter::from_fn(|| self.engine_requests.try_recv().ok()).map(Forwarded::from).collect() @@ -367,3 +376,92 @@ async fn poll_rejects_status_whose_finalized_is_ahead_of_its_safe( assert_eq!(harness.forwarded(), forwarded_if(accepted, &status)); } + +/// A status whose labels are out of order is rejected whole: a finalized block ahead of the safe +/// head would let a stale snapshot's safe head sit below finality, and a safe head ahead of the +/// local safe head is not a state any delegate reaches. +#[rstest] +#[case::ordered(15, 20, 20, true)] +#[case::finalized_at_safe(20, 20, 20, true)] +#[case::safe_behind_local_safe(15, 18, 20, true)] +#[case::finalized_ahead_of_safe(16, 15, 20, false)] +#[case::safe_ahead_of_local_safe(15, 21, 20, false)] +#[tokio::test(start_paused = true)] +async fn poll_rejects_status_with_labels_out_of_order( + #[case] finalized: u64, + #[case] safe: u64, + #[case] local_safe: u64, + #[case] accepted: bool, +) { + let status = SyncStatus { + finalized_l2: l2_block(finalized, FINALIZED_ORIGIN), + local_safe_l2: l2_block(local_safe, SAFE_ORIGIN), + safe_l2: l2_block(safe, SAFE_ORIGIN), + ..delegate_status() + }; + let mut harness = Harness::new([Ok(status.clone())]); + harness.complete_engine_sync().await; + + harness.actor.step().await.unwrap(); + + assert_eq!(harness.forwarded(), forwarded_if(accepted, &status)); + assert_eq!(harness.current_l1.borrow().is_some(), accepted); +} + +/// A zero local safe head is a label the delegate has not derived or does not report, so it does +/// not put the safe head out of order and the status is forwarded. +#[tokio::test(start_paused = true)] +async fn unreported_local_safe_head_does_not_order_the_labels() { + let status = SyncStatus { local_safe_l2: L2BlockInfo::default(), ..delegate_status() }; + let mut harness = Harness::new([Ok(status.clone())]); + harness.complete_engine_sync().await; + + harness.actor.step().await.unwrap(); + + assert_eq!(harness.forwarded(), forwarded_for(&status)); +} + +/// An engine sync state with consistent labels: unsafe at 30, safe at `safe`, finalized at 15. +fn engine_safe_at(safe: u64) -> EngineSyncState { + EngineSyncState::default().apply_update(EngineSyncStateUpdate { + finalized_head: Some(l2_block(15, FINALIZED_ORIGIN)), + local_safe_head: Some(l2_block(safe, SAFE_ORIGIN)), + safe_head: Some(l2_block(safe, SAFE_ORIGIN)), + unsafe_head: Some(l2_block(30, SAFE_ORIGIN)), + }) +} + +/// What a poll of `status` forwards when its safe head is held back: the finalized block alone. +fn finalized_only(status: &SyncStatus) -> Vec { + vec![Forwarded::Finalized(FinalizeBlockId::ByHash(status.finalized_l2.block_info.id()))] +} + +/// A safe head behind the engine's is forwarded once while the execution layer applied the +/// engine's forkchoice, since the engine's answer to it cannot change; finalized still goes out +/// on every poll. A moved engine safe head, or an execution layer that did not apply the last +/// forkchoice, sends it again. +#[rstest] +#[case::same_engine_view(true, 25, finalized_only)] +#[case::el_not_applied(false, 25, forwarded_for)] +#[case::engine_safe_moved(true, 26, forwarded_for)] +#[tokio::test(start_paused = true)] +async fn lagging_safe_head_is_forwarded_once_per_engine_view( + #[case] el_applied_last_forkchoice: bool, + #[case] engine_safe_on_second_poll: u64, + #[case] second_poll_forwards: fn(&SyncStatus) -> Vec, +) { + let status = delegate_status(); + let mut harness = Harness::new([Ok(status.clone()), Ok(status.clone())]); + harness.complete_engine_sync_with(engine_safe_at(25)).await; + harness.publish_engine_state(engine_safe_at(25), el_applied_last_forkchoice); + harness.actor.step().await.unwrap(); + assert_eq!(harness.forwarded(), forwarded_for(&status)); + harness.publish_engine_state( + engine_safe_at(engine_safe_on_second_poll), + el_applied_last_forkchoice, + ); + + harness.actor.step().await.unwrap(); + + assert_eq!(harness.forwarded(), second_poll_forwards(&status)); +} diff --git a/rust/kona/crates/node/service/src/actors/engine/actor.rs b/rust/kona/crates/node/service/src/actors/engine/actor.rs index f09047db8a2..b45bfa62238 100644 --- a/rust/kona/crates/node/service/src/actors/engine/actor.rs +++ b/rust/kona/crates/node/service/src/actors/engine/actor.rs @@ -235,7 +235,7 @@ where self.send_derivation_actor_safe_head_if_updated().await?; if !self.el_sync_complete && self.engine.state().el_sync_finished { - if self.engine.state().sync_state.finalized_head() == L2BlockInfo::default() { + if self.engine.state().sync_state.established_finalized_head().is_none() { self.begin_reset(); } else { self.mark_el_sync_complete_and_notify_derivation_actor().await?;