diff --git a/docs/public-docs/node-operators/kona-node/configuration.mdx b/docs/public-docs/node-operators/kona-node/configuration.mdx index 1b4adb066f9..a7bb2cc70f8 100644 --- a/docs/public-docs/node-operators/kona-node/configuration.mdx +++ b/docs/public-docs/node-operators/kona-node/configuration.mdx @@ -271,6 +271,16 @@ Before interop both heads are the same block. Under interop, `kona-node` verifies no cross-chain dependencies itself, so its safe label is only as safe as the source's cross-safe head: the source must verify them. +If the execution client rejects a forkchoice update that moved the head to the source's heads, for +example because the source's chain does not descend from the node's finalized block, `kona-node` +sends its own last forkchoice again to undo the rejected head and does not send those heads again +until its next reset. A source whose chain does not descend from the node's finalized block is held +back as a whole: its heads are sent again once the node's finality moves, which only a canonical +finalized block from the source can cause, and otherwise once every five minutes. It resets +instead when the execution client no longer holds the node's finalized block, or does not apply the +node's own forkchoice. A rejected update that moved only the safe label is retried on the next +poll. + ## RPC Trust Flags The `--l1-trust-rpc` and `--l2-trust-rpc` flags control whether Kona verifies the block hashes of RPC responses. For guidance on when to disable trust and worked examples for trusted, untrusted, and mixed provider setups, see [configure RPC trust](/node-operators/kona-node/run/rpc-trust). diff --git a/rust/kona/crates/node/engine/src/client.rs b/rust/kona/crates/node/engine/src/client.rs index d1423fbc9d4..408c6c50975 100644 --- a/rust/kona/crates/node/engine/src/client.rs +++ b/rust/kona/crates/node/engine/src/client.rs @@ -4,12 +4,14 @@ use crate::Metrics; use alloy_eips::{BlockId, eip1898::BlockNumberOrTag}; use alloy_json_rpc::{RequestPacket, ResponsePacket}; use alloy_network::{Ethereum, Network}; -use alloy_primitives::Bytes; +use alloy_primitives::{B256, Bytes}; use alloy_provider::{EthGetBlock, Provider, RootProvider}; use alloy_rpc_client::ClientBuilder; use alloy_rpc_types_engine::JwtSecret; use alloy_rpc_types_eth::Block; -use alloy_transport::{RpcError, TransportError, TransportErrorKind, TransportFut}; +use alloy_transport::{ + RpcError, TransportError, TransportErrorKind, TransportFut, TransportResult, +}; use alloy_transport_http::{ AuthLayer, Http, HyperClient, hyper_util::{client::legacy::Client, rt::TokioExecutor}, @@ -198,6 +200,17 @@ impl EngineClient { self.engine.get_block(block) } + /// Whether the execution layer stores the L2 block with `hash`, canonical or not. Only the + /// block's presence is needed, so it is fetched without its transactions. + pub async fn l2_block_is_stored(&self, hash: B256) -> TransportResult { + debug!(target: "engine", %hash, "Looking up the execution layer's block by hash"); + let is_stored = self.get_l2_block(hash.into()).await.inspect_err(|err| { + warn!(target: "engine", ?err, %hash, "Failed to look up the execution layer's block by hash"); + })?.is_some(); + debug!(target: "engine", %hash, is_stored, "Looked up the execution layer's block by hash"); + Ok(is_stored) + } + /// Fetches an L2 block with full transactions. pub async fn l2_block_by_label( &self, diff --git a/rust/kona/crates/node/engine/src/lib.rs b/rust/kona/crates/node/engine/src/lib.rs index 1f64ad82ac1..4bac1935c99 100644 --- a/rust/kona/crates/node/engine/src/lib.rs +++ b/rust/kona/crates/node/engine/src/lib.rs @@ -40,10 +40,10 @@ extern crate tracing; mod task_queue; pub use task_queue::{ BuildTask, BuildTaskError, CanonicalizeTask, CanonicalizeTaskError, ConsolidateInput, - ConsolidateTask, ConsolidateTaskError, DelegatedSafeHeads, Engine, EngineBuildError, - EngineResetError, EngineTask, EngineTaskError, EngineTaskErrorSeverity, EngineTaskErrors, - ExecutableEngineTask, FinalizeBlockId, FinalizeTask, FinalizeTaskError, - InconsistentSafeHeadsError, InsertTask, InsertTaskError, SealTask, SealTaskError, + ConsolidateTask, ConsolidateTaskError, DelegatedRejection, DelegatedSafeHeads, Engine, + EngineBuildError, EngineResetError, EngineTask, EngineTaskError, EngineTaskErrorSeverity, + EngineTaskErrors, ExecutableEngineTask, FinalizeBlockId, FinalizeTask, FinalizeTaskError, + InconsistentSafeHeadsError, InsertTask, InsertTaskError, SafeLabel, SealTask, SealTaskError, SynchronizeTaskError, }; diff --git a/rust/kona/crates/node/engine/src/state/core.rs b/rust/kona/crates/node/engine/src/state/core.rs index 67204d7d81f..591efccf6a8 100644 --- a/rust/kona/crates/node/engine/src/state/core.rs +++ b/rust/kona/crates/node/engine/src/state/core.rs @@ -77,7 +77,8 @@ impl EngineSyncState { } /// Applies the update to the provided sync state, using the current state values if the update - /// is not specified. Returns the new sync state. + /// is not specified, and records the updated labels in the block label metrics. Returns the + /// new sync state. pub fn apply_update(self, sync_state_update: EngineSyncStateUpdate) -> Self { if let Some(unsafe_head) = sync_state_update.unsafe_head { Self::update_block_label_metric( @@ -101,6 +102,12 @@ impl EngineSyncState { ); } + self.with_update(sync_state_update) + } + + /// The sync state with the update applied, using the current state values if the update is + /// not specified, without recording anything in the block label metrics. + pub fn with_update(self, sync_state_update: EngineSyncStateUpdate) -> Self { Self { unsafe_head: sync_state_update.unsafe_head.unwrap_or(self.unsafe_head), local_safe_head: sync_state_update.local_safe_head.unwrap_or(self.local_safe_head), diff --git a/rust/kona/crates/node/engine/src/sync/error.rs b/rust/kona/crates/node/engine/src/sync/error.rs index d98b69d642b..f46c7ef6a5a 100644 --- a/rust/kona/crates/node/engine/src/sync/error.rs +++ b/rust/kona/crates/node/engine/src/sync/error.rs @@ -1,6 +1,6 @@ //! Contains the error types used for finding the starting forkchoice state. -use alloy_eips::BlockId; +use alloy_eips::{BlockId, BlockNumHash}; use alloy_primitives::B256; use alloy_transport::{RpcError, TransportErrorKind}; use kona_protocol::FromBlockError; @@ -26,9 +26,45 @@ pub enum SyncStartError { /// Invalid L2 genesis hash. #[error("Invalid L2 genesis hash. Expected {0}, Got {1}")] InvalidL2GenesisHash(B256, B256), - /// Finalized block mismatch - #[error("Finalized block mismatch. Expected {0}, Got {1}")] - MismatchedFinalizedBlock(B256, B256), + /// The L1 provider holds a different block at the height of the L1 origin of the execution + /// layer's finalized block, so it follows another L1 chain. + #[error("L1 provider has {l1_block:?} at the height of finalized L1 origin {l1_origin:?}")] + FinalizedL1OriginNotCanonical { + /// The L1 provider's block at that height. + l1_block: BlockNumHash, + /// The finalized block's L1 origin. + l1_origin: BlockNumHash, + }, + /// The L1 provider serves the L1 origin of the execution layer's finalized block by number + /// but not by hash, as a broken or lagging block-hash index does. + #[error("L1 provider serves finalized L1 origin {l1_origin:?} by number but not by hash")] + FinalizedL1OriginNotServedByHash { + /// The L1 origin it serves only by number. + l1_origin: BlockNumHash, + }, + /// The L1 provider does not have the L1 origin of the execution layer's finalized block. + #[error("L1 origin {l1_origin:?} of finalized L2 block {l2_block:?} is not available")] + FinalizedL1OriginUnavailable { + /// The missing L1 origin. + l1_origin: BlockNumHash, + /// The finalized L2 block. + l2_block: BlockNumHash, + }, + /// The execution layer returned no block at its finalized height, although its canonical chain + /// extends above it, as a briefly inconsistent execution layer does. + #[error("Execution layer has no block at finalized height {0}")] + FinalizedHeightUnavailable(u64), + /// The chain walked from the execution layer's unsafe head does not contain its finalized + /// block. + #[error( + "Walked chain reached finalized height on {walked:?}, not finalized block {finalized:?}" + )] + MismatchedFinalizedBlock { + /// The execution layer's finalized block. + finalized: BlockNumHash, + /// The walked block at or below the finalized height. + walked: BlockNumHash, + }, /// L1 origin mismatch. #[error("L1 origin mismatch")] L1OriginMismatch, diff --git a/rust/kona/crates/node/engine/src/sync/mod.rs b/rust/kona/crates/node/engine/src/sync/mod.rs index cdf468ba554..2bc931bfb7f 100644 --- a/rust/kona/crates/node/engine/src/sync/mod.rs +++ b/rust/kona/crates/node/engine/src/sync/mod.rs @@ -1,5 +1,6 @@ //! Sync start algorithm for the OP Stack rollup node. +use alloy_eips::BlockNumHash; use kona_genesis::RollupConfig; use kona_protocol::L2BlockInfo; @@ -9,10 +10,23 @@ pub use forkchoice::L2ForkchoiceState; mod error; pub use error::SyncStartError; -use tracing::info; +use tracing::{error, info}; use crate::EngineClient; +/// A block known only by its number and hash, which is all a forkchoice update or a finality +/// comparison needs of it. +pub(crate) fn l2_block_info_at(id: BlockNumHash) -> L2BlockInfo { + L2BlockInfo { + block_info: kona_protocol::BlockInfo { + hash: id.hash, + number: id.number, + ..Default::default() + }, + ..Default::default() + } +} + /// Searches for the latest [`L2ForkchoiceState`] that we can use to start the sync process with. /// /// - The *unsafe L2 block*: This is the highest L2 block whose L1 origin is a *plausible* @@ -60,6 +74,23 @@ pub async fn find_starting_forkchoice( break; } None => { + // At or below the finalized height, only the finalized block itself proves the + // L1 provider is behind: its L1 data is finalized and cannot reorg. Any other + // block there is on a fork from below finality, and walking further back would + // move finality back. + if current_fc.un_safe.block_info.number <= current_fc.finalized.block_info.number { + if current_fc.un_safe.block_info.hash != current_fc.finalized.block_info.hash { + return Err(finalized_block_not_on_walked_chain( + ¤t_fc.finalized, + ¤t_fc.un_safe, + )); + } + return Err(finalized_l1_origin_missing_by_hash( + engine_client, + ¤t_fc.un_safe, + ) + .await); + } let l2_parent_hash = current_fc.un_safe.block_info.parent_hash.into(); let l2_parent = engine_client .get_l2_block(l2_parent_hash) @@ -88,6 +119,10 @@ pub async fn find_starting_forkchoice( current_fc.un_safe.l1_origin.number.saturating_sub(cfg.seq_window_size) > safe_cursor.l1_origin.number; let is_finalized = safe_cursor.block_info.hash == current_fc.finalized.block_info.hash; + if !is_finalized && safe_cursor.block_info.number <= current_fc.finalized.block_info.number + { + return Err(finalized_block_not_on_walked_chain(¤t_fc.finalized, &safe_cursor)); + } let is_genesis = safe_cursor.block_info.hash == cfg.genesis.l2.hash; if is_behind_sequence_window || is_finalized || is_genesis { info!( @@ -109,10 +144,268 @@ pub async fn find_starting_forkchoice( safe_cursor = L2BlockInfo::from_block_and_genesis(&block.into_consensus(), &cfg.genesis)?; } + ensure_finalized_block_on_walked_chain(engine_client, ¤t_fc).await?; + // Leave the finalized block as-is, and return the current forkchoice. Ok(current_fc) } +/// Checks that the finalized block is on the walked chain when the safe walk stopped above it, as +/// the sequencing window can stop it. The walk follows the execution layer's canonical chain, so +/// its block at the finalized height is the canonical one there. +async fn ensure_finalized_block_on_walked_chain( + engine_client: &EngineClient, + current_fc: &L2ForkchoiceState, +) -> Result<(), SyncStartError> { + let finalized = current_fc.finalized.block_info; + if current_fc.safe.block_info.number <= finalized.number { + return Ok(()); + } + info!(target: "sync_start", finalized = %finalized.number, "Checking that the finalized block is on the walked chain"); + let walked = engine_client + .get_l2_block(finalized.number.into()) + .await? + .ok_or(SyncStartError::FinalizedHeightUnavailable(finalized.number))? + .header; + if walked.hash == finalized.hash { + return Ok(()); + } + let walked = l2_block_info_at(BlockNumHash { hash: walked.hash, number: walked.number }); + Err(finalized_block_not_on_walked_chain(¤t_fc.finalized, &walked)) +} + +/// The error for the finalized block whose L1 origin the L1 provider does not serve by hash. A +/// provider without a block at that height lags and is retried; one with a different block there +/// follows another L1 chain, which retrying cannot fix; one with that very block there serves it +/// only by number, as a broken or lagging block-hash index does. +async fn finalized_l1_origin_missing_by_hash( + engine_client: &EngineClient, + finalized: &L2BlockInfo, +) -> SyncStartError { + let l1_origin = finalized.l1_origin; + let l1_block = match engine_client.get_l1_block(l1_origin.number.into()).await { + Ok(l1_block) => l1_block, + Err(err) => return err.into(), + }; + match l1_block { + Some(l1_block) if l1_block.header.hash != l1_origin.hash => { + error!( + target: "sync_start", + l1_origin = %l1_origin.number, + l1_block_hash = %l1_block.header.hash, + "The L1 provider follows another chain than the finalized block's L1 origin" + ); + SyncStartError::FinalizedL1OriginNotCanonical { + l1_block: BlockNumHash { hash: l1_block.header.hash, number: l1_origin.number }, + l1_origin, + } + } + Some(_) => { + error!( + target: "sync_start", + l1_origin = %l1_origin.number, + l1_origin_hash = %l1_origin.hash, + "The L1 provider serves the finalized block's L1 origin by number but not by hash" + ); + SyncStartError::FinalizedL1OriginNotServedByHash { l1_origin } + } + None => { + error!( + target: "sync_start", + l2_block = %finalized.block_info.number, + l1_origin = %l1_origin.number, + "The L1 provider does not have the L1 origin of the finalized block" + ); + SyncStartError::FinalizedL1OriginUnavailable { + l1_origin, + l2_block: finalized.block_info.id(), + } + } + } +} + +/// The error for a walk from the execution layer's unsafe head that reached the height of its +/// finalized block without meeting that block. A forkchoice built from such a walk would either +/// finalize a block that is not an ancestor of its head or move finality back, so the sync start +/// refuses it, like op-node's `ReorgFinalizedErr`. +fn finalized_block_not_on_walked_chain( + finalized: &L2BlockInfo, + walked: &L2BlockInfo, +) -> SyncStartError { + error!( + target: "sync_start", + finalized = %finalized.block_info.number, + finalized_hash = %finalized.block_info.hash, + walked = %walked.block_info.number, + walked_hash = %walked.block_info.hash, + "The chain walked from the execution layer's unsafe head does not contain its finalized block" + ); + SyncStartError::MismatchedFinalizedBlock { + finalized: finalized.block_info.id(), + walked: walked.block_info.id(), + } +} + +#[cfg(test)] +mod finality_tests { + use super::*; + use crate::test_utils::{ + expect_sync_start_walk, test_engine_client, test_l2_block_at_l1_origin, test_l2_chain, + test_rpc_block, + }; + use alloy_primitives::B256; + use std::sync::Arc; + + const CANONICAL: u8 = 0; + const FORK: u8 = 1; + + #[tokio::test] + async fn walk_that_meets_the_finalized_block_stops_there() { + let cfg = Arc::new(RollupConfig::default()); + let (client, l1, l2) = test_engine_client(cfg.clone()); + let canonical = test_l2_chain(B256::ZERO, 1, 4, CANONICAL); + expect_sync_start_walk( + &l1, + &l2, + &canonical[1], + &canonical[3], + true, + &[&canonical[2], &canonical[1]], + ); + + let start = find_starting_forkchoice(&cfg, &client).await.unwrap(); + + assert_eq!(start.safe.block_info.hash, canonical[1].header.hash_slow()); + assert_eq!(start.finalized, start.safe); + l1.assert_finished(); + l2.assert_finished(); + } + + /// The execution layer's unsafe head is on a fork from below its finalized block, as after a + /// rejected forkchoice update that it made canonical first, so the safe walk reaches the + /// finalized height on another block. + #[tokio::test] + async fn walk_on_a_fork_below_the_finalized_block_is_refused() { + let cfg = Arc::new(RollupConfig::default()); + let (client, l1, l2) = test_engine_client(cfg.clone()); + let canonical = test_l2_chain(B256::ZERO, 1, 2, CANONICAL); + let fork = test_l2_chain(canonical[0].header.hash_slow(), 2, 4, FORK); + expect_sync_start_walk(&l1, &l2, &canonical[1], &fork[2], true, &[&fork[1], &fork[0]]); + + let error = find_starting_forkchoice(&cfg, &client).await.unwrap_err(); + + assert!( + matches!( + error, + SyncStartError::MismatchedFinalizedBlock { finalized, walked } + if finalized.hash == canonical[1].header.hash_slow() + && walked.hash == fork[0].header.hash_slow() + ), + "got {error:?}" + ); + l1.assert_finished(); + l2.assert_finished(); + } + + /// A safe walk that the sequencing window stops above the finalized height still checks the + /// finalized block: on a fork from below it, the canonical block at its height differs. + #[tokio::test] + async fn walk_stopped_above_finality_by_the_sequencing_window_checks_the_finalized_block() { + let cfg = Arc::new(RollupConfig::default()); + let (client, l1, l2) = test_engine_client(cfg.clone()); + let canonical = test_l2_chain(B256::ZERO, 1, 2, CANONICAL); + let fork_at_2 = test_l2_block_at_l1_origin(2, canonical[0].header.hash_slow(), FORK, 0); + let fork_at_3 = test_l2_block_at_l1_origin(3, fork_at_2.header.hash_slow(), FORK, 1); + let fork_at_4 = test_l2_block_at_l1_origin(4, fork_at_3.header.hash_slow(), FORK, 2); + expect_sync_start_walk(&l1, &l2, &canonical[1], &fork_at_4, true, &[&fork_at_3]); + l2.expect_block_header_by_number(2, Some(test_rpc_block(&fork_at_2))); + + let error = find_starting_forkchoice(&cfg, &client).await.unwrap_err(); + + assert!( + matches!( + error, + SyncStartError::MismatchedFinalizedBlock { finalized, walked } + if finalized.hash == canonical[1].header.hash_slow() + && walked.hash == fork_at_2.header.hash_slow() + ), + "got {error:?}" + ); + l1.assert_finished(); + l2.assert_finished(); + } + + /// An execution layer that answers no block at its finalized height, although the walk shows + /// its canonical chain above it, answered inconsistently; the sync start reports that, rather + /// than a missing block, so the reset is retried. + #[tokio::test] + async fn missing_block_at_the_finalized_height_is_reported_as_such() { + let cfg = Arc::new(RollupConfig::default()); + let (client, l1, l2) = test_engine_client(cfg.clone()); + let canonical = test_l2_chain(B256::ZERO, 1, 2, CANONICAL); + let at_3 = test_l2_block_at_l1_origin(3, canonical[1].header.hash_slow(), CANONICAL, 1); + let at_4 = test_l2_block_at_l1_origin(4, at_3.header.hash_slow(), CANONICAL, 2); + expect_sync_start_walk(&l1, &l2, &canonical[1], &at_4, true, &[&at_3]); + l2.expect_block_header_by_number(2, None); + + let error = find_starting_forkchoice(&cfg, &client).await.unwrap_err(); + + assert!(matches!(error, SyncStartError::FinalizedHeightUnavailable(2)), "got {error:?}"); + l1.assert_finished(); + l2.assert_finished(); + } + + /// A missing L1 origin of the finalized block, with no L1 block at its height, means the L1 + /// provider is behind, since that block's L1 data is finalized, so the reset waits for it + /// instead of moving finality back. + #[tokio::test] + async fn missing_l1_origin_of_the_finalized_block_waits_for_the_l1_provider() { + let cfg = Arc::new(RollupConfig::default()); + let (client, l1, l2) = test_engine_client(cfg.clone()); + let canonical = test_l2_chain(B256::ZERO, 1, 2, CANONICAL); + expect_sync_start_walk(&l1, &l2, &canonical[1], &canonical[1], false, &[]); + l1.expect("eth_getBlockByNumber", None::); + + let error = find_starting_forkchoice(&cfg, &client).await.unwrap_err(); + + assert!( + matches!( + error, + SyncStartError::FinalizedL1OriginUnavailable { l2_block, .. } + if l2_block.hash == canonical[1].header.hash_slow() + ), + "got {error:?}" + ); + l1.assert_finished(); + l2.assert_finished(); + } + + /// A missing L1 origin of another block at the finalized height is that fork's L1 origin, so + /// the walk refuses the fork instead of waiting for the L1 provider. + #[tokio::test] + async fn missing_l1_origin_of_a_fork_block_at_the_finalized_height_is_refused() { + let cfg = Arc::new(RollupConfig::default()); + let (client, l1, l2) = test_engine_client(cfg.clone()); + let canonical = test_l2_chain(B256::ZERO, 1, 2, CANONICAL); + let fork = test_l2_chain(canonical[0].header.hash_slow(), 2, 2, FORK); + expect_sync_start_walk(&l1, &l2, &canonical[1], &fork[0], false, &[]); + + let error = find_starting_forkchoice(&cfg, &client).await.unwrap_err(); + + assert!( + matches!( + error, + SyncStartError::MismatchedFinalizedBlock { finalized, walked } + if finalized.hash == canonical[1].header.hash_slow() + && walked.hash == fork[0].header.hash_slow() + ), + "got {error:?}" + ); + l1.assert_finished(); + l2.assert_finished(); + } +} + #[cfg(test)] mod test { use alloy_provider::Network; diff --git a/rust/kona/crates/node/engine/src/task_queue/core.rs b/rust/kona/crates/node/engine/src/task_queue/core.rs index 7f2a7ec767b..b1473a486e4 100644 --- a/rust/kona/crates/node/engine/src/task_queue/core.rs +++ b/rust/kona/crates/node/engine/src/task_queue/core.rs @@ -2,10 +2,13 @@ use super::ExecutableEngineTask; use crate::{ - EngineClient, EngineState, EngineSyncStateUpdate, EngineTask, EngineTaskError, - EngineTaskErrorSeverity, Metrics, SyncStartError, SynchronizeTaskError, + EngineClient, EngineState, EngineSyncState, EngineSyncStateUpdate, EngineTask, EngineTaskError, + EngineTaskErrorSeverity, L2ForkchoiceState, Metrics, SyncStartError, SynchronizeTaskError, find_starting_forkchoice, - task_queue::{EngineTaskErrors, synchronize_unconditionally}, + sync::l2_block_info_at, + task_queue::{ + EngineTaskErrors, ForkchoiceReply, restore_forkchoice, synchronize_unconditionally, + }, }; use kona_genesis::RollupConfig; use kona_protocol::L2BlockInfo; @@ -113,7 +116,7 @@ impl Engine { client: Arc, config: Arc, ) -> Result { - let start = find_starting_forkchoice(&config, client.as_ref()).await?; + let start = self.find_starting_forkchoice_restoring_finality(&client, &config).await?; // One attempt. The actor retains the reset request and retries transient failures. The // execution layer may have lost or changed its forkchoice, so the reset sends one even when @@ -136,6 +139,66 @@ impl Engine { Ok(start.safe) } + /// Finds the sync start. A walk that misses the finalized block follows a forkchoice update + /// that the execution layer made canonical before rejecting it. The finalized block anchors + /// the restore: the engine's established one, or after a restart, the execution layer's own + /// finalized label. While the execution layer still holds the anchor, the engine restores a + /// forkchoice on its chain and walks once more; otherwise restoring cannot help, and the + /// mismatch stands. + async fn find_starting_forkchoice_restoring_finality( + &self, + client: &Arc, + config: &Arc, + ) -> Result { + let (mismatch, el_finalized) = match find_starting_forkchoice(config, client.as_ref()).await + { + Err(mismatch @ SyncStartError::MismatchedFinalizedBlock { finalized, .. }) => { + (mismatch, finalized) + } + result => return Ok(result?), + }; + let established_finalized = self.state.sync_state.established_finalized_head().is_some(); + let anchor = if established_finalized { + self.state.sync_state.finalized_head() + } else { + l2_block_info_at(el_finalized) + }; + if !client.l2_block_is_stored(anchor.block_info.hash).await.map_err(SyncStartError::from)? { + return Err(mismatch.into()); + } + + warn!(target: "engine", %mismatch, ?anchor, "Restoring a forkchoice on the finalized chain before walking again"); + if established_finalized { + self.restore_engine_forkchoice(client, anchor).await?; + } else { + restore_forkchoice(client.as_ref(), &finalized_only(anchor)).await?; + } + match find_starting_forkchoice(config, client.as_ref()).await { + Err(mismatch @ SyncStartError::MismatchedFinalizedBlock { .. }) => { + Err(EngineResetError::FinalityNotRestored(mismatch)) + } + result => Ok(result?), + } + } + + /// Sends the engine's own forkchoice again. Its labels may include a delegated safe head that + /// the execution layer never accepted, or an unsafe head it does not have; when it rejects + /// them or answers `SYNCING`, a forkchoice of the `finalized` block alone, which it holds, + /// restores its chain instead. + async fn restore_engine_forkchoice( + &self, + client: &Arc, + finalized: L2BlockInfo, + ) -> Result<(), SynchronizeTaskError> { + match restore_forkchoice(client.as_ref(), &self.state.sync_state).await { + Ok(ForkchoiceReply::Valid) => Ok(()), + Ok(ForkchoiceReply::Syncing) | Err(SynchronizeTaskError::InvalidForkchoiceState) => { + restore_forkchoice(client.as_ref(), &finalized_only(finalized)).await.map(|_| ()) + } + Err(err) => Err(err), + } + } + /// Clears the task queue. pub fn clear(&mut self) { self.tasks.clear(); @@ -175,9 +238,23 @@ impl Engine { } } +/// A sync state whose every label is `finalized`. +fn finalized_only(finalized: L2BlockInfo) -> EngineSyncState { + EngineSyncState::default().with_update(EngineSyncStateUpdate { + finalized_head: Some(finalized), + local_safe_head: Some(finalized), + safe_head: Some(finalized), + unsafe_head: Some(finalized), + }) +} + /// An error occurred while attempting to reset the [`Engine`]. #[derive(Debug, Error)] pub enum EngineResetError { + /// The walk still missed the finalized block after a forkchoice on that block's chain was + /// restored, as while the execution layer has not yet made it canonical. + #[error("Sync start still misses the finalized block after a restore: {0}")] + FinalityNotRestored(SyncStartError), /// An error that occurred while updating the forkchoice state. #[error(transparent)] Forkchoice(#[from] SynchronizeTaskError), @@ -190,7 +267,20 @@ impl EngineResetError { /// Whether retrying the reset can recover without changing the node configuration. pub fn is_temporary(&self) -> bool { match self { - Self::SyncStart(SyncStartError::RpcError(_)) => true, + // After a restore the execution layer holds the engine's chain, and retrying gives it + // time to make that chain canonical. A missing L1 origin of the finalized block means + // the L1 provider is behind; one it serves only by number may be a backend behind a + // load balancer whose hash index lags; a missing block at the finalized height means + // the execution layer answered inconsistently. + Self::FinalityNotRestored(_) | + Self::SyncStart( + SyncStartError::FinalizedHeightUnavailable(_) | + SyncStartError::FinalizedL1OriginNotServedByHash { .. } | + SyncStartError::FinalizedL1OriginUnavailable { .. } | + SyncStartError::RpcError(_), + ) => true, + // An L1 provider on another chain than the finalized block's L1 origin needs + // reconfiguring. Self::SyncStart(_) => false, Self::Forkchoice(err) => err.severity() != EngineTaskErrorSeverity::Critical, } @@ -204,16 +294,373 @@ mod recovery_tests { ConsolidateInput, ConsolidateTask, FinalizeBlockId, FinalizeTask, InsertTask, NoopBlockSink, test_utils::{ - TestAttributesBuilder, TestEngineStateBuilder, expect_sync_start, test_block_info, - test_engine_client, + RpcMock, TestAttributesBuilder, TestEngineStateBuilder, expect_sync_start, + expect_sync_start_walk, test_block_info, test_engine_client, test_l2_chain, }, }; + use alloy_primitives::B256; use alloy_rpc_types_engine::{ - BlobsBundleV1, ExecutionPayloadV1, ExecutionPayloadV3, ForkchoiceUpdated, PayloadId, - PayloadStatus, PayloadStatusEnum, + BlobsBundleV1, ExecutionPayloadV1, ExecutionPayloadV3, ForkchoiceState, ForkchoiceUpdated, + PayloadId, PayloadStatus, PayloadStatusEnum, }; + use op_alloy_consensus::OpBlock; use op_alloy_rpc_types_engine::{OpExecutionPayloadEnvelope, OpExecutionPayloadEnvelopeV3}; + /// A canonical chain `1..=4` and a fork from block 1 at `2..=4`, as an execution layer holds + /// them after a rejected forkchoice update made the fork canonical before checking block 2, + /// the finalized block. + struct ForkedExecutionLayer { + canonical: Vec, + fork: Vec, + } + + impl ForkedExecutionLayer { + fn new() -> Self { + let canonical = test_l2_chain(B256::ZERO, 1, 4, 0); + let fork = test_l2_chain(canonical[0].header.hash_slow(), 2, 4, 1); + Self { canonical, fork } + } + + fn info(block: &OpBlock) -> L2BlockInfo { + L2BlockInfo::from_block_and_genesis(block, &RollupConfig::default().genesis).unwrap() + } + + fn finalized(&self) -> L2BlockInfo { + Self::info(&self.canonical[1]) + } + + /// The engine's labels: unsafe at canonical block 4, safe and finalized at block 2. + fn engine_state(&self) -> EngineState { + TestEngineStateBuilder::new() + .with_unsafe_head(Self::info(&self.canonical[3])) + .with_safe_head(self.finalized()) + .with_finalized_head(self.finalized()) + .build() + } + + fn expect_walk_on_fork(&self, l1: &RpcMock, l2: &RpcMock) { + expect_sync_start_walk( + l1, + l2, + &self.canonical[1], + &self.fork[2], + true, + &[&self.fork[1], &self.fork[0]], + ); + } + + fn expect_walk_on_canonical_chain(&self, l1: &RpcMock, l2: &RpcMock) { + expect_sync_start_walk( + l1, + l2, + &self.canonical[1], + &self.canonical[3], + true, + &[&self.canonical[2], &self.canonical[1]], + ); + } + + /// The forkchoice of the finalized block alone. + fn finalized_only_forkchoice(&self) -> ForkchoiceState { + let finalized = self.finalized().block_info.hash; + ForkchoiceState { + finalized_block_hash: finalized, + head_block_hash: finalized, + safe_block_hash: finalized, + } + } + + /// The execution layer's answer to the lookup of the engine's finalized block by hash. + fn expect_finalized_lookup(&self, l2: &RpcMock, el_holds_it: bool) { + l2.expect_block_header_by_hash( + self.finalized().block_info.hash, + el_holds_it.then(|| crate::test_utils::test_rpc_block(&self.canonical[1])), + ); + } + } + + async fn reset_from( + initial: EngineState, + client: EngineClient, + ) -> Result { + let (state_tx, _state_rx) = tokio::sync::watch::channel(initial); + let mut engine = Engine::new(initial, state_tx); + engine.reset(Arc::new(client), Arc::new(RollupConfig::default())).await + } + + /// A reset whose walk misses the finalized block, which the execution layer still holds, + /// sends the engine's own labels again, which makes its chain canonical, and then starts from + /// the walk that meets the finalized block. + #[tokio::test] + async fn reset_restores_the_engine_forkchoice_when_the_walk_misses_finality() { + let el = ForkedExecutionLayer::new(); + let initial = el.engine_state(); + let forkchoice = initial.sync_state.create_forkchoice_state(); + let (client, l1, l2) = test_engine_client(Arc::new(RollupConfig::default())); + el.expect_walk_on_fork(&l1, &l2); + el.expect_finalized_lookup(&l2, true); + l2.expect_forkchoice_updated(forkchoice, PayloadStatusEnum::Valid); + el.expect_walk_on_canonical_chain(&l1, &l2); + l2.expect_forkchoice_updated(forkchoice, PayloadStatusEnum::Valid); + + let safe = reset_from(initial, client).await.unwrap(); + + assert_eq!(safe, el.finalized()); + l1.assert_finished(); + l2.assert_finished(); + } + + /// The engine's own labels may carry a safe head the execution layer never accepted; when it + /// rejects them, a forkchoice of the finalized block alone restores its chain instead. + #[tokio::test] + async fn rejected_restore_falls_back_to_the_finalized_block() { + let el = ForkedExecutionLayer::new(); + let initial = el.engine_state(); + let finalized_only = el.finalized_only_forkchoice(); + let (client, l1, l2) = test_engine_client(Arc::new(RollupConfig::default())); + el.expect_walk_on_fork(&l1, &l2); + el.expect_finalized_lookup(&l2, true); + l2.expect_error_code( + "engine_forkchoiceUpdatedV3", + i64::from(alloy_rpc_types_engine::INVALID_FORK_CHOICE_STATE_ERROR), + ); + l2.expect_forkchoice_updated(finalized_only, PayloadStatusEnum::Valid); + el.expect_walk_on_canonical_chain(&l1, &l2); + l2.expect_forkchoice_updated( + initial.sync_state.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + + let safe = reset_from(initial, client).await.unwrap(); + + assert_eq!(safe, el.finalized()); + l1.assert_finished(); + l2.assert_finished(); + } + + /// A walk that still misses the finalized block after the engine's labels and then the + /// finalized block alone were sent, as while the execution layer syncs back to that chain, + /// fails the reset with a retryable error. + #[tokio::test] + async fn reset_retries_while_the_restored_forkchoice_is_not_yet_canonical() { + let el = ForkedExecutionLayer::new(); + let initial = el.engine_state(); + let (client, l1, l2) = test_engine_client(Arc::new(RollupConfig::default())); + el.expect_walk_on_fork(&l1, &l2); + el.expect_finalized_lookup(&l2, true); + l2.expect_forkchoice_updated( + initial.sync_state.create_forkchoice_state(), + PayloadStatusEnum::Syncing, + ); + l2.expect_forkchoice_updated(el.finalized_only_forkchoice(), PayloadStatusEnum::Syncing); + el.expect_walk_on_fork(&l1, &l2); + + let error = reset_from(initial, client).await.unwrap_err(); + + assert!(matches!(error, EngineResetError::FinalityNotRestored(_)), "got {error:?}"); + assert!(error.is_temporary()); + l1.assert_finished(); + l2.assert_finished(); + } + + /// A walk that misses the finalized block stops the node, like op-node's `ReorgFinalizedErr`, + /// when the execution layer no longer holds the restore's anchor: the engine's established + /// finalized block, or after a restart, the execution layer's own finalized label. + #[rstest::rstest] + #[case::engine_finality_lost(ForkedExecutionLayer::new().engine_state())] + #[case::el_finalized_label_lost_after_a_restart(EngineState::default())] + #[tokio::test] + async fn walk_that_misses_finality_stops_the_node_when_the_anchor_is_lost( + #[case] initial: EngineState, + ) { + let el = ForkedExecutionLayer::new(); + let (client, l1, l2) = test_engine_client(Arc::new(RollupConfig::default())); + el.expect_walk_on_fork(&l1, &l2); + el.expect_finalized_lookup(&l2, false); + + let error = reset_from(initial, client).await.unwrap_err(); + + assert!( + matches!( + error, + EngineResetError::SyncStart(SyncStartError::MismatchedFinalizedBlock { .. }) + ), + "got {error:?}" + ); + assert!(!error.is_temporary()); + l1.assert_finished(); + l2.assert_finished(); + } + + /// After a restart the engine has no finality of its own, so a rejected forkchoice update + /// that left the execution layer on a fork from below its finalized block is undone with the + /// execution layer's own finalized label: a forkchoice of that block alone, from which the + /// walk then starts. + #[tokio::test] + async fn reset_after_a_restart_restores_the_el_finalized_block() { + let el = ForkedExecutionLayer::new(); + let (client, l1, l2) = test_engine_client(Arc::new(RollupConfig::default())); + el.expect_walk_on_fork(&l1, &l2); + el.expect_finalized_lookup(&l2, true); + l2.expect_forkchoice_updated(el.finalized_only_forkchoice(), PayloadStatusEnum::Valid); + expect_sync_start_walk(&l1, &l2, &el.canonical[1], &el.canonical[1], true, &[]); + l2.expect_forkchoice_updated(el.finalized_only_forkchoice(), PayloadStatusEnum::Valid); + + let safe = reset_from(EngineState::default(), client).await.unwrap(); + + assert_eq!(safe, el.finalized()); + l1.assert_finished(); + l2.assert_finished(); + } + + /// An engine whose labels the execution layer answers `SYNCING`, as when it lacks the + /// engine's unsafe head, restores the finalized block alone instead, and the walk then meets + /// it. + #[tokio::test] + async fn syncing_restore_falls_back_to_the_finalized_block() { + let el = ForkedExecutionLayer::new(); + let initial = el.engine_state(); + let (client, l1, l2) = test_engine_client(Arc::new(RollupConfig::default())); + el.expect_walk_on_fork(&l1, &l2); + el.expect_finalized_lookup(&l2, true); + l2.expect_forkchoice_updated( + initial.sync_state.create_forkchoice_state(), + PayloadStatusEnum::Syncing, + ); + l2.expect_forkchoice_updated(el.finalized_only_forkchoice(), PayloadStatusEnum::Valid); + el.expect_walk_on_canonical_chain(&l1, &l2); + l2.expect_forkchoice_updated( + initial.sync_state.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + + let safe = reset_from(initial, client).await.unwrap(); + + assert_eq!(safe, el.finalized()); + l1.assert_finished(); + l2.assert_finished(); + } + + /// A permanent sync-start failure on the walk after a restore stays fatal instead of being + /// retried as a finality that is not restored yet. + #[tokio::test] + async fn permanent_sync_start_error_after_a_restore_stays_fatal() { + let el = ForkedExecutionLayer::new(); + let initial = el.engine_state(); + let (client, l1, l2) = test_engine_client(Arc::new(RollupConfig::default())); + el.expect_walk_on_fork(&l1, &l2); + el.expect_finalized_lookup(&l2, true); + l2.expect_forkchoice_updated( + initial.sync_state.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + expect_sync_start_walk(&l1, &l2, &el.canonical[1], &el.canonical[3], true, &[]); + l2.expect("eth_getBlockByHash", None::); + + let error = reset_from(initial, client).await.unwrap_err(); + + assert!( + matches!(error, EngineResetError::SyncStart(SyncStartError::BlockNotFound(_))), + "got {error:?}" + ); + assert!(!error.is_temporary()); + l1.assert_finished(); + l2.assert_finished(); + } + + /// A reset retries the sync-start errors that a lagging or briefly inconsistent provider + /// causes, and stops the node on the others. + #[rstest::rstest] + #[case::finalized_height_unavailable(SyncStartError::FinalizedHeightUnavailable(2), true)] + #[case::block_not_found(SyncStartError::BlockNotFound(2u64.into()), false)] + fn sync_start_errors_a_reset_retries( + #[case] error: SyncStartError, + #[case] is_temporary: bool, + ) { + assert_eq!(EngineResetError::SyncStart(error).is_temporary(), is_temporary); + } + + /// An L1 provider without a block at the height of the finalized block's L1 origin lags, so + /// the reset retries. + fn l1_lacks_the_height(l1: &RpcMock) { + l1.expect("eth_getBlockByNumber", None::); + } + + /// An L1 provider with another block at that height follows another L1 chain, which retrying + /// cannot fix. + fn l1_on_another_chain(l1: &RpcMock) { + let other_l1_block = alloy_rpc_types_eth::Block:: { + header: alloy_rpc_types_eth::Header { + hash: B256::repeat_byte(0x11), + ..Default::default() + }, + ..Default::default() + }; + l1.expect("eth_getBlockByNumber", other_l1_block); + } + + /// An L1 provider whose block at that height is the finalized block's L1 origin serves it by + /// number but not by hash, as a lagging block-hash index behind a load balancer does. + fn l1_serves_the_origin_only_by_number(l1: &RpcMock) { + let l1_origin = ForkedExecutionLayer::new().finalized().l1_origin; + let l1_block = alloy_rpc_types_eth::Block:: { + header: alloy_rpc_types_eth::Header { hash: l1_origin.hash, ..Default::default() }, + ..Default::default() + }; + l1.expect("eth_getBlockByNumber", l1_block); + } + + fn is_l1_origin_not_served_by_hash(error: &EngineResetError) -> bool { + matches!( + error, + EngineResetError::SyncStart(SyncStartError::FinalizedL1OriginNotServedByHash { .. }) + ) + } + + fn is_l1_origin_unavailable(error: &EngineResetError) -> bool { + matches!( + error, + EngineResetError::SyncStart(SyncStartError::FinalizedL1OriginUnavailable { .. }) + ) + } + + fn is_l1_origin_not_canonical(error: &EngineResetError) -> bool { + matches!( + error, + EngineResetError::SyncStart(SyncStartError::FinalizedL1OriginNotCanonical { .. }) + ) + } + + /// A finalized block whose L1 origin the L1 provider does not serve by hash is retried only + /// when the provider lags behind that height, and stops the node when the provider follows + /// another L1 chain. + #[rstest::rstest] + #[case::l1_lags(l1_lacks_the_height, is_l1_origin_unavailable, true)] + #[case::l1_on_another_chain(l1_on_another_chain, is_l1_origin_not_canonical, false)] + #[case::l1_serves_the_origin_only_by_number( + l1_serves_the_origin_only_by_number, + is_l1_origin_not_served_by_hash, + true + )] + #[tokio::test] + async fn missing_l1_origin_of_the_finalized_block( + #[case] expect_l1_at_origin_height: fn(&RpcMock), + #[case] is_expected: fn(&EngineResetError) -> bool, + #[case] is_temporary: bool, + ) { + let el = ForkedExecutionLayer::new(); + let (client, l1, l2) = test_engine_client(Arc::new(RollupConfig::default())); + expect_sync_start_walk(&l1, &l2, &el.canonical[1], &el.canonical[1], false, &[]); + expect_l1_at_origin_height(&l1); + + let error = reset_from(el.engine_state(), client).await.unwrap_err(); + + assert!(is_expected(&error), "got {error:?}"); + assert_eq!(error.is_temporary(), is_temporary); + l1.assert_finished(); + l2.assert_finished(); + } + #[tokio::test] async fn temporary_failure_returns_with_task_retained_for_recovery() { let payload = OpExecutionPayloadEnvelope::V1(ExecutionPayloadV1::from_block_slow( diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/delegated_rejection.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/delegated_rejection.rs new file mode 100644 index 00000000000..371a5bce673 --- /dev/null +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/delegated_rejection.rs @@ -0,0 +1,27 @@ +//! Why the execution layer rejected a forkchoice update that followed delegated safe heads. + +/// Why the execution layer rejected a forkchoice update that followed delegated safe heads, as +/// far as the engine can tell. +#[derive(Debug, Clone, Copy, PartialEq, Eq, derive_more::Display)] +pub enum DelegatedRejection { + /// The update moved the head onto a chain that does not contain the engine's finalized block. + /// Every later head on that chain is rejected too. + #[display("chain below finality")] + ChainBelowFinality, + /// The update moved the head onto a chain that contains the finalized block, and the execution + /// layer rejected its labels, such as a safe block that is not an ancestor of the head. The + /// same heads are rejected again. + #[display("head labels")] + HeadLabels, + /// The update moved only the safe label. The execution layer may not have stored that block + /// yet, so the same heads may be accepted on a later poll. + #[display("safe label")] + SafeLabel, +} + +impl DelegatedRejection { + /// Whether the same delegated heads are rejected again until the engine resets. + pub const fn is_permanent(self) -> bool { + !matches!(self, Self::SafeLabel) + } +} diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/error.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/error.rs index 0c9869ab7ae..8e9d2113e08 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/error.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/error.rs @@ -1,9 +1,11 @@ //! Contains error types for the [`crate::ConsolidateTask`]. use crate::{ - BuildTaskError, EngineTaskError, FinalityViolation, SealTaskError, SynchronizeTaskError, + BuildTaskError, DelegatedRejection, DelegatedSafeHeads, EngineTaskError, FinalityViolation, + SealTaskError, SynchronizeTaskError, task_queue::tasks::{BuildAndImportError, task::EngineTaskErrorSeverity}, }; +use alloy_primitives::B256; use thiserror::Error; /// An error that occurs when running the [`crate::ConsolidateTask`]. @@ -12,9 +14,21 @@ pub enum ConsolidateTaskError { /// A delegated safe head would move the safe labels below finality. #[error("Delegated safe head breaks finality: {0}")] BreaksFinality(#[from] FinalityViolation), + /// The execution layer rejected a forkchoice update that followed delegated safe heads, and + /// the engine's own forkchoice is in place on the execution layer again. + #[error("Execution layer rejected delegated safe heads {heads:?}: {rejection}")] + DelegatedForkchoiceRejected { + /// The delegated safe heads. + heads: DelegatedSafeHeads, + /// Why the execution layer rejected them. + rejection: DelegatedRejection, + }, /// Failed to fetch the execution layer's block at the height of a delegated safe head. #[error("Failed to fetch the block at delegated safe head height {0}")] FailedToFetchDelegatedSafeBlock(u64), + /// Failed to look up the engine's finalized block by hash after a rejected forkchoice update. + #[error("Failed to fetch the finalized block {0}")] + FailedToFetchFinalizedBlock(B256), /// The unsafe L2 block is missing. #[error("Unsafe L2 block is missing {0}")] MissingUnsafeL2Block(u64), @@ -44,11 +58,13 @@ impl From for ConsolidateTaskError { impl EngineTaskError for ConsolidateTaskError { fn severity(&self) -> EngineTaskErrorSeverity { match self { - Self::BreaksFinality(_) => EngineTaskErrorSeverity::Skip, - Self::MissingUnsafeL2Block(_) => EngineTaskErrorSeverity::Reset, - Self::FailedToFetchDelegatedSafeBlock(_) | Self::FailedToFetchUnsafeL2Block => { - EngineTaskErrorSeverity::Temporary + Self::BreaksFinality(_) | Self::DelegatedForkchoiceRejected { .. } => { + EngineTaskErrorSeverity::Skip } + Self::MissingUnsafeL2Block(_) => EngineTaskErrorSeverity::Reset, + Self::FailedToFetchDelegatedSafeBlock(_) | + Self::FailedToFetchFinalizedBlock(_) | + Self::FailedToFetchUnsafeL2Block => EngineTaskErrorSeverity::Temporary, Self::BuildTaskFailed(inner) => inner.severity(), Self::SealTaskFailed(inner) => inner.severity(), Self::ForkchoiceUpdateFailed(inner) => inner.severity(), diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/mod.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/mod.rs index 110983b601f..a423b030eaf 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/mod.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/mod.rs @@ -1,5 +1,8 @@ //! Consolidation Task +mod delegated_rejection; +pub use delegated_rejection::DelegatedRejection; + mod delegated_safe_heads; pub use delegated_safe_heads::{DelegatedSafeHeads, InconsistentSafeHeadsError}; diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs index f9d6c5dd2cc..4bdc3cf2ac6 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs @@ -1,11 +1,16 @@ //! A task to consolidate the engine state. use crate::{ - ConsolidateTaskError, DelegatedSafeHeads, EngineClient, EngineState, ExecutableEngineTask, - FinalityViolation, ImportedBlockSink, + ConsolidateTaskError, DelegatedRejection, DelegatedSafeHeads, EngineClient, EngineState, + EngineSyncState, ExecutableEngineTask, FinalityViolation, ImportedBlockSink, + SynchronizeTaskError, state::EngineSyncStateUpdate, - task_queue::{build_and_import, synchronize, synchronize_unconditionally}, + task_queue::{ + ForkchoiceReply, build_and_import, restore_forkchoice, synchronize, + synchronize_unconditionally, + }, }; +use alloy_eips::BlockNumHash; use alloy_primitives::B256; use alloy_rpc_types_eth::Block; use async_trait::async_trait; @@ -170,6 +175,12 @@ impl ConsolidateTask { /// The safe label for delegated safe heads whose local-safe head is on the engine's chain: the /// cross-safe head when it advances the label, or when it replaces, at the label's height, a /// safe label that a reorg left on abandoned blocks, and `None`, which keeps the label. + /// + /// A cross-safe head below the safe label never replaces it. The execution layer answered + /// `VALID` to a forkchoice update carrying the label only if the label was a canonical + /// ancestor of the head, and a label it has not applied cannot be judged against its chain, so + /// no lookup can show such a label abandoned. A stale label instead makes the execution layer + /// reject the next forkchoice update. async fn safe_label_on_engine_chain( &self, state: &EngineState, @@ -181,7 +192,6 @@ impl ConsolidateTask { } let safe = state.sync_state.safe_head(); let cross_safe = heads.safe_head(); - // Below the safe label the cross-safe head can neither advance nor replace it. if cross_safe.block_info.number < safe.block_info.number { return Ok(None); } @@ -200,7 +210,7 @@ impl ConsolidateTask { } let is_proven_on_engine_chain = chain_hash.is_some(); let replaces_stale_safe_label = is_proven_on_engine_chain && - cross_safe.block_info.number == safe.block_info.number && + cross_safe.block_info.number <= safe.block_info.number && cross_safe.block_info.hash != safe.block_info.hash; Ok(if replaces_stale_safe_label { Some(cross_safe) @@ -367,10 +377,18 @@ impl ConsolidateTask { update: EngineSyncStateUpdate, outcome: &'static str, ) -> Result<(), ConsolidateTaskError> { - synchronize_unconditionally(self.client.as_ref(), state, update).await.map_err(|e| { - warn!(target: "engine", ?e, ?heads, "Applying the delegated safe heads failed"); - e - })?; + let previous = state.sync_state; + match synchronize_unconditionally(self.client.as_ref(), state, update).await { + Ok(()) => {} + Err(SynchronizeTaskError::InvalidForkchoiceState) => { + let attempted = previous.with_update(update); + return Err(self.classify_rejected_forkchoice(&previous, &attempted, heads).await); + } + Err(err) => { + warn!(target: "engine", ?err, ?heads, "Applying the delegated safe heads failed"); + return Err(err.into()); + } + } info!( target: "engine", @@ -384,6 +402,95 @@ impl ConsolidateTask { Ok(()) } + /// Classifies a forkchoice update that followed delegated safe heads and that the execution + /// layer rejected. op-geth and reth make the requested head canonical before checking the + /// forkchoice, so a rejection that moved the head can leave the execution layer on the + /// delegate's chain. + /// + /// - An update that moved neither the head nor the safe label rejected the engine's own labels, + /// so the engine resets. + /// - An update that moved only the safe label left the execution layer's chain alone. Its block + /// may not be stored yet, so the heads are skipped for this poll only. + /// - An update that moved the head is permanent. An execution layer that no longer holds the + /// engine's finalized block lost its chain, and only a reset can rebuild the labels. + /// Otherwise the canonical block at the finalized height tells whether the delegate's chain + /// contains the finalized block, and the engine's own forkchoice is sent again to restore the + /// execution layer's chain. + async fn classify_rejected_forkchoice( + &self, + previous: &EngineSyncState, + attempted: &EngineSyncState, + heads: &DelegatedSafeHeads, + ) -> ConsolidateTaskError { + let rejection = ConsolidateTaskError::ForkchoiceUpdateFailed( + SynchronizeTaskError::InvalidForkchoiceState, + ); + let moves_head = attempted.unsafe_head() != previous.unsafe_head(); + let moves_safe_label = attempted.safe_head() != previous.safe_head(); + if !moves_head && !moves_safe_label { + warn!(target: "engine", ?heads, "Execution layer rejected the engine's own forkchoice; resetting"); + return rejection; + } + if !moves_head { + warn!(target: "engine", ?heads, "Execution layer rejected the delegated safe label; skipping it for this poll"); + return ConsolidateTaskError::DelegatedForkchoiceRejected { + heads: *heads, + rejection: DelegatedRejection::SafeLabel, + }; + } + + let is_chain_below_finality = match previous.established_finalized_head() { + Some(finalized) => match self.is_off_finalized_chain(finalized).await { + Ok(Some(is_off_finalized_chain)) => is_off_finalized_chain, + Ok(None) => { + warn!(target: "engine", ?finalized, ?heads, "Execution layer no longer holds the finalized block; resetting"); + return rejection; + } + Err(err) => return err, + }, + None => false, + }; + // A `SYNCING` answer leaves the execution layer's canonical chain on the delegate's head, + // so only a reset, which walks it and restores the engine's forkchoice, can recover. + match restore_forkchoice(self.client.as_ref(), previous).await { + Ok(ForkchoiceReply::Valid) => {} + Ok(ForkchoiceReply::Syncing) | Err(SynchronizeTaskError::InvalidForkchoiceState) => { + warn!(target: "engine", ?heads, "Execution layer did not apply the engine's restored forkchoice; resetting"); + return rejection; + } + Err(err) => return err.into(), + } + let rejection = if is_chain_below_finality { + DelegatedRejection::ChainBelowFinality + } else { + DelegatedRejection::HeadLabels + }; + warn!(target: "engine", ?heads, %rejection, "Execution layer rejected the delegated safe heads; restored the engine's forkchoice"); + ConsolidateTaskError::DelegatedForkchoiceRejected { heads: *heads, rejection } + } + + /// Whether the execution layer's canonical block at the `finalized` height differs from it, + /// after a rejected forkchoice update made the update's head canonical. `None` when the + /// execution layer no longer stores the finalized block at all. + async fn is_off_finalized_chain( + &self, + finalized: BlockNumHash, + ) -> Result, ConsolidateTaskError> { + let lookup_failed = |err| { + warn!(target: "engine", ?err, ?finalized, "Failed to look up the finalized block"); + ConsolidateTaskError::FailedToFetchFinalizedBlock(finalized.hash) + }; + if !self.client.l2_block_is_stored(finalized.hash).await.map_err(lookup_failed)? { + return Ok(None); + } + debug!(target: "engine", number = finalized.number, "Looking up the canonical block at the finalized height"); + let canonical = + self.client.get_l2_block(finalized.number.into()).await.map_err(lookup_failed)?; + let canonical_hash = canonical.map(|block| block.header.hash); + debug!(target: "engine", ?canonical_hash, "Looked up the canonical block at the finalized height"); + Ok(Some(canonical_hash.is_some_and(|hash| hash != finalized.hash))) + } + /// Follows delegated safe heads below the engine's local-safe head. Only a different block at /// that height in a synced execution layer proves divergence and reorgs; otherwise the /// cross-safe head may only move the safe label. After a delegate L1 reorg whose fork point is @@ -568,19 +675,19 @@ impl ConsolidateTask { safe_head, ..Default::default() }; - let synchronized = match &self.input { - // Derived attributes move the safe head on every consolidation, so an unchanged - // forkchoice was already sent. - ConsolidateInput::Attributes(_) => { - synchronize(self.client.as_ref(), state, update).await - } - // A delegated safe head can repeat a forkchoice the execution layer answered `SYNCING` - // or lost, so it is always sent. - ConsolidateInput::Delegated(_) => { - synchronize_unconditionally(self.client.as_ref(), state, update).await - } - }; - synchronized.map_err(|e| { + if let ConsolidateInput::Delegated(heads) = &self.input { + return self + .apply_delegated_safe_heads( + state, + heads, + update, + "Updated safe head via L1 consolidation", + ) + .await; + } + // Derived attributes move the safe head on every consolidation, so an unchanged forkchoice + // was already sent. + synchronize(self.client.as_ref(), state, update).await.map_err(|e| { warn!(target: "engine", ?e, "Consolidation failed"); e })?; @@ -649,8 +756,9 @@ impl ExecutableEngineTask for ConsolidateTask { mod tests { use super::*; use crate::{ - EngineSyncState, EngineTaskError, EngineTaskErrorSeverity, FinalityViolation, - ImportedBlockSink, NoopBlockSink, SealTaskError, + DelegatedRejection, EngineSyncState, EngineTaskError, + EngineTaskErrorSeverity::{self, Reset, Temporary}, + FinalityViolation, ImportedBlockSink, NoopBlockSink, SealTaskError, SynchronizeTaskError, test_utils::{ RpcMock, TestAttributesBuilder, TestEngineStateBuilder, delegated_safe_heads, l1_info_deposit, test_block_info, test_engine_client, @@ -1226,8 +1334,8 @@ mod tests { l2.expect_block_by_number(10, ElHas::EngineChain.block(10)); } - /// A cross-safe head behind the safe label can neither advance nor replace it, so it is not - /// looked up. + /// A cross-safe head behind the safe label cannot advance it, so neither it nor the safe label + /// is looked up. fn cross_safe_behind_safe_label(l2: &RpcMock) { l2.expect_block_by_number(10, ElHas::EngineChain.block(10)); } @@ -1420,13 +1528,17 @@ mod tests { assert_eq!(state, state_before); } - /// A lagging delegate is not judged against an execution layer that has not applied the - /// engine's last forkchoice, even after its first sync finished: its canonical block at the - /// local-safe height may predate the engine's labels, so no lookup is made and a delegate on - /// another chain reorgs nothing. + /// A lagging delegate is not judged against an execution layer that did not apply the last + /// forkchoice update, during its first sync or after a later `SYNCING` answer: its canonical + /// block at the local-safe height may predate the engine's labels, so no lookup is made and a + /// delegate on another chain reorgs nothing. + #[rstest::rstest] + #[case::first_sync(false)] + #[case::syncing_after_first_sync(true)] #[tokio::test] - async fn lagging_delegate_is_not_judged_against_a_syncing_el() { + async fn lagging_delegate_is_not_judged_against_a_syncing_el(#[case] el_sync_finished: bool) { let mut state = engine_state_with_safe_labels(12, 12); + state.el_sync_finished = el_sync_finished; state.el_applied_last_forkchoice = false; let state_before = state; @@ -1545,17 +1657,21 @@ mod tests { assert_eq!(state.sync_state, expected); } - /// While the execution layer has not finished syncing, its canonical blocks may be from before - /// the sync, so the cross-safe head is not checked against them and only advances the safe - /// label. + /// While the execution layer has not applied the last forkchoice update, during its first sync + /// or after a later `SYNCING` answer, its canonical blocks may be from before the sync, so the + /// cross-safe head is not checked against them and only advances the safe label. + #[rstest::rstest] + #[case::first_sync(false)] + #[case::syncing_after_first_sync(true)] #[tokio::test] - async fn cross_safe_head_is_not_judged_against_a_syncing_el() { + async fn cross_safe_head_is_not_judged_against_a_syncing_el(#[case] el_sync_finished: bool) { let (local_safe, cross_safe) = (engine_chain_block(20), engine_chain_block(17)); let mut state = TestEngineStateBuilder::new() .with_unsafe_head(engine_chain_block(20)) .with_safe_head(engine_chain_block(15)) .with_finalized_head(engine_chain_block(5)) - .with_el_sync_finished(false) + .with_el_sync_finished(el_sync_finished) + .with_el_applied_last_forkchoice(false) .build(); let expected = labels_after_reorg(&state, local_safe, cross_safe); @@ -1627,15 +1743,14 @@ mod tests { } /// Delegated heads at the unsafe head on the engine's chain, as when the engine has no unsafe - /// gossip, keep the safe label for a cross-safe head at or below it, so a lagging backend - /// behind a load balancer does not move safe back. + /// gossip, keep the safe label for a cross-safe head at or below it without a lookup, so a + /// lagging backend behind a load balancer does not move safe back. #[rstest::rstest] - #[case::at_safe_label(15, no_lookup)] - #[case::behind_safe_label(12, no_lookup)] + #[case::at_safe_label(15)] + #[case::behind_safe_label(12)] #[tokio::test] async fn delegated_heads_at_the_unsafe_head_keep_the_safe_label( #[case] cross_safe_number: u64, - #[case] expect_cross_safe_lookup: fn(&RpcMock), ) { let local_safe = engine_chain_block(20); let mut state = engine_state_with_safe_labels(15, 15); @@ -1646,7 +1761,6 @@ mod tests { }); run_delegated(&mut state, local_safe, engine_chain_block(cross_safe_number), |l2| { - expect_cross_safe_lookup(l2); l2.expect_forkchoice_updated( expected.create_forkchoice_state(), PayloadStatusEnum::Valid, @@ -1658,9 +1772,6 @@ mod tests { assert_eq!(state.sync_state, expected); } - /// A cross-safe head at one of the engine's labels is not looked up. - const fn no_lookup(_: &RpcMock) {} - /// The execution layer has the engine chain's block at 15, which the safe label alone cannot /// prove, since it can stay on blocks a reorg abandoned. fn cross_safe_15_on_the_engine_chain(l2: &RpcMock) { @@ -1674,27 +1785,312 @@ mod tests { ); } - /// A forkchoice update for delegated safe heads, whether a reorg off the engine's chain or an - /// extension of it, that the execution layer refuses as an invalid forkchoice state resets the - /// engine, which rebuilds the labels from the execution layer. + /// The engine's finalized head at 5, which the execution layer still holds. + fn el_holds_finalized(l2: &RpcMock) { + l2.expect_block_header_by_hash( + engine_chain_block(5).block_info.hash, + ElHas::EngineChain.block(5), + ); + } + + /// After the rejected update made its head canonical, the execution layer's block at the + /// finalized height is the engine chain's or another chain's. + fn canonical_at_finalized_height(l2: &RpcMock, chain: ElHas) { + l2.expect_block_header_by_number(5, chain.block(5)); + } + + /// The execution layer answers the restoring forkchoice update for the labels in `state`. + fn expect_restore(l2: &RpcMock, state: &EngineState, status: PayloadStatusEnum) { + l2.expect_forkchoice_updated(state.sync_state.create_forkchoice_state(), status); + } + + /// Delegated heads that reorg the engine onto another chain at the unsafe head. + fn reorg_at_unsafe_head() -> (L2BlockInfo, L2BlockInfo) { + (other_chain_block(20), other_chain_block(20)) + } + + /// Delegated heads above the unsafe head whose ancestry is unknown, so the engine reorgs. + fn unproven_extension() -> (L2BlockInfo, L2BlockInfo) { + (engine_chain_block(25), engine_chain_block(25)) + } + + /// A forkchoice update that moved the head onto delegated heads and that the execution layer + /// rejects while it still holds the engine's finalized block restores the engine's own + /// forkchoice and skips the heads permanently. The canonical block at the finalized height, + /// after the rejected head became canonical, tells whether the delegate's chain contains the + /// finalized block. #[rstest::rstest] - #[case::reorg(other_chain_block(20))] - #[case::extension(engine_chain_block(25))] + #[case::fork_below_finality( + reorg_at_unsafe_head(), + ElHas::OtherChain, + DelegatedRejection::ChainBelowFinality + )] + #[case::labels_on_the_finalized_chain( + unproven_extension(), + ElHas::EngineChain, + DelegatedRejection::HeadLabels + )] #[tokio::test] - async fn rejected_forkchoice_for_delegated_heads_resets_the_engine( - #[case] delegated: L2BlockInfo, + async fn rejected_head_is_restored_and_skipped( + #[case] (local_safe, cross_safe): (L2BlockInfo, L2BlockInfo), + #[case] canonical_after_rejection: ElHas, + #[case] expected: DelegatedRejection, ) { let mut state = engine_state_with_safe_labels(15, 15); let state_before = state; - let err = run_delegated(&mut state, delegated, delegated, expect_invalid_forkchoice_state) - .await - .unwrap_err(); + let err = run_delegated(&mut state, local_safe, cross_safe, |l2| { + expect_invalid_forkchoice_state(l2); + el_holds_finalized(l2); + canonical_at_finalized_height(l2, canonical_after_rejection); + expect_restore(l2, &state_before, PayloadStatusEnum::Valid); + }) + .await + .unwrap_err(); + + assert!( + matches!( + err, + ConsolidateTaskError::DelegatedForkchoiceRejected { rejection, .. } + if rejection == expected + ), + "got {err:?}" + ); + assert!(expected.is_permanent()); + assert_eq!(err.severity(), EngineTaskErrorSeverity::Skip); + assert_eq!(state, state_before); + } + + /// Delegated heads below the unsafe head that the execution layer has, consolidated with an + /// advancing cross-safe head. + fn consolidation_lookups(l2: &RpcMock) { + l2.expect_block_by_number(10, ElHas::EngineChain.block(10)); + l2.expect_block_header_by_number(8, ElHas::EngineChain.block(8)); + } + + /// A rejected update that moved only the safe label, from consolidation or from a lagging + /// delegate, left the execution layer's chain alone: nothing is restored or looked up, and the + /// heads are skipped for this poll only, since the execution layer may store the safe block + /// later. + #[rstest::rstest] + #[case::consolidation( + engine_chain_block(10), + engine_chain_block(8), + 7, + 7, + consolidation_lookups + )] + #[case::lagging_delegate( + engine_chain_block(10), + engine_chain_block(9), + 12, + 8, + el_has_local_safe_10_and_cross_safe_9 + )] + #[tokio::test] + async fn rejected_safe_label_is_skipped_for_the_poll_without_restoring( + #[case] local_safe: L2BlockInfo, + #[case] cross_safe: L2BlockInfo, + #[case] engine_local_safe: u64, + #[case] engine_safe: u64, + #[case] expect_lookups: fn(&RpcMock), + ) { + let mut state = engine_state_with_safe_labels(engine_local_safe, engine_safe); + let state_before = state; + + let err = run_delegated(&mut state, local_safe, cross_safe, |l2| { + expect_lookups(l2); + expect_invalid_forkchoice_state(l2); + }) + .await + .unwrap_err(); + + assert!( + matches!( + err, + ConsolidateTaskError::DelegatedForkchoiceRejected { + rejection: DelegatedRejection::SafeLabel, + .. + } + ), + "got {err:?}" + ); + assert_eq!(err.severity(), EngineTaskErrorSeverity::Skip); + assert_eq!(state, state_before); + } + + /// The execution layer no longer holds the engine's finalized block, so restoring cannot help. + fn el_lost_finality(l2: &RpcMock) { + l2.expect_block_header_by_hash(engine_chain_block(5).block_info.hash, None); + } + + /// The execution layer lost the engine's finalized block, so restoring cannot help. + fn after_rejection_el_lost_finality(l2: &RpcMock, _: &EngineState) { + el_lost_finality(l2); + } + + /// The execution layer holds the finalized block but rejects the engine's own forkchoice too. + fn restore_rejected(l2: &RpcMock, _: &EngineState) { + el_holds_finalized(l2); + canonical_at_finalized_height(l2, ElHas::OtherChain); + expect_invalid_forkchoice_state(l2); + } - assert_eq!(err.severity(), EngineTaskErrorSeverity::Reset, "got {err:?}"); + /// The execution layer answers `SYNCING` to the engine's own forkchoice, so its canonical + /// chain stays on the delegate's head. + fn restore_syncing(l2: &RpcMock, state: &EngineState) { + el_holds_finalized(l2); + canonical_at_finalized_height(l2, ElHas::OtherChain); + expect_restore(l2, state, PayloadStatusEnum::Syncing); + } + + /// A rejected update that moved the head resets the engine when restoring cannot help: the + /// execution layer lost the finalized block, rejects the engine's own forkchoice as well, or + /// answers it `SYNCING` and so does not apply it. + #[rstest::rstest] + #[case::el_lost_finality(after_rejection_el_lost_finality)] + #[case::restore_rejected(restore_rejected)] + #[case::restore_syncing(restore_syncing)] + #[tokio::test] + async fn rejected_head_resets_when_restoring_cannot_help( + #[case] expect_after_rejection: fn(&RpcMock, &EngineState), + ) { + let (local_safe, cross_safe) = reorg_at_unsafe_head(); + let mut state = engine_state_with_safe_labels(15, 15); + let state_before = state; + + let err = run_delegated(&mut state, local_safe, cross_safe, |l2| { + expect_invalid_forkchoice_state(l2); + expect_after_rejection(l2, &state_before); + }) + .await + .unwrap_err(); + + assert!( + matches!( + err, + ConsolidateTaskError::ForkchoiceUpdateFailed( + SynchronizeTaskError::InvalidForkchoiceState + ) + ), + "got {err:?}" + ); + assert_eq!(err.severity(), Reset); assert_eq!(state, state_before); } + /// A finalized lookup that fails after a rejection, by hash or at the finalized height, + /// changes nothing and is retried on the failure backoff, which sends the delegated heads + /// again. + fn finalized_lookup_by_hash_fails(l2: &RpcMock) { + l2.expect_error("eth_getBlockByHash"); + } + + fn canonical_lookup_at_finalized_height_fails(l2: &RpcMock) { + el_holds_finalized(l2); + l2.expect_error("eth_getBlockByNumber"); + } + + #[rstest::rstest] + #[case::by_hash(finalized_lookup_by_hash_fails)] + #[case::at_finalized_height(canonical_lookup_at_finalized_height_fails)] + #[tokio::test] + async fn failed_finalized_lookup_after_a_rejection_is_retried( + #[case] expect_failing_lookup: fn(&RpcMock), + ) { + let (local_safe, cross_safe) = reorg_at_unsafe_head(); + let mut state = engine_state_with_safe_labels(15, 15); + let state_before = state; + + let err = run_delegated(&mut state, local_safe, cross_safe, |l2| { + expect_invalid_forkchoice_state(l2); + expect_failing_lookup(l2); + }) + .await + .unwrap_err(); + + assert!(matches!(err, ConsolidateTaskError::FailedToFetchFinalizedBlock(_)), "got {err:?}"); + assert_eq!(err.severity(), Temporary); + assert_eq!(state, state_before); + } + + /// An update that moves only the local-safe label leaves the engine's own forkchoice, so its + /// rejection is not blamed on the delegate and resets the engine without further calls. + #[tokio::test] + async fn rejected_forkchoice_that_moved_no_forkchoice_label_resets_the_engine() { + let mut state = engine_state_with_safe_labels(9, 7); + let state_before = state; + + let err = run_delegated(&mut state, engine_chain_block(10), engine_chain_block(6), |l2| { + l2.expect_block_by_number(10, ElHas::Missing.block(10)); + expect_invalid_forkchoice_state(l2); + }) + .await + .unwrap_err(); + + assert_eq!(err.severity(), Reset, "got {err:?}"); + assert_eq!(state, state_before); + } + + /// A cross-safe head below the safe label keeps the label without a lookup at the label's + /// height: a label the execution layer applied is on its canonical chain, so no lookup there + /// can show it abandoned. + #[tokio::test] + async fn cross_safe_head_below_the_safe_label_keeps_it_without_a_lookup() { + let (local_safe, cross_safe) = (engine_chain_block(20), engine_chain_block(12)); + let mut state = TestEngineStateBuilder::new() + .with_unsafe_head(engine_chain_block(20)) + .with_local_safe_head(engine_chain_block(15)) + .with_safe_head(engine_chain_block(15)) + .with_finalized_head(engine_chain_block(5)) + .build(); + let expected = state.sync_state.apply_update(EngineSyncStateUpdate { + local_safe_head: Some(local_safe), + ..Default::default() + }); + + run_delegated(&mut state, local_safe, cross_safe, |l2| { + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected); + } + + /// After a `SYNCING` answer the execution layer's canonical blocks may predate its sync, even + /// though its sync finished once, so a different block at the safe label's height does not + /// replace the label: nothing is looked up there. + #[tokio::test] + async fn safe_label_is_kept_while_the_el_did_not_apply_the_last_forkchoice() { + let (local_safe, cross_safe) = (engine_chain_block(20), engine_chain_block(12)); + let mut state = TestEngineStateBuilder::new() + .with_unsafe_head(engine_chain_block(20)) + .with_local_safe_head(other_chain_block(15)) + .with_safe_head(other_chain_block(15)) + .with_finalized_head(engine_chain_block(5)) + .with_el_applied_last_forkchoice(false) + .build(); + let expected = state.sync_state.apply_update(EngineSyncStateUpdate { + local_safe_head: Some(local_safe), + ..Default::default() + }); + + run_delegated(&mut state, local_safe, cross_safe, |l2| { + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected); + } + /// A reorg whose cross-safe head breaks finality is skipped before the execution layer is /// asked: the cross-safe head cannot become the safe label, and the old safe label would sit /// above the new unsafe head. @@ -1740,6 +2136,9 @@ mod tests { assert_eq!(state.sync_state, expected); } + /// A cross-safe head at one of the engine's labels is not looked up. + const fn no_lookup(_: &RpcMock) {} + /// A cross-safe head that is the engine's safe label leaves it in place without asking the /// execution layer. One that differs from the safe label at its height is checked against the /// execution layer, which has the engine chain's block there, so the safe label stays too. diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/mod.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/mod.rs index 12ae6c8ba16..57c5d35ea5d 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/mod.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/mod.rs @@ -6,8 +6,10 @@ pub use task::{ }; mod synchronize; -pub use synchronize::SynchronizeTaskError; -pub(super) use synchronize::{synchronize, synchronize_unconditionally}; +pub(super) use synchronize::{ + ForkchoiceReply, restore_forkchoice, synchronize, synchronize_unconditionally, +}; +pub use synchronize::{SafeLabel, SynchronizeTaskError}; mod insert; pub(super) use insert::insert_payload; @@ -26,8 +28,8 @@ pub use canonicalize::{CanonicalizeTask, CanonicalizeTaskError}; mod consolidate; pub use consolidate::{ - ConsolidateInput, ConsolidateTask, ConsolidateTaskError, DelegatedSafeHeads, - InconsistentSafeHeadsError, + ConsolidateInput, ConsolidateTask, ConsolidateTaskError, DelegatedRejection, + DelegatedSafeHeads, InconsistentSafeHeadsError, }; mod finalize; diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/error.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/error.rs index a244a93326b..bef19d0c284 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/error.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/error.rs @@ -5,12 +5,35 @@ use alloy_rpc_types_engine::PayloadStatusEnum; use alloy_transport::{RpcError, TransportErrorKind}; use thiserror::Error; +/// A safe label of the engine's sync state. +#[derive(Debug, Clone, Copy, PartialEq, Eq, derive_more::Display)] +pub enum SafeLabel { + /// The local-safe head. + #[display("local-safe")] + LocalSafe, + /// The safe head. + #[display("safe")] + Safe, +} + /// An error that occurs when synchronizing the execution layer's forkchoice with the engine state. #[derive(Debug, Error)] pub enum SynchronizeTaskError { /// The forkchoice update call to the engine api failed. #[error("Forkchoice update engine api call failed due to an RPC error: {0}")] ForkchoiceUpdateFailed(RpcError), + /// A safe label would sit below the finalized head. + #[error( + "Invalid forkchoice state: {label} head {label_number} is behind finalized head {finalized_number}" + )] + FinalizedAheadOfSafe { + /// The number of the finalized head. + finalized_number: u64, + /// The safe label behind the finalized head. + label: SafeLabel, + /// The number of that safe label. + label_number: u64, + }, /// The finalized head is behind the unsafe head. #[error("Invalid forkchoice state: unsafe head {0} is ahead of finalized head {1}")] FinalizedAheadOfUnsafe(u64, u64), @@ -26,6 +49,9 @@ impl EngineTaskError for SynchronizeTaskError { fn severity(&self) -> EngineTaskErrorSeverity { match self { Self::FinalizedAheadOfUnsafe(_, _) => EngineTaskErrorSeverity::Critical, + // A reset rebuilds the labels, so a label update that would break finality is + // recoverable without stopping the node. + Self::FinalizedAheadOfSafe { .. } | Self::InvalidForkchoiceState | Self::UnexpectedPayloadStatus(PayloadStatusEnum::Invalid { .. }) => { EngineTaskErrorSeverity::Reset diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs index fc28ccf0026..e9ef6c5fa1d 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs @@ -1,11 +1,21 @@ //! The `engine_forkchoiceUpdated` call, with no attributes, that the engine tasks share. use crate::{ - EngineClient, EngineState, EngineSyncState, SynchronizeTaskError, state::EngineSyncStateUpdate, + EngineClient, EngineState, EngineSyncState, SafeLabel, SynchronizeTaskError, + state::EngineSyncStateUpdate, }; use alloy_rpc_types_engine::{INVALID_FORK_CHOICE_STATE_ERROR, PayloadStatusEnum}; use op_alloy_provider::ext::engine::OpEngineApi; +/// How the execution layer answered a forkchoice update it accepted. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(in crate::task_queue) enum ForkchoiceReply { + /// The execution layer does not have the head block yet and is syncing toward it. + Syncing, + /// The execution layer applied the forkchoice. + Valid, +} + /// Applies `state_update` to the engine's sync state and synchronizes the execution layer's /// forkchoice with it, using an `engine_forkchoiceUpdated` call without payload attributes. An /// update that leaves the labels unchanged is skipped. @@ -38,7 +48,7 @@ pub(in crate::task_queue) async fn synchronize( return Ok(()); } - send_forkchoice(client, state, new_sync_state).await + send_and_apply_forkchoice(client, state, new_sync_state).await } /// Like [`synchronize`], but sends the forkchoice update even when it leaves the labels unchanged. @@ -58,12 +68,12 @@ pub(in crate::task_queue) async fn synchronize_unconditionally( state_update: EngineSyncStateUpdate, ) -> Result<(), SynchronizeTaskError> { let new_sync_state = state.sync_state.apply_update(state_update); - send_forkchoice(client, state, new_sync_state).await + send_and_apply_forkchoice(client, state, new_sync_state).await } /// Sends the forkchoice of `new_sync_state` and, once the execution layer accepts it, makes it the /// engine's sync state. -async fn send_forkchoice( +async fn send_and_apply_forkchoice( client: &EngineClient, state: &mut EngineState, new_sync_state: EngineSyncState, @@ -78,15 +88,84 @@ async fn send_forkchoice( )); } + ensure_safe_labels_not_behind_finality(&new_sync_state)?; + // Send the forkchoice update through the input. let forkchoice = new_sync_state.create_forkchoice_state(); tracing::Span::current().record("forkchoice", tracing::field::debug(forkchoice)); + let reply = send_forkchoice(client, &new_sync_state).await?; + record_forkchoice_reply(state, reply); + + // Apply the new sync state to the engine state. + state.sync_state = new_sync_state; + + Ok(()) +} + +/// Refuses a sync state whose safe or local-safe head sits below its finalized head. +fn ensure_safe_labels_not_behind_finality( + sync_state: &EngineSyncState, +) -> Result<(), SynchronizeTaskError> { + let labels = [ + (SafeLabel::LocalSafe, sync_state.local_safe_head()), + (SafeLabel::Safe, sync_state.safe_head()), + ]; + for (label, head) in labels { + if let Err(violation) = sync_state.ensure_not_behind_finality(head.block_info.number) { + debug!(target: "engine", %violation, %label, "Safe label would move below finality"); + return Err(SynchronizeTaskError::FinalizedAheadOfSafe { + finalized_number: sync_state.finalized_head().block_info.number, + label, + label_number: head.block_info.number, + }); + } + } + Ok(()) +} - // Handle the forkchoice update result. +/// Sends the engine's own `sync_state` forkchoice again, to undo a forkchoice update that the +/// execution layer made canonical before rejecting it. Neither the engine state nor its record of +/// the execution layer's sync changes. +pub(in crate::task_queue) async fn restore_forkchoice( + client: &EngineClient, + sync_state: &EngineSyncState, +) -> Result { + warn!( + target: "engine", + forkchoice = ?sync_state.create_forkchoice_state(), + "Restoring the engine's forkchoice on the execution layer" + ); + let reply = send_forkchoice(client, sync_state).await.inspect_err(|err| { + warn!(target: "engine", ?err, "Restoring the engine's forkchoice failed"); + })?; + info!(target: "engine", ?reply, "Restored the engine's forkchoice on the execution layer"); + Ok(reply) +} + +/// Records how the execution layer answered the engine's forkchoice: whether it applied it, and +/// whether its sync has finished, which only a `VALID` answer marks and nothing clears. +pub(in crate::task_queue) fn record_forkchoice_reply( + state: &mut EngineState, + reply: ForkchoiceReply, +) { + state.el_applied_last_forkchoice = reply == ForkchoiceReply::Valid; + if reply == ForkchoiceReply::Valid && !state.el_sync_finished { + info!(target: "engine", "Finished execution layer sync."); + state.el_sync_finished = true; + } +} + +/// Sends `sync_state`'s forkchoice to the execution layer, with no payload attributes, and returns +/// whether it applied the forkchoice or is still syncing toward its head. Neither the engine state +/// nor its record of the execution layer's sync changes. +pub(in crate::task_queue) async fn send_forkchoice( + client: &EngineClient, + sync_state: &EngineSyncState, +) -> Result { // NOTE: it doesn't matter which version we use here, because we're not sending any payload // attributes. The forkchoice updated call is version agnostic if no payload attributes are // provided. - let response = client.fork_choice_updated_v3(forkchoice, None).await; + let response = client.fork_choice_updated_v3(sync_state.create_forkchoice_state(), None).await; let valid_response = response.map_err(|e| { // Fatal forkchoice update error. @@ -99,24 +178,13 @@ async fn send_forkchoice( })?; match &valid_response.payload_status.status { - PayloadStatusEnum::Valid => { - state.el_applied_last_forkchoice = true; - if !state.el_sync_finished { - info!(target: "engine", "Finished execution layer sync."); - state.el_sync_finished = true; - } - } + PayloadStatusEnum::Valid => Ok(ForkchoiceReply::Valid), // If we're not building a new payload, we're driving EL sync. PayloadStatusEnum::Syncing => { - state.el_applied_last_forkchoice = false; - debug!(target: "engine", "Attempting to update forkchoice state while EL syncing") + debug!(target: "engine", "Attempting to update forkchoice state while EL syncing"); + Ok(ForkchoiceReply::Syncing) } // Other codes are not expected. - status => return Err(SynchronizeTaskError::UnexpectedPayloadStatus(status.clone())), + status => Err(SynchronizeTaskError::UnexpectedPayloadStatus(status.clone())), } - - // Apply the new sync state to the engine state. - state.sync_state = new_sync_state; - - Ok(()) } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/mod.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/mod.rs index 9de83da3a5c..6dcf6e969e8 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/mod.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/mod.rs @@ -1,10 +1,12 @@ //! The forkchoice engine update shared by the engine tasks, and its error type. mod helper; -pub(in crate::task_queue) use helper::{synchronize, synchronize_unconditionally}; +pub(in crate::task_queue) use helper::{ + ForkchoiceReply, restore_forkchoice, synchronize, synchronize_unconditionally, +}; mod error; -pub use error::SynchronizeTaskError; +pub use error::{SafeLabel, SynchronizeTaskError}; #[cfg(test)] mod tests; diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/tests.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/tests.rs index 64909cb394f..17f35feeb31 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/tests.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/tests.rs @@ -2,6 +2,7 @@ use super::{synchronize, synchronize_unconditionally}; use crate::{ EngineState, EngineSyncState, EngineSyncStateUpdate, EngineTaskError, EngineTaskErrorSeverity::{self, Reset, Temporary}, + SafeLabel, SynchronizeTaskError, test_utils::{RpcMock, TestEngineStateBuilder, test_block_info, test_engine_client}, }; use alloy_rpc_types_engine::{ForkchoiceState, INVALID_FORK_CHOICE_STATE_ERROR, PayloadStatusEnum}; @@ -151,3 +152,34 @@ async fn synchronize_applies_labels_only_when_accepted( l1.assert_finished(); l2.assert_finished(); } + +/// An update that would put a safe label below the finalized head is refused before the +/// execution layer is asked, leaves the labels unchanged, and resets the engine rather than +/// stopping it. +#[rstest] +#[case::safe_head(EngineSyncStateUpdate { safe_head: Some(test_block_info(4)), ..Default::default() }, SafeLabel::Safe)] +#[case::local_safe_head(EngineSyncStateUpdate { local_safe_head: Some(test_block_info(4)), ..Default::default() }, SafeLabel::LocalSafe)] +#[tokio::test] +async fn synchronize_refuses_a_safe_label_below_finality( + #[case] update: EngineSyncStateUpdate, + #[case] expected_label: SafeLabel, +) { + let mut state = engine_state(); + let state_before = state; + let (client, l1, l2) = test_engine_client(Arc::new(RollupConfig::default())); + + let error = synchronize_unconditionally(&client, &mut state, update).await.unwrap_err(); + + assert!( + matches!( + error, + SynchronizeTaskError::FinalizedAheadOfSafe { finalized_number: 5, label, label_number: 4 } + if label == expected_label + ), + "{error:?}" + ); + assert_eq!(error.severity(), Reset); + assert_eq!(state, state_before); + l1.assert_finished(); + l2.assert_finished(); +} diff --git a/rust/kona/crates/node/engine/src/test_utils/blocks.rs b/rust/kona/crates/node/engine/src/test_utils/blocks.rs index 434d3bd01eb..12f31e4baf2 100644 --- a/rust/kona/crates/node/engine/src/test_utils/blocks.rs +++ b/rust/kona/crates/node/engine/src/test_utils/blocks.rs @@ -1,6 +1,6 @@ //! L2 blocks, and their RPC form as the execution layer serves them. -use super::l1_info_deposit; +use super::l1_info_deposit_at; use alloy_consensus::transaction::Recovered; use alloy_primitives::{Address, B256, Bytes}; use alloy_rpc_types_eth::{BlockTransactions, Header as RpcHeader}; @@ -13,15 +13,38 @@ pub type TestRpcBlock = alloy_rpc_types_eth::Block OpBlock { + test_l2_block_at_l1_origin(number, parent_hash, extra_data, 0) +} + +/// [`test_l2_block`] with its L1 origin at `l1_origin_number`. +pub fn test_l2_block_at_l1_origin( + number: u64, + parent_hash: B256, + extra_data: u8, + l1_origin_number: u64, +) -> OpBlock { let mut block = OpBlock::default(); block.header.extra_data = Bytes::from(vec![extra_data]); block.header.number = number; block.header.parent_hash = parent_hash; block.header.timestamp = number * 2; - block.body.transactions.push(l1_info_deposit()); + block.body.transactions.push(l1_info_deposit_at(l1_origin_number)); block } +/// Blocks `from..=to` built on `parent_hash` with [`test_l2_block`], all tagged `extra_data`, so +/// that chains with different tags share heights but not hashes. +pub fn test_l2_chain(parent_hash: B256, from: u64, to: u64, extra_data: u8) -> Vec { + let mut parent_hash = parent_hash; + (from..=to) + .map(|number| { + let block = test_l2_block(number, parent_hash, extra_data); + parent_hash = block.header.hash_slow(); + block + }) + .collect() +} + /// `block` as the execution layer serves it over RPC. pub fn test_rpc_block(block: &OpBlock) -> TestRpcBlock { let transactions = block diff --git a/rust/kona/crates/node/engine/src/test_utils/deposits.rs b/rust/kona/crates/node/engine/src/test_utils/deposits.rs index 78d60d4bf01..ea47ab2ca4b 100644 --- a/rust/kona/crates/node/engine/src/test_utils/deposits.rs +++ b/rust/kona/crates/node/engine/src/test_utils/deposits.rs @@ -1,14 +1,20 @@ //! Deposit transaction fixtures. -use alloy_primitives::Sealed; +use alloy_primitives::{B256, Sealed}; use kona_protocol::L1BlockInfoBedrock; use op_alloy_consensus::{OpTxEnvelope, TxDeposit}; /// The L1-info deposit a block needs for `L2BlockInfo::from_block_and_genesis` to read its L1 /// origin. pub fn l1_info_deposit() -> OpTxEnvelope { + l1_info_deposit_at(0) +} + +/// The L1-info deposit of a block whose L1 origin is at `l1_origin_number`. +pub fn l1_info_deposit_at(l1_origin_number: u64) -> OpTxEnvelope { OpTxEnvelope::Deposit(Sealed::new(TxDeposit { - input: L1BlockInfoBedrock::default().encode_calldata(), + input: L1BlockInfoBedrock::new_from_number_and_block_hash(l1_origin_number, B256::ZERO) + .encode_calldata(), ..Default::default() })) } diff --git a/rust/kona/crates/node/engine/src/test_utils/engine_state.rs b/rust/kona/crates/node/engine/src/test_utils/engine_state.rs index a1399be192e..f7d59f2fee1 100644 --- a/rust/kona/crates/node/engine/src/test_utils/engine_state.rs +++ b/rust/kona/crates/node/engine/src/test_utils/engine_state.rs @@ -76,7 +76,6 @@ impl TestEngineStateBuilder { } /// Sets whether the execution layer applied the last forkchoice update. - #[allow(dead_code)] pub const fn with_el_applied_last_forkchoice(mut self, applied: bool) -> Self { self.el_applied_last_forkchoice = applied; self diff --git a/rust/kona/crates/node/engine/src/test_utils/mod.rs b/rust/kona/crates/node/engine/src/test_utils/mod.rs index bfde4784cac..8ff59f8d831 100644 --- a/rust/kona/crates/node/engine/src/test_utils/mod.rs +++ b/rust/kona/crates/node/engine/src/test_utils/mod.rs @@ -2,13 +2,15 @@ mod attributes; pub use attributes::TestAttributesBuilder; mod blocks; -pub use blocks::{TestRpcBlock, test_l2_block, test_rpc_block}; +pub use blocks::{ + TestRpcBlock, test_l2_block, test_l2_block_at_l1_origin, test_l2_chain, test_rpc_block, +}; mod deposits; -pub use deposits::l1_info_deposit; +pub use deposits::{l1_info_deposit, l1_info_deposit_at}; mod rpc; -pub use rpc::{RpcMock, expect_sync_start, test_engine_client}; +pub use rpc::{RpcMock, expect_sync_start, expect_sync_start_walk, test_engine_client}; mod engine_state; pub use engine_state::TestEngineStateBuilder; diff --git a/rust/kona/crates/node/engine/src/test_utils/rpc.rs b/rust/kona/crates/node/engine/src/test_utils/rpc.rs index cda70c224a6..0247fce15d6 100644 --- a/rust/kona/crates/node/engine/src/test_utils/rpc.rs +++ b/rust/kona/crates/node/engine/src/test_utils/rpc.rs @@ -1,8 +1,12 @@ //! Scripted RPC expectations over Alloy's mock transport. -use crate::{EngineClient, test_utils::TestRpcBlock}; +use crate::{ + EngineClient, + test_utils::{TestRpcBlock, test_rpc_block}, +}; use alloy_json_rpc::{ErrorPayload, RequestPacket, ResponsePacket}; use alloy_network::Network; +use alloy_primitives::B256; use alloy_provider::RootProvider; use alloy_rpc_client::RpcClient; use alloy_rpc_types_engine::{ @@ -14,6 +18,7 @@ use alloy_transport::{ mock::{Asserter, MockResponse, MockTransport}, }; use kona_genesis::RollupConfig; +use op_alloy_consensus::OpBlock; use serde::Serialize; use serde_json::Value; use std::{ @@ -80,6 +85,12 @@ impl RpcMock { ); } + /// Expects a header-only `eth_getBlockByHash` call for `hash` and answers it with `block`, or + /// with null for `None`. + pub fn expect_block_header_by_hash(&self, hash: B256, block: Option) { + self.expect_params("eth_getBlockByHash", serde_json::json!([hash, false]), block); + } + /// Expects an `engine_forkchoiceUpdatedV3` call for `forkchoice` without payload attributes and /// answers it with `status`. pub fn expect_forkchoice_updated( @@ -230,3 +241,30 @@ pub fn expect_sync_start( l1.expect("eth_getBlockByHash", EthBlock::::default()); l2.expect_forkchoice_updated(forkchoice, PayloadStatusEnum::Valid); } + +/// Expects a sync-start walk: the execution layer answering `finalized` for its finalized block +/// and `latest` for its safe and latest blocks, an L1 origin lookup answered by `l1_origin`, and +/// the parent lookups of the walk answered by `parents` in order. +pub fn expect_sync_start_walk( + l1: &RpcMock, + l2: &RpcMock, + finalized: &OpBlock, + latest: &OpBlock, + l1_origin_is_available: bool, + parents: &[&OpBlock], +) { + for (label, block) in [("finalized", finalized), ("safe", latest), ("latest", latest)] { + l2.expect_params( + "eth_getBlockByNumber", + serde_json::json!([label, true]), + test_rpc_block(block), + ); + } + l1.expect( + "eth_getBlockByHash", + l1_origin_is_available.then(EthBlock::::default), + ); + for parent in parents { + l2.expect("eth_getBlockByHash", test_rpc_block(parent)); + } +} diff --git a/rust/kona/crates/node/service/src/actors/derivation/actor/core.rs b/rust/kona/crates/node/service/src/actors/derivation/actor/core.rs index 2b602e11ff9..c98e9b0c004 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/actor/core.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/actor/core.rs @@ -375,6 +375,9 @@ where self.attempt_derivation().await?; } + DerivationActorRequest::ProcessRejectedDelegatedSafeHeads { heads, .. } => { + debug!(target: "derivation", ?heads, "Ignoring rejected delegated safe heads while deriving"); + } DerivationActorRequest::ProcessEngineSyncCompletionRequest(safe_head) => { info!(target: "derivation", "Engine finished syncing, starting derivation."); self.derivation_state_machine diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs index b3d64c7202b..955a0aa345f 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs @@ -7,7 +7,8 @@ use async_trait::async_trait; use jsonrpsee::core::ClientError; use kona_derive::{ChainProvider, Signal}; use kona_engine::{ - DelegatedSafeHeads, EngineState, FinalityViolation, FinalizeBlockId, InconsistentSafeHeadsError, + DelegatedRejection, DelegatedSafeHeads, EngineState, FinalityViolation, FinalizeBlockId, + InconsistentSafeHeadsError, }; use kona_protocol::{BlockInfo, L2BlockInfo, SyncStatus}; use kona_rpc::RollupNodeApiClient; @@ -15,7 +16,7 @@ use thiserror::Error; use tokio::{ select, sync::{mpsc, watch}, - time, + time::{self, Instant}, }; /// How often the derivation delegate is polled for its sync status. @@ -41,6 +42,14 @@ impl SyncStatusSource for C { } } +/// How long safe heads stay held back after the engine found the delegate's chain below its +/// finality, unless the engine's finality moves first. Each retry costs one rejected forkchoice +/// update and a restore if the delegate is still on that chain; without retries, a delegate that +/// returned to the canonical chain with its finalized head above the engine's frozen safe head +/// would stay held back until the next reset. +pub(super) const CHAIN_BELOW_FINALITY_RETRY_INTERVAL: std::time::Duration = + std::time::Duration::from_secs(5 * 60); + /// The [`NodeActor`] for the delegate derivation sub-routine. /// /// This actor is responsible for receiving messages from [`NodeActor`]s and polls @@ -85,6 +94,9 @@ where engine_l2_safe_head: L2BlockInfo, /// Whether the engine sync has completed. This will only ever go from false -> true. has_engine_sync_completed: bool, + /// What the execution layer rejected and would reject again until the engine resets, so it is + /// not sent again before then. + rejected_by_engine: Option, /// Ticker driving periodic polls of the derivation delegate. delegated_derivation_ticker: time::Interval, @@ -125,6 +137,7 @@ where engine_state_rx, engine_l2_safe_head: L2BlockInfo::default(), has_engine_sync_completed: false, + rejected_by_engine: None, delegated_derivation_ticker, forwarded_lagging_safe_heads: None, last_rejection: None, @@ -320,7 +333,11 @@ where self.derivation_origin_tx.send_replace(Some(sync_status.current_l1)); let lagging_safe_heads = self.lagging_safe_heads_to_remember(safe_heads); - if lagging_safe_heads.is_some() && lagging_safe_heads == self.forwarded_lagging_safe_heads { + if self.is_rejected_by_engine(safe_heads) { + debug!(target: "derivation", ?safe_heads, "Not sending safe heads the engine rejected before its next reset"); + } else if lagging_safe_heads.is_some() && + lagging_safe_heads == self.forwarded_lagging_safe_heads + { debug!( target: "derivation", ?safe_heads, @@ -355,6 +372,55 @@ where Ok(()) } + /// What to hold back after the engine rejected `heads` for `rejection`. + fn rejected_by_engine_for( + &self, + heads: DelegatedSafeHeads, + rejection: DelegatedRejection, + ) -> RejectedByEngine { + match rejection { + DelegatedRejection::ChainBelowFinality => RejectedByEngine::ChainBelowFinality { + engine_finalized: self.engine_finalized_head(), + retry_at: Instant::now() + CHAIN_BELOW_FINALITY_RETRY_INTERVAL, + }, + DelegatedRejection::HeadLabels | DelegatedRejection::SafeLabel => { + RejectedByEngine::SafeHeads(Box::new(heads)) + } + } + } + + /// The finalized head the engine applied, once it established finality. + fn engine_finalized_head(&self) -> Option { + self.engine_state_rx.borrow().sync_state.established_finalized_head() + } + + /// Whether `safe_heads` are held back because the engine rejected them, or the delegate's + /// chain, since its last reset. + /// + /// A chain below finality is held back as a whole rather than by the blocks it reported: an + /// advancing delegate reports new blocks on every poll, but none of its finalized targets is + /// canonical, so the engine's finality cannot move while the delegate stays on that chain. The + /// engine's finality moving therefore shows the delegate's finalized target is canonical, and + /// lifts the hold; so does the retry interval passing. + fn is_rejected_by_engine(&mut self, safe_heads: DelegatedSafeHeads) -> bool { + match &self.rejected_by_engine { + Some(RejectedByEngine::SafeHeads(rejected)) => **rejected == safe_heads, + Some(RejectedByEngine::ChainBelowFinality { engine_finalized, retry_at }) => { + let current_finalized = self.engine_finalized_head(); + if current_finalized != *engine_finalized { + info!(target: "derivation", ?current_finalized, "Engine finality moved since it rejected the delegate's chain; sending its safe heads again"); + } else if Instant::now() >= *retry_at { + info!(target: "derivation", ?safe_heads, "Retrying the safe heads of a delegate whose chain the engine found below its finality"); + } else { + return true; + } + self.rejected_by_engine = None; + false + } + None => false, + } + } + async fn handle_derivation_delegation_actor_request( &mut self, request_type: DerivationActorRequest, @@ -369,8 +435,16 @@ where self.engine_l2_safe_head = *safe_head; self.has_engine_sync_completed = true; } + DerivationActorRequest::ProcessRejectedDelegatedSafeHeads { heads, rejection } => { + info!(target: "derivation", ?heads, %rejection, "Engine rejected delegated safe heads; not sending them again before its next reset"); + self.rejected_by_engine = Some(self.rejected_by_engine_for(*heads, rejection)); + } DerivationActorRequest::ProcessEngineSignalRequest(signal) => match *signal { - Signal::Reset(reset) => self.engine_l2_safe_head = reset.l2_safe_head, + // A reset rebuilds the engine's labels, after which rejected heads may apply. + Signal::Reset(reset) => { + self.engine_l2_safe_head = reset.l2_safe_head; + self.rejected_by_engine = None; + } signal => { debug!(target: "derivation", ?signal, "Ignoring signal while derivation delegation"); } @@ -384,6 +458,20 @@ where } } +/// What the execution layer rejected and would reject again until the engine resets. +#[derive(Debug, Clone, PartialEq, Eq)] +enum RejectedByEngine { + /// The delegate's chain, which does not contain the engine's finalized block. + ChainBelowFinality { + /// The engine's finalized head when it rejected the chain. + engine_finalized: Option, + /// When the delegate's safe heads are sent once more. + retry_at: Instant, + }, + /// These exact safe heads. + SafeHeads(Box), +} + #[derive(Error, Debug)] enum DerivationDelegationError { /// The delegate's finalized L2 block is ahead of its own safe head. diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs index b9e522b66a7..f2fcf80e808 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs @@ -1,15 +1,15 @@ use super::{ DelegateDerivationActor, - actor::DERIVATION_DELEGATE_POLL_INTERVAL, + actor::{CHAIN_BELOW_FINALITY_RETRY_INTERVAL, DERIVATION_DELEGATE_POLL_INTERVAL}, test_utils::{ScriptedDelegate, SyncStatusReply, sync_status}, }; use crate::{DerivationActorRequest, EngineActorRequest, NodeActor, QueuedDerivationEngineClient}; use alloy_eips::BlockNumHash; use alloy_primitives::B256; -use kona_derive::test_utils::TestChainProvider; +use kona_derive::{ResetSignal, Signal, test_utils::TestChainProvider}; use kona_engine::{ - ConsolidateInput, DelegatedSafeHeads, EngineState, EngineSyncState, EngineSyncStateUpdate, - FinalizeBlockId, + ConsolidateInput, DelegatedRejection, DelegatedSafeHeads, EngineState, EngineSyncState, + EngineSyncStateUpdate, FinalizeBlockId, }; use kona_protocol::{BlockInfo, L2BlockInfo, SyncStatus}; use rstest::rstest; @@ -80,6 +80,12 @@ impl Harness { self.engine_state.send_modify(|state| state.sync_state = sync_state); } + /// Hands the actor `request`, which it handles before its next poll. + async fn handle(&mut self, request: DerivationActorRequest) { + self.derivation_requests.send(request).await.unwrap(); + self.actor.step().await.unwrap(); + } + /// Publishes the engine's labels along with whether its execution layer applied the engine's /// last forkchoice. fn publish_engine_state(&self, sync_state: EngineSyncState, el_applied_last_forkchoice: bool) { @@ -441,6 +447,175 @@ async fn poll_forwards_local_safe_and_cross_safe_heads_separately( assert_eq!(harness.forwarded()[0], Forwarded::Safe(Box::new(expected_heads))); } +/// The safe heads of `status`. +fn safe_heads_of(status: &SyncStatus) -> DelegatedSafeHeads { + DelegatedSafeHeads::try_new(status.local_safe_l2, status.safe_l2).unwrap() +} + +/// What a poll of `status` forwards while its safe heads are rejected: the finalized block alone. +fn forwarded_without_safe_heads(status: &SyncStatus) -> Vec { + vec![Forwarded::Finalized(FinalizeBlockId::ByHash(status.finalized_l2.block_info.id()))] +} + +/// The engine's report that it rejected the safe heads of `status` for `rejection`. +fn rejected(status: &SyncStatus, rejection: DelegatedRejection) -> DerivationActorRequest { + DerivationActorRequest::ProcessRejectedDelegatedSafeHeads { + heads: Box::new(safe_heads_of(status)), + rejection, + } +} + +/// Safe heads the engine rejected permanently are not sent again while the delegate keeps +/// reporting them, but the finalized block still is; a reset rebuilds the engine's labels, after +/// which they are sent again. +#[tokio::test(start_paused = true)] +async fn rejected_safe_heads_are_not_sent_again_until_a_reset() { + let status = delegate_status(); + let mut harness = Harness::new((0..3).map(|_| Ok(status.clone()))); + harness.complete_engine_sync().await; + harness.actor.step().await.unwrap(); + assert_eq!(harness.forwarded(), forwarded_for(&status)); + harness.handle(rejected(&status, DelegatedRejection::HeadLabels)).await; + + harness.actor.step().await.unwrap(); + assert_eq!(harness.forwarded(), forwarded_without_safe_heads(&status)); + + let reset = Signal::Reset(ResetSignal { l2_safe_head: L2BlockInfo::default() }); + harness.handle(DerivationActorRequest::ProcessEngineSignalRequest(Box::new(reset))).await; + harness.actor.step().await.unwrap(); + assert_eq!(harness.forwarded(), forwarded_for(&status)); +} + +/// Only the exact rejected safe heads are held back: a status with another cross-safe head is +/// sent, since the rejection may have been that head's alone. +#[tokio::test(start_paused = true)] +async fn safe_heads_other_than_the_rejected_ones_are_sent() { + let rejected_status = delegate_status(); + let next = sync_status( + canonical_l1_block(CURRENT_L1), + l2_block(20, SAFE_ORIGIN), + l2_block(18, SAFE_ORIGIN), + l2_block(15, FINALIZED_ORIGIN), + ); + let mut harness = Harness::new([Ok(next.clone())]); + harness.complete_engine_sync().await; + harness.handle(rejected(&rejected_status, DelegatedRejection::HeadLabels)).await; + + harness.actor.step().await.unwrap(); + + assert_eq!(harness.forwarded(), forwarded_for(&next)); +} + +/// A status on a delegate chain with blocks tagged `chain`, with its local-safe and safe heads at +/// `safe`, its unsafe head two blocks above and its finalized block at `finalized`. Each block's +/// parent is the block below it on the same chain. +fn chain_status(chain: u8, safe: u64, finalized: u64) -> SyncStatus { + let block = |number: u64, l1_origin: u64| L2BlockInfo { + block_info: BlockInfo { + hash: chain_hash(chain, number), + number, + parent_hash: chain_hash(chain, number - 1), + ..Default::default() + }, + ..l2_block(number, l1_origin) + }; + SyncStatus { + unsafe_l2: block(safe + 2, SAFE_ORIGIN), + ..sync_status( + canonical_l1_block(CURRENT_L1), + block(safe, SAFE_ORIGIN), + block(safe, SAFE_ORIGIN), + block(finalized, FINALIZED_ORIGIN), + ) + } +} + +fn chain_hash(chain: u8, number: u64) -> B256 { + let mut hash = B256::with_last_byte(number as u8); + hash.0[0] = chain; + hash +} + +const FORK: u8 = 0xf0; +const CANONICAL: u8 = 0xc0; + +/// The engine's finalized head when it rejects a delegate's chain below its finality. +fn engine_finalized_below_the_fork() -> EngineSyncState { + engine_finalized_at(l2_block(10, FINALIZED_ORIGIN)) +} + +/// Polls `rejected_status`, whose chain the engine then reports below its finality. +async fn reject_chain_below_finality(harness: &mut Harness, rejected_status: &SyncStatus) { + harness.complete_engine_sync_with(engine_finalized_below_the_fork()).await; + harness.actor.step().await.unwrap(); + assert_eq!(harness.forwarded(), forwarded_for(rejected_status)); + harness.handle(rejected(rejected_status, DelegatedRejection::ChainBelowFinality)).await; +} + +/// After the engine found a delegate's chain below its finality, the delegate's safe heads stay +/// held back while it advances on that chain as a live delegate does: its unsafe head several +/// blocks per poll, its safe heads a span batch at a time and its finality an epoch at a time, so +/// no status shares a block with an earlier one. Its finalized block is still sent. +#[tokio::test(start_paused = true)] +async fn delegate_advancing_on_a_rejected_chain_is_held_back() { + let rejected_status = chain_status(FORK, 20, 15); + let advancing = + [chain_status(FORK, 26, 15), chain_status(FORK, 32, 15), chain_status(FORK, 38, 26)]; + let mut harness = Harness::new( + std::iter::once(&rejected_status).chain(&advancing).map(|status| Ok(status.clone())), + ); + reject_chain_below_finality(&mut harness, &rejected_status).await; + + for status in &advancing { + harness.actor.step().await.unwrap(); + assert_eq!(harness.forwarded(), forwarded_without_safe_heads(status)); + } +} + +/// The engine's finality moves only to a canonical finalized target, which shows the delegate is +/// back on the engine's chain, so its safe heads are sent again. +#[tokio::test(start_paused = true)] +async fn engine_finality_moving_lifts_the_hold_on_a_rejected_chain() { + let rejected_status = chain_status(FORK, 20, 15); + let canonical = chain_status(CANONICAL, 26, 20); + let mut harness = + Harness::new([Ok(rejected_status.clone()), Ok(canonical.clone()), Ok(canonical.clone())]); + reject_chain_below_finality(&mut harness, &rejected_status).await; + harness.actor.step().await.unwrap(); + assert_eq!(harness.forwarded(), forwarded_without_safe_heads(&canonical)); + + harness.publish_engine_sync_state(engine_finalized_at(canonical.finalized_l2)); + harness.actor.step().await.unwrap(); + + assert_eq!(harness.forwarded(), forwarded_for(&canonical)); +} + +/// Without the engine's finality moving, the delegate's safe heads are sent once more after the +/// retry interval, and held back again if the engine rejects the chain again. +#[tokio::test(start_paused = true)] +async fn delegate_on_a_rejected_chain_is_retried_after_the_interval() { + let rejected_status = chain_status(FORK, 20, 15); + let (held, retried, held_again) = + (chain_status(FORK, 26, 15), chain_status(FORK, 32, 15), chain_status(FORK, 38, 15)); + let mut harness = Harness::new([ + Ok(rejected_status.clone()), + Ok(held.clone()), + Ok(retried.clone()), + Ok(held_again.clone()), + ]); + reject_chain_below_finality(&mut harness, &rejected_status).await; + harness.actor.step().await.unwrap(); + assert_eq!(harness.forwarded(), forwarded_without_safe_heads(&held)); + + time::sleep(CHAIN_BELOW_FINALITY_RETRY_INTERVAL).await; + harness.actor.step().await.unwrap(); + assert_eq!(harness.forwarded(), forwarded_for(&retried)); + + harness.handle(rejected(&retried, DelegatedRejection::ChainBelowFinality)).await; + harness.actor.step().await.unwrap(); + assert_eq!(harness.forwarded(), forwarded_without_safe_heads(&held_again)); +} + /// A zero local safe head is a label the delegate has not derived or does not report, so it does /// not put the safe head out of order and the status is forwarded. #[tokio::test(start_paused = true)] @@ -465,17 +640,12 @@ fn engine_safe_at(safe: u64) -> EngineSyncState { }) } -/// What a poll of `status` forwards when its safe head is held back: the finalized block alone. -fn finalized_only(status: &SyncStatus) -> Vec { - vec![Forwarded::Finalized(FinalizeBlockId::ByHash(status.finalized_l2.block_info.id()))] -} - /// Safe heads behind the engine's are forwarded once while the execution layer applied the /// engine's forkchoice, since the engine's answer to it cannot change; finalized still goes out /// on every poll. A moved engine safe head, or an execution layer that did not apply the last /// forkchoice, sends it again. #[rstest] -#[case::same_engine_view(true, 25, finalized_only)] +#[case::same_engine_view(true, 25, forwarded_without_safe_heads)] #[case::el_not_applied(false, 25, forwarded_for)] #[case::engine_safe_moved(true, 26, forwarded_for)] #[tokio::test(start_paused = true)] diff --git a/rust/kona/crates/node/service/src/actors/derivation/request.rs b/rust/kona/crates/node/service/src/actors/derivation/request.rs index 9482eef5582..d600e3ba1f1 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/request.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/request.rs @@ -1,4 +1,5 @@ use kona_derive::Signal; +use kona_engine::{DelegatedRejection, DelegatedSafeHeads}; use kona_protocol::{BlockInfo, L2BlockInfo}; use thiserror::Error; @@ -33,4 +34,12 @@ pub enum DerivationActorRequest { ProcessFinalizedL1Block(Box), /// Request to process the provided L1 head block update. ProcessL1HeadUpdateRequest(Box), + /// The execution layer rejected a forkchoice update for these delegated safe heads, and would + /// reject them again until the engine resets. + ProcessRejectedDelegatedSafeHeads { + /// The rejected delegated safe heads. + heads: Box, + /// Why the execution layer rejected them. + rejection: DelegatedRejection, + }, } diff --git a/rust/kona/crates/node/service/src/actors/engine/actor.rs b/rust/kona/crates/node/service/src/actors/engine/actor.rs index 546358ca87c..d7f967862d2 100644 --- a/rust/kona/crates/node/service/src/actors/engine/actor.rs +++ b/rust/kona/crates/node/service/src/actors/engine/actor.rs @@ -4,9 +4,9 @@ use crate::{ use async_trait::async_trait; use kona_derive::{ResetSignal, Signal}; use kona_engine::{ - BuildTask, CanonicalizeTask, ConsolidateTask, Engine, EngineActorRequest, EngineClient, - EngineTask, EngineTaskError, EngineTaskErrorSeverity, EngineTaskErrors, FinalizeTask, - ImportedBlockSink, InsertTask, SealTask, + BuildTask, CanonicalizeTask, ConsolidateTask, ConsolidateTaskError, Engine, EngineActorRequest, + EngineClient, EngineTask, EngineTaskError, EngineTaskErrorSeverity, EngineTaskErrors, + FinalizeTask, ImportedBlockSink, InsertTask, SealTask, }; use kona_genesis::RollupConfig; use kona_protocol::L2BlockInfo; @@ -234,7 +234,10 @@ where self.defer_retry(); return Ok(()); } - EngineTaskErrorSeverity::Skip => self.log_skipped_task(&err), + EngineTaskErrorSeverity::Skip => { + self.log_skipped_task(&err); + self.forward_permanently_rejected_delegated_safe_heads(&err).await?; + } } } } @@ -252,6 +255,30 @@ where Ok(()) } + /// Tells derivation about delegated safe heads whose forkchoice update the execution layer + /// permanently rejected, so the delegate does not send them again before a reset. + async fn forward_permanently_rejected_delegated_safe_heads( + &self, + err: &EngineTaskErrors, + ) -> Result<(), EngineError> { + let EngineTaskErrors::Consolidate(ConsolidateTaskError::DelegatedForkchoiceRejected { + heads, + rejection, + }) = err + else { + return Ok(()); + }; + if !rejection.is_permanent() { + return Ok(()); + } + self.derivation_client.send_rejected_delegated_safe_heads(*heads, *rejection).await.map_err( + |err| { + error!(target: "engine", ?err, "Failed to send rejected delegated safe heads"); + EngineError::ChannelClosed + }, + ) + } + async fn mark_el_sync_complete_and_notify_derivation_actor( &mut self, ) -> Result<(), EngineError> { diff --git a/rust/kona/crates/node/service/src/actors/engine/client.rs b/rust/kona/crates/node/service/src/actors/engine/client.rs index 25322a5a7f2..411245cfe73 100644 --- a/rust/kona/crates/node/service/src/actors/engine/client.rs +++ b/rust/kona/crates/node/service/src/actors/engine/client.rs @@ -2,6 +2,7 @@ use crate::{DerivationActorRequest, DerivationClientError, DerivationClientResul use async_trait::async_trait; use derive_more::Constructor; use kona_derive::Signal; +use kona_engine::{DelegatedRejection, DelegatedSafeHeads}; use kona_protocol::L2BlockInfo; use std::fmt::Debug; use tokio::sync::mpsc; @@ -22,6 +23,16 @@ pub trait EngineDerivationClient: Debug + Send + Sync { /// Sends the [`crate::DerivationActor`] the provided [`Signal`]. /// Note: Does not wait for the derivation client to process this message. async fn send_signal(&self, signal: Signal) -> DerivationClientResult<()>; + + /// Tells the derivation actor that the execution layer permanently rejected a forkchoice + /// update for the delegated `heads`, for `rejection`, so they need not be sent again before a + /// reset. + /// Note: Does not wait for the derivation client to process this message. + async fn send_rejected_delegated_safe_heads( + &self, + heads: DelegatedSafeHeads, + rejection: DelegatedRejection, + ) -> DerivationClientResult<()>; } /// Client to use to send messages to the [`crate::DerivationActor`]'s inbound channel. @@ -74,4 +85,24 @@ impl EngineDerivationClient for QueuedEngineDerivationClient { Ok(()) } + + async fn send_rejected_delegated_safe_heads( + &self, + heads: DelegatedSafeHeads, + rejection: DelegatedRejection, + ) -> DerivationClientResult<()> { + info!(target: "engine", ?heads, %rejection, "Sending rejected delegated safe heads to derivation actor"); + + self.derivation_actor_request_tx + .send(DerivationActorRequest::ProcessRejectedDelegatedSafeHeads { + heads: Box::new(heads), + rejection, + }) + .await + .map_err(|_| { + DerivationClientError::RequestError("request channel closed.".to_string()) + })?; + + Ok(()) + } } diff --git a/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs b/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs index defb95a85b2..7e1b357e3d5 100644 --- a/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs +++ b/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs @@ -1,14 +1,15 @@ use super::*; -use crate::QueuedEngineDerivationClient; +use crate::{DerivationActorRequest, QueuedEngineDerivationClient}; use alloy_primitives::B256; use alloy_rpc_types_engine::{ ExecutionPayloadV1, ForkchoiceState, PayloadStatus, PayloadStatusEnum, }; use kona_engine::{ - EngineState, EngineSyncStateUpdate, FinalizeBlockId, NoopBlockSink, ResetRequest, + ConsolidateInput, DelegatedRejection, DelegatedSafeHeads, EngineState, EngineSyncStateUpdate, + FinalizeBlockId, NoopBlockSink, ResetRequest, test_utils::{ RpcMock, TestEngineStateBuilder, TestRpcBlock, delegated_safe_heads, expect_sync_start, - test_block_info, test_engine_client, test_l2_block, test_rpc_block, + test_block_info, test_engine_client, test_l2_block, test_l2_chain, test_rpc_block, }, }; use op_alloy_consensus::{OpBlock, OpTxEnvelope}; @@ -134,6 +135,130 @@ async fn delegated_safe_head_is_sent_on_every_poll() { l2.assert_finished(); } +/// Runs the engine actor on one delegated poll of `heads` against an execution layer that +/// `expect_engine_calls` scripts, and returns the rejected delegated safe heads it tells derivation +/// about. The labels must end where they started. +async fn rejections_reported_for( + state: EngineState, + heads: ConsolidateInput, + expect_engine_calls: impl FnOnce(&RpcMock), +) -> Vec<(DelegatedSafeHeads, DelegatedRejection)> { + let config = Arc::new(RollupConfig::default()); + let (client, l1, l2) = test_engine_client(config.clone()); + expect_engine_calls(&l2); + let (state_tx, state_rx) = watch::channel(state); + let (requests, request_rx) = mpsc::channel(8); + let (derivation_tx, mut derivation_rx) = mpsc::channel(8); + let mut actor = EngineActor::new( + Arc::new(client), + config, + QueuedEngineDerivationClient { derivation_actor_request_tx: derivation_tx }, + Engine::new(state, state_tx), + None, + request_rx, + Arc::new(NoopBlockSink), + ); + + requests.send(EngineActorRequest::ProcessSafeL2Signal(heads)).await.unwrap(); + time::timeout(Duration::from_secs(1), actor.step()).await.unwrap().unwrap(); + drop(requests); + assert!(matches!(actor.step().await, Err(EngineError::ChannelClosed))); + + assert!(actor.engine.is_empty()); + assert_eq!(state_rx.borrow().sync_state, state.sync_state); + l1.assert_finished(); + l2.assert_finished(); + std::iter::from_fn(|| derivation_rx.try_recv().ok()) + .filter_map(|request| match request { + DerivationActorRequest::ProcessRejectedDelegatedSafeHeads { heads, rejection } => { + Some((*heads, rejection)) + } + _ => None, + }) + .collect() +} + +fn expect_invalid_forkchoice_state(l2: &RpcMock) { + l2.expect_error_code( + "engine_forkchoiceUpdatedV3", + i64::from(alloy_rpc_types_engine::INVALID_FORK_CHOICE_STATE_ERROR), + ); +} + +/// A rejected update that moved the head onto delegated heads, restored while the execution layer +/// holds the engine's finalized block, reaches derivation with why it was rejected: whether the +/// canonical block at the finalized height, after the rejected head became canonical, is the +/// finalized block. +#[rstest::rstest] +#[case::chain_below_finality(false, DelegatedRejection::ChainBelowFinality)] +#[case::head_labels(true, DelegatedRejection::HeadLabels)] +#[tokio::test(start_paused = true)] +async fn permanently_rejected_delegated_safe_heads_reach_derivation( + #[case] finalized_stays_canonical: bool, + #[case] expected: DelegatedRejection, +) { + let finalized = test_l2_block(5, B256::ZERO, 0); + let finalized_info = + L2BlockInfo::from_block_and_genesis(&finalized, &RollupConfig::default().genesis).unwrap(); + let state = TestEngineStateBuilder::new() + .with_unsafe_head(test_block_info(20)) + .with_safe_head(test_block_info(10)) + .with_finalized_head(finalized_info) + .build(); + let delegated = test_block_info(30); + let canonical_at_finalized_height = + if finalized_stays_canonical { finalized.clone() } else { test_l2_block(5, B256::ZERO, 1) }; + + let reported = + rejections_reported_for(state, delegated_safe_heads(delegated, delegated), |l2| { + expect_invalid_forkchoice_state(l2); + l2.expect_block_header_by_hash( + finalized_info.block_info.hash, + Some(test_rpc_block(&finalized)), + ); + l2.expect_block_header_by_number( + 5, + Some(test_rpc_block(&canonical_at_finalized_height)), + ); + l2.expect_forkchoice_updated( + state.sync_state.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await; + + assert_eq!( + reported, + vec![(DelegatedSafeHeads::try_new(delegated, delegated).unwrap(), expected)] + ); +} + +/// A rejected update that moved only the safe label, here from a lagging delegate, may be accepted +/// on a later poll, so derivation is not told to hold the heads back. +#[tokio::test(start_paused = true)] +async fn rejected_safe_label_does_not_reach_derivation() { + let chain = test_l2_chain(B256::ZERO, 1, 20, 0); + let info = |number: usize| { + L2BlockInfo::from_block_and_genesis(&chain[number - 1], &RollupConfig::default().genesis) + .unwrap() + }; + let state = TestEngineStateBuilder::new() + .with_unsafe_head(info(20)) + .with_local_safe_head(info(12)) + .with_safe_head(info(8)) + .with_finalized_head(info(5)) + .build(); + + let reported = rejections_reported_for(state, delegated_safe_heads(info(10), info(9)), |l2| { + l2.expect_block_header_by_number(10, Some(test_rpc_block(&chain[9]))); + l2.expect_block_header_by_number(9, Some(test_rpc_block(&chain[8]))); + expect_invalid_forkchoice_state(l2); + }) + .await; + + assert_eq!(reported, Vec::new()); +} + /// A delegate poll while the execution layer syncs toward the unsafe head: the delegate's safe /// head is below the unsafe head but not in the execution layer yet, and neither is its finalized /// block. The consolidate moves the safe labels and the finalize is skipped, so the actor goes on. diff --git a/rust/kona/crates/node/service/src/actors/sequencer/actor.rs b/rust/kona/crates/node/service/src/actors/sequencer/actor.rs index 3e0e7211066..c783603039e 100644 --- a/rust/kona/crates/node/service/src/actors/sequencer/actor.rs +++ b/rust/kona/crates/node/service/src/actors/sequencer/actor.rs @@ -840,6 +840,7 @@ const fn is_insert_task_err_fatal(err: &InsertTaskError) -> bool { match err { InsertTaskError::ForkchoiceUpdateFailed(synchronize_error) => match synchronize_error { SynchronizeTaskError::FinalizedAheadOfUnsafe(_, _) => true, + SynchronizeTaskError::FinalizedAheadOfSafe { .. } | SynchronizeTaskError::ForkchoiceUpdateFailed(_) | SynchronizeTaskError::InvalidForkchoiceState | SynchronizeTaskError::UnexpectedPayloadStatus(_) => false,