diff --git a/rust/kona/crates/node/engine/src/lib.rs b/rust/kona/crates/node/engine/src/lib.rs index eb26238be30..df34ce92baf 100644 --- a/rust/kona/crates/node/engine/src/lib.rs +++ b/rust/kona/crates/node/engine/src/lib.rs @@ -58,7 +58,7 @@ pub use client::{EngineClient, EngineClientError, EngineQueryClient}; mod request; pub use request::{ BuildRequest, CanonicalizeRequest, EngineActorRequest, EngineRequestError, EngineRequestResult, - ResetRequest, SealRequest, + ResetKind, ResetRequest, SealRequest, }; mod versions; @@ -74,7 +74,10 @@ mod metrics; pub use metrics::Metrics; mod sync; -pub use sync::{L2ForkchoiceState, SyncStartError, find_starting_forkchoice}; +pub use sync::{ + DerivedSafeHead, L2ForkchoiceState, SyncStartError, find_reorg_forkchoice, + find_starting_forkchoice, +}; #[cfg(any(test, feature = "test-utils"))] /// Utilities that are useful when creating unit tests using structs within this library. diff --git a/rust/kona/crates/node/engine/src/metrics/mod.rs b/rust/kona/crates/node/engine/src/metrics/mod.rs index f7d1d298edb..ad45157bee9 100644 --- a/rust/kona/crates/node/engine/src/metrics/mod.rs +++ b/rust/kona/crates/node/engine/src/metrics/mod.rs @@ -60,6 +60,10 @@ impl Metrics { /// Identifier for the counter that tracks the number of times the engine has been reset. pub const ENGINE_RESET_COUNT: &str = "kona_node_engine_reset_count"; + /// Identifier for the counter that tracks the number of L1 reorgs the engine answered by + /// keeping a safe head derived from canonical L1 data, instead of a full reset. + pub const ENGINE_REORG_RESET_COUNT: &str = "kona_node_engine_reorg_reset_count"; + /// Initializes metrics for the engine. /// /// This does two things: @@ -85,6 +89,11 @@ impl Metrics { metrics::Unit::Count, "Engine reset count" ); + metrics::describe_counter!( + Self::ENGINE_REORG_RESET_COUNT, + metrics::Unit::Count, + "L1 reorgs answered without a full engine reset" + ); } /// Initializes metrics to `0` so they can be queried immediately by consumers of prometheus @@ -107,5 +116,6 @@ impl Metrics { // Engine reset count metrics::counter!(Self::ENGINE_RESET_COUNT).absolute(0); + metrics::counter!(Self::ENGINE_REORG_RESET_COUNT).absolute(0); } } diff --git a/rust/kona/crates/node/engine/src/request.rs b/rust/kona/crates/node/engine/src/request.rs index 7def359094c..d2309663407 100644 --- a/rust/kona/crates/node/engine/src/request.rs +++ b/rust/kona/crates/node/engine/src/request.rs @@ -1,7 +1,8 @@ //! Requests sent to the node engine actor and their replies. use crate::{ - BuildTaskError, CanonicalizeTaskError, ConsolidateInput, FinalizeBlockId, SealTaskError, + BuildTaskError, CanonicalizeTaskError, ConsolidateInput, DerivedSafeHead, FinalizeBlockId, + SealTaskError, }; use alloy_rpc_types_engine::PayloadId; use kona_protocol::OpAttributesWithParent; @@ -76,6 +77,20 @@ pub struct BuildRequest { pub struct ResetRequest { /// The channel on which the reset result will be sent. pub result_tx: mpsc::Sender>, + /// The reset to perform. + pub kind: ResetKind, +} + +/// The reset a [`ResetRequest`] asks the engine for. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ResetKind { + /// Find a sync starting point with [`crate::find_starting_forkchoice`]. + Full, + /// Answer an L1 reorg detected by derivation. Holds the safe heads derivation recently + /// produced, newest first, each with an L1 block that all of the L1 data it depends on is at + /// or below. The engine keeps the newest one the reorg left canonical; see + /// [`crate::Engine::reset_after_l1_reorg`]. + AfterL1Reorg(Vec), } /// A request to fetch a sealed payload without changing forkchoice. diff --git a/rust/kona/crates/node/engine/src/sync/error.rs b/rust/kona/crates/node/engine/src/sync/error.rs index 1bb9197629c..ba60da44175 100644 --- a/rust/kona/crates/node/engine/src/sync/error.rs +++ b/rust/kona/crates/node/engine/src/sync/error.rs @@ -33,6 +33,22 @@ pub enum SyncStartError { /// Finalized block mismatch #[error("Finalized block mismatch. Expected {0}, Got {1}")] MismatchedFinalizedBlock(B256, B256), + /// The unsafe chain does not lead back to a block it must descend from. + #[error("Unsafe chain does not lead back to {expected}; found {found}")] + UnsafeChainMismatch { + /// The block the unsafe chain must descend from. + expected: B256, + /// The block the unsafe chain has at that height. + found: B256, + }, + /// The unsafe head is below a block it must descend from. + #[error("Unsafe head {unsafe_head} is below block {floor}, which it must descend from")] + UnsafeHeadBelowFloor { + /// The unsafe head's number. + unsafe_head: u64, + /// The number of the block it must descend from. + floor: u64, + }, /// The L1 RPC reported a head at or above a height, then served no block there. #[error("L1 block {number} is missing below the L1 head {l1_head}")] L1BlockMissing { diff --git a/rust/kona/crates/node/engine/src/sync/mod.rs b/rust/kona/crates/node/engine/src/sync/mod.rs index 8631123d50e..cfed50d9de0 100644 --- a/rust/kona/crates/node/engine/src/sync/mod.rs +++ b/rust/kona/crates/node/engine/src/sync/mod.rs @@ -9,6 +9,11 @@ pub use forkchoice::L2ForkchoiceState; mod error; pub use error::SyncStartError; +mod reorg; +pub use reorg::{DerivedSafeHead, find_reorg_forkchoice}; + +mod unsafe_head; + use tracing::info; use crate::EngineClient; @@ -19,12 +24,18 @@ use crate::EngineClient; /// kona does not derive. const MAX_REORG_SEQ_WINDOWS: u64 = 5; +/// How many sequencing windows the unsafe head's L1 origin must be past L1 genesis, with the safe +/// and finalized heads still at genesis, for the safe head search to treat the node as just +/// EL-synced. Matches op-node's `RecoverMinSeqWindows`. +const RECOVER_MIN_SEQ_WINDOWS: u64 = 14; + /// Searches for the latest [`L2ForkchoiceState`] that we can use to start the sync process with. /// /// - The *unsafe L2 block*: This is the highest L2 block whose L1 origin is a *plausible* /// extension of the canonical L1 chain (as known to the rollup node). -/// - The *safe L2 block*: This is the highest L2 block whose epoch's sequencing window is -/// complete within the canonical L1 chain (as known to the rollup node). +/// - The *safe L2 block*: This is the highest L2 block, at or below the previous safe one, whose +/// epoch's sequencing window is complete within the canonical L1 chain (as known to the rollup +/// node). /// - The *finalized L2 block*: This is the L2 block which is known to be fully derived from /// finalized L1 block data. /// @@ -143,7 +154,20 @@ pub async fn find_starting_forkchoice( ); // Search for the highest `safe` block that's L1 origin is at least older than the sequencing - // window, relative to the highest canonical L1 origin found above. + // window, relative to the highest canonical L1 origin found above, and that is no higher than + // the previous safe head: blocks above it were never derived, and may never be. Like op-node's + // `FindL2Heads`. After EL sync, though, the safe and finalized heads can still be at genesis + // with the unsafe head far ahead; there the cap would walk the whole chain back to genesis, + // so, like op-node's `RecoverMinSeqWindows` check, the search is left uncapped. + let el_synced_without_safe_head = current_fc.finalized.block_info.hash == cfg.genesis.l2.hash && + current_fc.safe.block_info.hash == cfg.genesis.l2.hash && + canonical_origin_head.l1_origin.number > + cfg.genesis + .l1 + .number + .saturating_add(RECOVER_MIN_SEQ_WINDOWS * cfg.seq_window_size); + let previous_safe = + if el_synced_without_safe_head { u64::MAX } else { current_fc.safe.block_info.number }; let mut safe_cursor = canonical_origin_head; loop { info!( @@ -157,9 +181,10 @@ pub async fn find_starting_forkchoice( let is_behind_sequence_window = canonical_origin_head.l1_origin.number.saturating_sub(cfg.seq_window_size) > safe_cursor.l1_origin.number; + let is_derived = safe_cursor.block_info.number <= previous_safe; let is_finalized = safe_cursor.block_info.hash == current_fc.finalized.block_info.hash; let is_genesis = safe_cursor.block_info.hash == cfg.genesis.l2.hash; - if is_behind_sequence_window || is_finalized || is_genesis { + if (is_behind_sequence_window && is_derived) || is_finalized || is_genesis { info!( target: "sync_start", l2_safe = %safe_cursor.block_info.number, @@ -301,7 +326,9 @@ mod test { l1_head: u64, ) { l2.expect_params("eth_getBlockByNumber", json!(["finalized", true]), finalized); - l2.expect_params("eth_getBlockByNumber", json!(["safe", true]), finalized); + // The previous safe head is the unsafe head, so it never caps the safe head search here; + // `safe_cap_tests` covers the cap. + l2.expect_params("eth_getBlockByNumber", json!(["safe", true]), unsafe_head); l2.expect_params("eth_getBlockByNumber", json!(["latest", true]), unsafe_head); l1.expect_params("eth_getBlockByNumber", json!(["latest", false]), l1_block(l1_head, 0)); } @@ -532,3 +559,47 @@ mod test { l2.assert_finished(); } } + +#[cfg(test)] +mod safe_cap_tests { + use super::*; + use crate::test_utils::{TestChain, expect_l1_block, expect_l1_head, test_engine_client}; + use rstest::rstest; + use std::sync::Arc; + + /// The safe head never moves up. With a sequencing window of 2, block 22 (L1 origin 102) is + /// the first block below the unsafe head (L1 origin 105) more than a window behind it, but + /// derivation has only reached block 20, so the search keeps going down to block 20. Just + /// after EL sync, with the safe and finalized heads at genesis (here block 18) and the unsafe + /// head's L1 origin more than 14 windows past L1 genesis, it stops at block 22 instead of + /// walking the chain back to genesis. + #[rstest] + #[case::derivation_reached_block_20(false, 20)] + #[case::just_el_synced(true, 22)] + #[tokio::test] + async fn the_safe_head_stays_at_or_below_the_previous_one( + #[case] el_synced: bool, + #[case] expected_safe: u64, + ) { + let chain = TestChain::new(); + let mut config = RollupConfig { seq_window_size: 2, ..Default::default() }; + if el_synced { + config.genesis.l2 = chain.info(18).block_info.id(); + } + let config = Arc::new(config); + let (client, l1, l2) = test_engine_client(config.clone()); + chain.expect_heads(&l2, 18, if el_synced { 18 } else { 20 }, 26); + expect_l1_head(&l1, 106); + expect_l1_block(&l1, 105, 106); + for number in (expected_safe..26).rev() { + chain.expect_by_hash(&l2, number); + } + + let forkchoice = find_starting_forkchoice(&config, &client).await.unwrap(); + + assert_eq!(forkchoice.un_safe, chain.info(26)); + assert_eq!(forkchoice.safe, chain.info(expected_safe)); + l1.assert_finished(); + l2.assert_finished(); + } +} diff --git a/rust/kona/crates/node/engine/src/sync/reorg.rs b/rust/kona/crates/node/engine/src/sync/reorg.rs new file mode 100644 index 00000000000..0ba28ac3e91 --- /dev/null +++ b/rust/kona/crates/node/engine/src/sync/reorg.rs @@ -0,0 +1,108 @@ +//! Choosing the forkchoice to keep across an L1 reorg. +//! +//! [`find_starting_forkchoice`] cannot know which L1 block made the engine's safe head safe, so it +//! walks the safe head back a full sequencing window, stopping early only at the finalized head. +//! That is the right answer on a cold start, but after an L1 reorg detected mid-derivation it +//! makes the cost of a reset independent of the reorg's depth: a one-block replacement of the L1 +//! tip rewinds the safe head as far as a cold start does, and while L1 finality is stalled that is +//! the whole sequencing window. +//! +//! Derivation does know where each safe head came from. A [`DerivedSafeHead`] records that all of +//! the L1 data `safe_head` depends on is at or below `derived_from`. Derivation is a function of +//! the L1 chain it has read, so while `derived_from` is canonical the reorg changed none of that +//! input and `safe_head` is still the safe head that L1 chain derives. Keeping it is then exactly +//! as sound as any reset: the pipeline restarts from a known-correct safe head, a channel timeout +//! behind its L1 origin, as it does after [`find_starting_forkchoice`]. +//! +//! The unsafe chain is checked on its own terms. An unsafe block whose L1 origin the reorg +//! replaced commits to a non-canonical L1 block and can never become safe, so the unsafe head +//! moves back to the highest block whose L1 origin is still canonical. That walk visits only the +//! blocks built on replaced L1 blocks, and never passes the kept safe head, whose L1 data is +//! canonical. +//! +//! [`find_starting_forkchoice`]: crate::find_starting_forkchoice + +use super::unsafe_head::{find_canonical_unsafe_head, is_canonical_l1}; +use crate::{EngineClient, EngineState, EngineSyncState, L2ForkchoiceState, SyncStartError}; +use alloy_eips::{BlockId, BlockNumHash}; +use kona_protocol::L2BlockInfo; +use tracing::info; + +/// A safe head, and an L1 block that all of the L1 data the safe head depends on is at or below. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DerivedSafeHead { + /// The safe head. + pub safe_head: L2BlockInfo, + /// An L1 block at or above every L1 block `safe_head` depends on, including through its + /// ancestors. + pub derived_from: BlockNumHash, +} + +/// Picks the forkchoice to keep after an L1 reorg, or `None` when only a full reset is sound. +/// +/// `candidates` must be ordered newest first. The safe head is the first candidate whose +/// `derived_from` is still canonical, kept only if it is still on the engine's chain, at or below +/// the local safe head and at or above the finalized head. The unsafe head is the highest block +/// between it and the current unsafe head whose L1 origin is still canonical. The finalized head +/// is unchanged. +pub async fn find_reorg_forkchoice( + engine_client: &EngineClient, + state: &EngineState, + candidates: &[DerivedSafeHead], +) -> Result, SyncStartError> { + let sync_state = state.sync_state; + let Some(safe) = find_reorg_safe_head(engine_client, &sync_state, candidates).await? else { + return Ok(None); + }; + let un_safe = + match find_canonical_unsafe_head(engine_client, sync_state.unsafe_head(), safe).await { + Ok(un_safe) => un_safe, + // An unsafe chain that misses a block or does not lead to the kept safe head is left to + // the full reset. + Err( + SyncStartError::BlockNotFound(_) | + SyncStartError::UnsafeChainMismatch { .. } | + SyncStartError::UnsafeHeadBelowFloor { .. }, + ) => { + return Ok(None); + } + Err(err) => return Err(err), + }; + + info!( + target: "sync_start", + safe_head = %safe.block_info.number, + unsafe_head = %un_safe.block_info.number, + "Keeping heads derived from canonical L1 data across the L1 reorg" + ); + Ok(Some(L2ForkchoiceState { un_safe, safe, finalized: sync_state.finalized_head() })) +} + +/// The first of `candidates` whose L1 data is canonical, if the engine can keep it as its safe +/// head. +async fn find_reorg_safe_head( + engine_client: &EngineClient, + sync_state: &EngineSyncState, + candidates: &[DerivedSafeHead], +) -> Result, SyncStartError> { + for candidate in candidates { + if !is_canonical_l1(engine_client, candidate.derived_from).await? { + continue; + } + + let safe_head = candidate.safe_head; + let number = safe_head.block_info.number; + if number > sync_state.local_safe_head().block_info.number || + number < sync_state.finalized_head().block_info.number + { + return Ok(None); + } + let on_chain = engine_client + .get_l2_block(BlockId::number(number)) + .await? + .is_some_and(|block| block.header.hash == safe_head.block_info.hash); + return Ok(on_chain.then_some(safe_head)); + } + + Ok(None) +} diff --git a/rust/kona/crates/node/engine/src/sync/unsafe_head.rs b/rust/kona/crates/node/engine/src/sync/unsafe_head.rs new file mode 100644 index 00000000000..42ea49e05ff --- /dev/null +++ b/rust/kona/crates/node/engine/src/sync/unsafe_head.rs @@ -0,0 +1,87 @@ +//! Finding the highest unsafe block whose L1 origin is canonical. + +use crate::{EngineClient, SyncStartError}; +use alloy_eips::BlockNumHash; +use kona_protocol::L2BlockInfo; +use tracing::info; + +/// Walks back from `unsafe_head` to the highest block whose L1 origin is canonical, never below +/// `floor`, a block whose L1 data is canonical. +/// +/// An L1 origin is canonical when the L1 node has that block at that height. A block built on an +/// origin the L1 node has replaced cannot become safe. One built on an origin the L1 node has not +/// seen yet is not known to be canonical, and is walked back from too; op-node keeps such a head +/// as plausible. The walk visits only blocks built on such origins, checking each origin once, +/// and links the block just above `floor` to it by its parent hash. +/// +/// Fails with [`SyncStartError::UnsafeHeadBelowFloor`] if `unsafe_head` is below `floor`, with +/// [`SyncStartError::BlockNotFound`] if the engine lacks a block on the way, and with +/// [`SyncStartError::UnsafeChainMismatch`] if the chain does not lead back to `floor`. +pub(super) async fn find_canonical_unsafe_head( + engine_client: &EngineClient, + mut unsafe_head: L2BlockInfo, + floor: L2BlockInfo, +) -> Result { + let (head, floor_number) = (unsafe_head.block_info, floor.block_info.number); + if head.number < floor_number { + return Err(SyncStartError::UnsafeHeadBelowFloor { + unsafe_head: head.number, + floor: floor_number, + }); + } + if head.number == floor_number { + return if head.hash == floor.block_info.hash { + Ok(floor) + } else { + Err(SyncStartError::UnsafeChainMismatch { + expected: floor.block_info.hash, + found: head.hash, + }) + }; + } + + let genesis = &engine_client.cfg().genesis; + // Consecutive blocks mostly share an L1 origin; check each replaced one only once. + let mut replaced_origin = None; + loop { + if replaced_origin != Some(unsafe_head.l1_origin) { + if is_canonical_l1(engine_client, unsafe_head.l1_origin).await? { + return Ok(unsafe_head); + } + info!( + target: "sync_start", + l2_unsafe = %unsafe_head.block_info.number, + l1_origin = ?unsafe_head.l1_origin, + "Unsafe block's L1 origin is not canonical; walking back" + ); + replaced_origin = Some(unsafe_head.l1_origin); + } + + let parent_hash = unsafe_head.block_info.parent_hash; + if unsafe_head.block_info.number == floor_number + 1 { + return if parent_hash == floor.block_info.hash { + Ok(floor) + } else { + Err(SyncStartError::UnsafeChainMismatch { + expected: floor.block_info.hash, + found: parent_hash, + }) + }; + } + let parent = engine_client + .get_l2_block(parent_hash.into()) + .full() + .await? + .ok_or(SyncStartError::BlockNotFound(parent_hash.into()))?; + unsafe_head = L2BlockInfo::from_block_and_genesis(&parent.into_consensus(), genesis)?; + } +} + +/// Whether `block` is the canonical L1 block at its height. A block the L1 node does not have at +/// that height yet is not known to be canonical. +pub(super) async fn is_canonical_l1( + engine_client: &EngineClient, + block: BlockNumHash, +) -> Result { + Ok(engine_client.l1_block_hash_by_number(block.number).await? == Some(block.hash)) +} diff --git a/rust/kona/crates/node/engine/src/task_queue/core.rs b/rust/kona/crates/node/engine/src/task_queue/core.rs index 65ef7bdc252..688d7ed9d3d 100644 --- a/rust/kona/crates/node/engine/src/task_queue/core.rs +++ b/rust/kona/crates/node/engine/src/task_queue/core.rs @@ -2,10 +2,10 @@ use super::ExecutableEngineTask; use crate::{ - EngineClient, EngineState, EngineSyncStateUpdate, EngineTask, EngineTaskError, - EngineTaskErrorSeverity, Metrics, SyncStartError, SynchronizeTaskError, - find_starting_forkchoice, - task_queue::{EngineTaskErrors, synchronize_unconditionally}, + DerivedSafeHead, EngineClient, EngineState, EngineSyncStateUpdate, EngineTask, EngineTaskError, + EngineTaskErrorSeverity, L2ForkchoiceState, Metrics, SyncStartError, SynchronizeTaskError, + find_reorg_forkchoice, find_starting_forkchoice, + task_queue::{EngineTaskErrors, synchronize, synchronize_unconditionally}, }; use kona_genesis::RollupConfig; use kona_protocol::L2BlockInfo; @@ -136,6 +136,44 @@ impl Engine { Ok(start.safe) } + /// Resets the engine after an L1 reorg to the forkchoice [`find_reorg_forkchoice`] keeps: the + /// newest of `candidates` (ordered newest first) derived from L1 data the reorg left canonical, + /// and the unsafe chain back to its last block with a canonical L1 origin. The safe heads are + /// rewound to the kept one if they are above it. Falls back to [`Engine::reset`] when no + /// candidate can be kept. + pub async fn reset_after_l1_reorg( + &mut self, + client: Arc, + config: Arc, + candidates: &[DerivedSafeHead], + ) -> Result { + let Some(L2ForkchoiceState { un_safe, safe, .. }) = + find_reorg_forkchoice(client.as_ref(), &self.state, candidates).await? + else { + return self.reset(client, config).await; + }; + + let sync_state = self.state.sync_state; + if un_safe != sync_state.unsafe_head() || safe != sync_state.local_safe_head() { + let above = |head: L2BlockInfo| head.block_info.number > safe.block_info.number; + synchronize( + client.as_ref(), + &mut self.state, + EngineSyncStateUpdate { + unsafe_head: (un_safe != sync_state.unsafe_head()).then_some(un_safe), + local_safe_head: Some(safe), + safe_head: above(sync_state.safe_head()).then_some(safe), + finalized_head: None, + }, + ) + .await?; + } + + metrics::counter!(Metrics::ENGINE_REORG_RESET_COUNT).increment(1); + + Ok(safe) + } + /// Clears the task queue. pub fn clear(&mut self) { self.tasks.clear(); @@ -413,3 +451,279 @@ mod recovery_tests { l2.assert_finished(); } } + +#[cfg(test)] +mod l1_reorg_reset_tests { + use super::*; + use crate::test_utils::{ + L2RpcBlock, RpcMock, TestChain, expect_l1_block, test_engine_client, test_l1_block, + }; + use alloy_primitives::B256; + use alloy_rpc_types_engine::{ForkchoiceUpdated, PayloadStatus, PayloadStatusEnum}; + use kona_protocol::BlockInfo; + use rstest::rstest; + use serde_json::json; + + fn l2_block(number: u64, l1_origin: u64) -> L2BlockInfo { + let hash = |n: u64| B256::left_padding_from(&[0xee, n as u8]); + L2BlockInfo { + block_info: BlockInfo { + number, + hash: hash(number), + parent_hash: hash(number - 1), + timestamp: number, + }, + l1_origin: test_l1_block(l1_origin, 0), + seq_num: 0, + } + } + + /// Scripts the engine's lookup of L2 block `number`, answered with `hash`. + fn expect_l2(l2: &RpcMock, number: u64, hash: B256) { + let mut block = L2RpcBlock::default(); + block.header.hash = hash; + block.header.inner.number = number; + l2.expect_params("eth_getBlockByNumber", json!([format!("{number:#x}"), false]), block); + } + + /// Scripts the forkchoice update that moves the heads, checking the heads it sends. + fn expect_forkchoice_update(l2: &RpcMock, head: L2BlockInfo, safe: L2BlockInfo) { + let state = alloy_rpc_types_engine::ForkchoiceState { + head_block_hash: head.block_info.hash, + safe_block_hash: safe.block_info.hash, + finalized_block_hash: l2_block(FINALIZED, 90).block_info.hash, + }; + l2.expect_params( + "engine_forkchoiceUpdatedV3", + json!([state, null]), + ForkchoiceUpdated::new(PayloadStatus::from_status(PayloadStatusEnum::Valid)), + ); + } + + const FINALIZED: u64 = 5; + const UNSAFE: u64 = 40; + /// Older than every reorg in the tests that keep the unsafe head. + const UNSAFE_ORIGIN: u64 = 102; + + /// Derivation's recent safe heads, newest first: L2 block 20 depends on L1 data up to block + /// 106, and L2 block 18 on L1 data up to block 103. + fn recent_safe_heads() -> [DerivedSafeHead; 2] { + [ + DerivedSafeHead { safe_head: l2_block(20, 100), derived_from: test_l1_block(106, 0) }, + DerivedSafeHead { safe_head: l2_block(18, 99), derived_from: test_l1_block(103, 0) }, + ] + } + + /// An engine with the given heads and finalized head [`FINALIZED`]. + fn engine_at( + unsafe_head: L2BlockInfo, + safe_head: L2BlockInfo, + ) -> (Engine, Arc, RpcMock, RpcMock, Arc) { + let mut config = RollupConfig::default(); + config.hardforks.ecotone_time = Some(0); + let config = Arc::new(config); + let (client, l1, l2) = test_engine_client(config.clone()); + + let mut state = EngineState::default(); + state.sync_state = state.sync_state.apply_update(EngineSyncStateUpdate { + unsafe_head: Some(unsafe_head), + local_safe_head: Some(safe_head), + safe_head: Some(safe_head), + finalized_head: Some(l2_block(FINALIZED, 90)), + }); + let (state_tx, _) = tokio::sync::watch::channel(state); + (Engine::new(state, state_tx), Arc::new(client), l1, l2, config) + } + + /// An engine at the local safe head L2 block 20 and the unsafe head L2 block 40, whose L1 + /// origin [`UNSAFE_ORIGIN`] every reorg below leaves canonical. + fn setup() -> (Engine, Arc, RpcMock, RpcMock, Arc) { + engine_at(l2_block(UNSAFE, UNSAFE_ORIGIN), l2_block(20, 100)) + } + + /// The reset keeps the newest recent safe head whose L1 block is canonical. Its lookups are + /// scripted in order: each recent safe head's L1 block until one is canonical, that safe head + /// on the engine's chain, then the unsafe head's L1 origin. Only the safe labels move, with a + /// forkchoice update only if they do; a cross-safe head already below the kept one stays. + #[rstest] + #[case::tip_reorg_keeps_the_safe_head(108, None, 20)] + #[case::reorg_of_its_l1_block_keeps_the_previous_one(105, None, 18)] + #[case::cross_safe_below_the_kept_one_stays(105, Some(15), 18)] + #[tokio::test] + async fn reset_keeps_the_newest_safe_head_derived_from_canonical_l1( + #[case] reorg_height: u64, + #[case] cross_safe: Option, + #[case] kept: u64, + ) { + let (mut engine, client, l1, l2, config) = setup(); + if let Some(cross_safe) = cross_safe { + engine.state.sync_state = engine.state.sync_state.apply_update(EngineSyncStateUpdate { + safe_head: Some(l2_block(cross_safe, 98)), + ..Default::default() + }); + } + let before = engine.state().sync_state; + let kept = recent_safe_heads() + .into_iter() + .map(|derived| derived.safe_head) + .find(|head| head.block_info.number == kept) + .unwrap(); + let safe_after = if cross_safe.is_some() { before.safe_head() } else { kept }; + expect_l1_block(&l1, 106, reorg_height); + if reorg_height <= 106 { + expect_l1_block(&l1, 103, reorg_height); + } + expect_l2(&l2, kept.block_info.number, kept.block_info.hash); + expect_l1_block(&l1, UNSAFE_ORIGIN, reorg_height); + if kept != before.local_safe_head() { + expect_forkchoice_update(&l2, before.unsafe_head(), safe_after); + } + + let safe_head = + engine.reset_after_l1_reorg(client, config, &recent_safe_heads()).await.unwrap(); + + assert_eq!(safe_head, kept); + let after = engine.state().sync_state; + assert_eq!(after.local_safe_head(), kept); + assert_eq!(after.safe_head(), safe_after); + assert_eq!(after.unsafe_head(), before.unsafe_head()); + assert_eq!(after.finalized_head(), before.finalized_head()); + l1.assert_finished(); + l2.assert_finished(); + } + + /// No recent safe head survives a reorg of every one's L1 block. + #[tokio::test] + async fn finds_nothing_to_keep_when_every_recent_safe_heads_l1_block_was_replaced() { + let (engine, client, l1, l2, _) = setup(); + expect_l1_block(&l1, 106, 103); + expect_l1_block(&l1, 103, 103); + + let forkchoice = + find_reorg_forkchoice(&client, engine.state(), &recent_safe_heads()).await.unwrap(); + + assert_eq!(forkchoice, None); + l1.assert_finished(); + l2.assert_finished(); + } + + /// A safe head with a canonical L1 block is still not kept unless it is on the engine's chain, + /// between the finalized and local-safe heads. + #[rstest] + #[case::above_the_local_safe_head(25, None)] + #[case::below_the_finalized_head(FINALIZED - 1, None)] + #[case::off_the_engines_chain(20, Some(B256::repeat_byte(0xaa)))] + #[tokio::test] + async fn does_not_keep_a_safe_head_outside_the_engines_safe_chain( + #[case] number: u64, + #[case] engine_hash: Option, + ) { + let (engine, client, l1, l2, _) = setup(); + expect_l1_block(&l1, 106, 108); + if let Some(hash) = engine_hash { + expect_l2(&l2, number, hash); + } + let candidate = DerivedSafeHead { + safe_head: l2_block(number, 99), + derived_from: test_l1_block(106, 0), + }; + + let forkchoice = + find_reorg_forkchoice(&client, engine.state(), &[candidate]).await.unwrap(); + + assert_eq!(forkchoice, None); + l1.assert_finished(); + l2.assert_finished(); + } + + /// When the reorg replaced the unsafe head's L1 origin, the unsafe head moves back to the + /// highest block whose L1 origin is canonical, never below the kept safe head. Each walk step + /// fetches a parent by hash and checks its L1 origin, skipping an origin already found + /// replaced; the block just above the safe head is linked to it by its parent hash alone. + #[rstest] + #[case::tip_reorg(105, &[(20, 101), (18, 100)], 20, 24, &[101, 105, 104])] + #[case::deeper_reorg(103, &[(20, 101), (18, 100)], 20, 22, &[101, 105, 104, 103, 102])] + #[case::reorg_of_the_safe_heads_l1_data( + 101, &[(20, 101), (18, 100)], 18, 20, &[101, 100, 105, 104, 103, 102, 101, 100] + )] + #[case::back_to_the_safe_head(101, &[(20, 100)], 20, 20, &[100, 105, 104, 103, 102, 101])] + #[tokio::test] + async fn reset_walks_the_unsafe_chain_back_to_a_canonical_l1_origin( + #[case] reorg_height: u64, + #[case] candidates: &[(u64, u64)], + #[case] kept: u64, + #[case] unsafe_after: u64, + #[case] l1_lookups: &[u64], + ) { + let chain = TestChain::new(); + let head = |number: u64| chain.info(number); + let (mut engine, client, l1, l2, config) = engine_at(head(26), head(20)); + let candidates: Vec<_> = candidates + .iter() + .map(|&(safe_head, derived_from)| DerivedSafeHead { + safe_head: head(safe_head), + derived_from: test_l1_block(derived_from, 0), + }) + .collect(); + for &number in l1_lookups { + expect_l1_block(&l1, number, reorg_height); + } + expect_l2(&l2, kept, head(kept).block_info.hash); + for number in (unsafe_after.max(kept + 1)..26).rev() { + chain.expect_by_hash(&l2, number); + } + let safe_after = head(kept); + expect_forkchoice_update(&l2, head(unsafe_after), safe_after); + + let safe_head = engine.reset_after_l1_reorg(client, config, &candidates).await.unwrap(); + + assert_eq!(safe_head, safe_after); + let after = engine.state().sync_state; + assert_eq!(after.unsafe_head(), head(unsafe_after)); + assert_eq!(after.local_safe_head(), safe_after); + assert_eq!(after.safe_head(), safe_after); + l1.assert_finished(); + l2.assert_finished(); + } + + /// An unsafe chain that does not lead back to the kept safe head gives nothing to keep. + #[tokio::test] + async fn finds_nothing_to_keep_when_the_unsafe_chain_does_not_lead_to_the_safe_head() { + let chain = TestChain::new(); + let mut other_20 = chain.info(20); + other_20.block_info.hash = B256::repeat_byte(0xaa); + let (engine, client, l1, l2, _) = engine_at(chain.info(21), other_20); + expect_l1_block(&l1, 100, 101); + expect_l2(&l2, 20, other_20.block_info.hash); + expect_l1_block(&l1, 101, 101); + + let candidate = + DerivedSafeHead { safe_head: other_20, derived_from: test_l1_block(100, 0) }; + let forkchoice = + find_reorg_forkchoice(&client, engine.state(), &[candidate]).await.unwrap(); + + assert_eq!(forkchoice, None); + l1.assert_finished(); + l2.assert_finished(); + } + + /// With no safe head to keep, the reset falls back to the full walk, which starts by reading + /// the engine's finalized head. + #[tokio::test] + async fn falls_back_to_the_full_walk() { + let (mut engine, client, l1, l2, config) = setup(); + expect_l1_block(&l1, 106, 103); + expect_l1_block(&l1, 103, 103); + l2.expect_error("eth_getBlockByNumber"); + + let err = + engine.reset_after_l1_reorg(client, config, &recent_safe_heads()).await.unwrap_err(); + + assert!( + matches!(err, EngineResetError::SyncStart(SyncStartError::RpcError(_))), + "got {err:?}" + ); + l1.assert_finished(); + l2.assert_finished(); + } +} diff --git a/rust/kona/crates/node/engine/src/test_utils/chain.rs b/rust/kona/crates/node/engine/src/test_utils/chain.rs new file mode 100644 index 00000000000..add47c0b8ac --- /dev/null +++ b/rust/kona/crates/node/engine/src/test_utils/chain.rs @@ -0,0 +1,151 @@ +//! A real L2 chain over a scripted L1 chain, for tests of the sync start and reorg walks. + +use crate::test_utils::RpcMock; +use alloy_eips::BlockNumHash; +use alloy_primitives::{Address, B256, U256}; +use kona_genesis::RollupConfig; +use kona_protocol::{L1BlockInfoBedrock, L2BlockInfo}; +use serde_json::json; + +/// An L2 block as the engine's RPC returns it. +pub type L2RpcBlock = ::BlockResponse; + +/// L1 block `number` on `fork`: fork 0 is the chain derivation read, fork 1 replaced it from +/// some height on. +pub fn test_l1_block(number: u64, fork: u8) -> BlockNumHash { + BlockNumHash { number, hash: B256::left_padding_from(&[fork, number as u8]) } +} + +/// Scripts the L1 node's lookup of block `number`, on the chain whose blocks from +/// `replaced_from` on are on fork 1. +pub fn expect_l1_block(l1: &RpcMock, number: u64, replaced_from: u64) { + let mut block = alloy_rpc_types_eth::Block::::default(); + block.header.hash = test_l1_block(number, u8::from(number >= replaced_from)).hash; + block.header.inner.number = number; + l1.expect_params("eth_getBlockByNumber", json!([format!("{number:#x}"), false]), block); +} + +/// Scripts the L1 node's answer for its head: block `number`. +pub fn expect_l1_head(l1: &RpcMock, number: u64) { + let mut block = alloy_rpc_types_eth::Block::::default(); + block.header.inner.number = number; + l1.expect_params("eth_getBlockByNumber", json!(["latest", false]), block); +} + +/// A real L2 chain from block 18 to block 26, so the unsafe walks can fetch and decode its +/// blocks. Blocks 18 to 20 have L1 origins 99, 100 and 100, blocks 21 to 24 have L1 origins +/// 101 to 104, and blocks 25 and 26 share L1 origin 105. +#[derive(Debug)] +pub struct TestChain { + blocks: Vec<(L2BlockInfo, L2RpcBlock)>, +} + +impl Default for TestChain { + fn default() -> Self { + Self::new() + } +} + +impl TestChain { + /// Builds the chain. + pub fn new() -> Self { + let mut blocks: Vec<(L2BlockInfo, L2RpcBlock)> = Vec::new(); + for number in 18..=26 { + let origin = match number { + 18 => 99, + 19 | 20 => 100, + 25 | 26 => 105, + n => 80 + n, + }; + let info = L1BlockInfoBedrock::new( + origin, + 0, + 0, + test_l1_block(origin, 0).hash, + 0, + Address::ZERO, + U256::ZERO, + U256::ZERO, + ); + let deposit = op_alloy_consensus::OpTxEnvelope::Deposit(alloy_primitives::Sealed::new( + op_alloy_consensus::TxDeposit { + input: info.encode_calldata(), + ..Default::default() + }, + )); + let header = alloy_consensus::Header { + number, + parent_hash: blocks.last().map_or(B256::ZERO, |(parent, _)| parent.block_info.hash), + timestamp: number, + ..Default::default() + }; + let consensus = alloy_consensus::Block:: { + header: header.clone(), + body: alloy_consensus::BlockBody { + transactions: vec![deposit.clone()], + ..Default::default() + }, + }; + let block_info = + L2BlockInfo::from_block_and_genesis(&consensus, &RollupConfig::default().genesis) + .unwrap(); + let rpc = L2RpcBlock { + header: alloy_rpc_types_eth::Header { + hash: block_info.block_info.hash, + inner: header, + ..Default::default() + }, + transactions: alloy_rpc_types_eth::BlockTransactions::Full(vec![ + op_alloy_rpc_types::Transaction { + inner: alloy_rpc_types_eth::Transaction { + inner: alloy_consensus::transaction::Recovered::new_unchecked( + deposit, + Address::ZERO, + ), + block_hash: Some(block_info.block_info.hash), + block_number: Some(number), + effective_gas_price: Some(0), + transaction_index: Some(0), + block_timestamp: None, + }, + deposit_nonce: None, + deposit_receipt_version: None, + }, + ]), + ..Default::default() + }; + blocks.push((block_info, rpc)); + } + Self { blocks } + } + + /// Block `number`'s info. + pub fn info(&self, number: u64) -> L2BlockInfo { + self.block(number).0 + } + + /// Block `number` as the engine's RPC returns it. + pub fn rpc(&self, number: u64) -> &L2RpcBlock { + &self.block(number).1 + } + + /// Scripts the engine's lookup of block `number` by hash, with transactions. + pub fn expect_by_hash(&self, l2: &RpcMock, number: u64) { + let (info, rpc) = self.block(number); + l2.expect_params("eth_getBlockByHash", json!([info.block_info.hash, true]), rpc); + } + + /// Scripts the engine's lookups of its finalized, safe and latest blocks. + pub fn expect_heads(&self, l2: &RpcMock, finalized: u64, safe: u64, latest: u64) { + for (tag, number) in [("finalized", finalized), ("safe", safe), ("latest", latest)] { + l2.expect_params("eth_getBlockByNumber", json!([tag, true]), self.rpc(number)); + } + } + + fn block(&self, number: u64) -> &(L2BlockInfo, L2RpcBlock) { + self.blocks + .iter() + .find(|(info, _)| info.block_info.number == number) + .unwrap_or_else(|| panic!("no block {number} in the test chain")) + } +} diff --git a/rust/kona/crates/node/engine/src/test_utils/mod.rs b/rust/kona/crates/node/engine/src/test_utils/mod.rs index dd8c55020bd..a59d7204f25 100644 --- a/rust/kona/crates/node/engine/src/test_utils/mod.rs +++ b/rust/kona/crates/node/engine/src/test_utils/mod.rs @@ -15,3 +15,6 @@ pub use engine_state::TestEngineStateBuilder; mod misc; pub use misc::test_block_info; + +mod chain; +pub use chain::{L2RpcBlock, TestChain, expect_l1_block, expect_l1_head, test_l1_block}; diff --git a/rust/kona/crates/node/rpc/src/admin.rs b/rust/kona/crates/node/rpc/src/admin.rs index bdf2389e7af..a84e093e51f 100644 --- a/rust/kona/crates/node/rpc/src/admin.rs +++ b/rust/kona/crates/node/rpc/src/admin.rs @@ -8,7 +8,7 @@ use jsonrpsee::{ core::RpcResult, types::{ErrorCode, ErrorObject}, }; -use kona_engine::{EngineActorRequest, ResetRequest}; +use kona_engine::{EngineActorRequest, ResetKind, ResetRequest}; use op_alloy_rpc_types_engine::{OpExecutionPayloadEnvelope, OpPayloadError}; use thiserror::Error; use tokio::sync::{mpsc, oneshot, watch}; @@ -177,7 +177,10 @@ impl AdminApiServer for AdminRpc { async fn admin_reset_derivation_pipeline(&self) -> RpcResult<()> { let (result_tx, mut result_rx) = mpsc::channel(1); self.engine - .send(EngineActorRequest::Reset(Box::new(ResetRequest { result_tx }))) + .send(EngineActorRequest::Reset(Box::new(ResetRequest { + result_tx, + kind: ResetKind::Full, + }))) .await .map_err(|_| ErrorObject::from(ErrorCode::InternalError))?; result_rx diff --git a/rust/kona/crates/node/service/src/actors/derivation/actor/core.rs b/rust/kona/crates/node/service/src/actors/derivation/actor/core.rs index 2b602e11ff9..86600d301ad 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/actor/core.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/actor/core.rs @@ -3,7 +3,7 @@ use crate::{ DerivationActorRequest, DerivationEngineClient, DerivationState, DerivationStateMachine, DerivationStateTransitionError, DerivationStateUpdate, Metrics, NodeActor, - actors::derivation::L2Finalizer, + actors::derivation::{L2Finalizer, SafeHeadHistory}, }; use alloy_eips::BlockNumHash; use async_trait::async_trait; @@ -51,6 +51,8 @@ where l1_block_zero: BlockNumHash, /// The attributes sent to the engine and not yet confirmed as a safe head. pub(super) attributes_awaiting_confirmation: Option, + /// The recent safe heads an L1 reorg may keep, with the L1 data they depend on. + pub(super) safe_head_history: SafeHeadHistory, /// A pipeline reset that has not yet succeeded. Derivation must not resume before it does. pending_reset: Option, } @@ -121,6 +123,7 @@ where safe_db, l1_block_zero, attributes_awaiting_confirmation: None, + safe_head_history: SafeHeadHistory::default(), pending_reset: None, } } @@ -138,6 +141,40 @@ where self.publish_origin(); } + /// Restarts the [`SafeHeadHistory`] from the safe head derivation restarts from. + fn reset_safe_head_history(&mut self, safe_head: L2BlockInfo) { + let seq_window_size = self.pipeline.rollup_config().seq_window_size; + self.safe_head_history.reset_to(safe_head, seq_window_size); + } + + /// Asks the engine to reset for a pipeline [`ResetError`]. An L1 reorg lets the engine keep a + /// safe head the reorg left derivable, rather than rewind a full sequencing window. + async fn reset_engine(&self, e: &ResetError) -> Result<(), DerivationError> { + let reset = if let ResetError::ReorgDetected(expected, new) = e { + warn!(target: "derivation", "L1 reorg detected! Expected: {expected} | New: {new}"); + metrics::counter!(Metrics::L1_REORG_COUNT).increment(1); + + // The pipeline's origin has no canonical child, so it was reorged out, and so was every + // L1 block at or above it. Drop the safe heads that depend on one without asking the + // engine, whose L1 node may not have seen the reorg yet. + let reorged_from = self.pipeline.origin().map_or(u64::MAX, |origin| origin.number); + let recent_safe_heads = self + .safe_head_history + .newest_first() + .into_iter() + .filter(|derived| derived.derived_from.number < reorged_from) + .collect(); + self.engine_client.reset_engine_after_l1_reorg(recent_safe_heads).await + } else { + self.engine_client.reset_engine_forkchoice().await + }; + + reset.map_err(|e| { + error!(target: "derivation", ?e, "Failed to send reset request"); + DerivationError::Sender(Box::new(e)) + }) + } + /// Resets the pipeline to the engine's safe head, adopts that head, and resumes derivation. /// /// `SafeDB` is truncated before the pipeline resets, so derivation must not resume until the @@ -153,6 +190,7 @@ where self.derivation_origin_tx.send_replace(None); self.finalizer.clear(); self.attributes_awaiting_confirmation = None; + self.reset_safe_head_history(reset.l2_safe_head); self.truncate_safe_db(reset.l2_safe_head, "engine reset")?; match self.pipeline.signal(Signal::Reset(reset)).await { Ok(()) => info!(target: "derivation", ?reset, "[SIGNAL] Executed Successfully"), @@ -292,20 +330,9 @@ where })) .await?; } else { - if let ResetError::ReorgDetected(expected, new) = e { - warn!( - target: "derivation", - "L1 reorg detected! Expected: {expected} | New: {new}" - ); - - metrics::counter!(Metrics::L1_REORG_COUNT).increment(1); - } // Like op-node, report no origin until the reset sets a new one. self.derivation_origin_tx.send_replace(None); - self.engine_client.reset_engine_forkchoice().await.map_err(|e| { - error!(target: "derivation", ?e, "Failed to send reset request"); - DerivationError::Sender(Box::new(e)) - })?; + self.reset_engine(&e).await?; self.derivation_state_machine .update(&DerivationStateUpdate::SignalNeeded)?; return Err(DerivationError::Yield); @@ -379,6 +406,7 @@ where info!(target: "derivation", "Engine finished syncing, starting derivation."); self.derivation_state_machine .update(&DerivationStateUpdate::ELSyncCompleted(safe_head.clone()))?; + self.reset_safe_head_history(*safe_head); self.truncate_safe_db(*safe_head, "EL sync completion")?; self.attempt_derivation().await?; @@ -414,12 +442,20 @@ where ); return Ok(()); } - if let (Some(database), Some(derived_from)) = (&self.safe_db, attributes.derived_from) { - debug!(target: "derivation", l1_block = derived_from.number, safe_head = safe_head.block_info.number, "Recording safe head"); - database.safe_head_updated(safe_head, derived_from.id()).inspect_err(|error| { - error!(target: "derivation", %error, l1_block = derived_from.number, safe_head = safe_head.block_info.number, "Failed to record safe head"); - })?; - debug!(target: "derivation", l1_block = derived_from.number, safe_head = safe_head.block_info.number, "Recorded safe head"); + if let Some(derived_from) = attributes.derived_from.map(|block| block.id()) { + // Both records take the L1 block the history works out: the derivation source, + // raised to the L1 block the parent was recorded at, since a block cannot become safe + // before its parent. Below a floor known only as a bound, the history records nothing + // and the database keeps the derivation source. + let safe_at = + self.safe_head_history.record(safe_head, derived_from).unwrap_or(derived_from); + if let Some(database) = &self.safe_db { + debug!(target: "derivation", l1_block = safe_at.number, safe_head = safe_head.block_info.number, "Recording safe head"); + database.safe_head_updated(safe_head, safe_at).inspect_err(|error| { + error!(target: "derivation", %error, l1_block = safe_at.number, safe_head = safe_head.block_info.number, "Failed to record safe head"); + })?; + debug!(target: "derivation", l1_block = safe_at.number, safe_head = safe_head.block_info.number, "Recorded safe head"); + } } self.finalizer.enqueue_for_finalization(&attributes); Ok(()) diff --git a/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs b/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs index 818294a343b..c7dc69bbcda 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs @@ -10,7 +10,7 @@ use kona_derive::{ ActivationSignal, Pipeline, PipelineError, PipelineErrorKind, PipelineResult, ResetError, Signal, SignalReceiver, StepResult, }; -use kona_engine::FinalizeBlockId; +use kona_engine::{DerivedSafeHead, FinalizeBlockId}; use kona_genesis::{HardForkConfig, RollupConfig, SystemConfig}; use kona_protocol::{BlockInfo, L2BlockInfo, OpAttributesWithParent}; use kona_safedb::{SafeDatabase, SafeDb, SafeDbError, SafeHeadRecord}; @@ -179,19 +179,31 @@ impl Pipeline for StubPipeline { /// A pipeline-driven reset must always reach the engine actor. The engine's reset is what /// sends the pipeline its [`Signal`] back; without it the actor parks in -/// [`DerivationState::AwaitingSignal`] forever. +/// [`DerivationState::AwaitingSignal`] forever. An L1 reorg asks for the reset after a reorg; +/// any other reset asks for the full one. #[rstest] -#[case::interop_inactive(None)] -#[case::interop_active(Some(0))] +#[case::reorg_interop_inactive(None, true)] +#[case::reorg_interop_active(Some(0), true)] +#[case::other_reset_interop_inactive(None, false)] +#[case::other_reset_interop_active(Some(0), false)] #[tokio::test] -async fn test_pipeline_reset_always_resets_engine(#[case] lagoon_time: Option) { +async fn test_pipeline_reset_always_resets_engine( + #[case] lagoon_time: Option, + #[case] reorg: bool, +) { let rollup_config = Arc::new(RollupConfig { hardforks: HardForkConfig { lagoon_time, ..Default::default() }, ..Default::default() }); let mut engine_client = MockDerivationEngineClient::new(); - engine_client.expect_reset_engine_forkchoice().times(1).returning(|| Ok(())); + let reset = if reorg { + engine_client.expect_reset_engine_after_l1_reorg().times(1).returning(|_| Ok(())); + ResetError::ReorgDetected(B256::ZERO, B256::repeat_byte(1)) + } else { + engine_client.expect_reset_engine_forkchoice().times(1).returning(|| Ok(())); + ResetError::BadParentHash(B256::ZERO, B256::repeat_byte(1)) + }; let (derivation_origin_tx, derivation_origin_rx) = watch::channel(None); let (request_tx, request_rx) = mpsc::channel(1); @@ -202,9 +214,7 @@ async fn test_pipeline_reset_always_resets_engine(#[case] lagoon_time: Option, derived_from: BlockInfo, rollup_config: Arc, + reorg_when_exhausted: bool, signal_error: Option, } @@ -433,10 +445,14 @@ impl Pipeline for QueuedPipeline { } async fn step(&mut self, _: L2BlockInfo) -> StepResult { - if self.attributes.is_empty() { - StepResult::StepFailed(PipelineError::Eof.temp()) - } else { + if !self.attributes.is_empty() { StepResult::PreparedAttributes + } else if self.reorg_when_exhausted { + StepResult::StepFailed( + ResetError::ReorgDetected(B256::ZERO, B256::repeat_byte(1)).reset(), + ) + } else { + StepResult::StepFailed(PipelineError::Eof.temp()) } } @@ -515,6 +531,7 @@ impl RecordingActor { attributes: VecDeque::from([derived_attributes()]), derived_from: l1_block(DERIVED_FROM_L1), rollup_config: Arc::new(RollupConfig::default()), + reorg_when_exhausted: false, signal_error: None, }; let (request_tx, request_rx) = mpsc::channel(1); @@ -572,6 +589,143 @@ impl RecordingActor { } } +/// The recorded safe head [`DERIVED_HEAD`], derived from [`DERIVED_FROM_L1`]. +fn derived_head() -> DerivedSafeHead { + DerivedSafeHead { + safe_head: l2_block(DERIVED_HEAD), + derived_from: l1_block(DERIVED_FROM_L1).id(), + } +} + +/// An engine client that accepts attributes and expects one reset after an L1 reorg, handed +/// `recent_safe_heads`. +fn engine_client_expecting_reorg_reset( + recent_safe_heads: Vec, +) -> MockDerivationEngineClient { + let mut engine_client = engine_client_accepting_attributes(); + engine_client + .expect_reset_engine_after_l1_reorg() + .withf(move |heads| *heads == recent_safe_heads) + .times(1) + .returning(|_| Ok(())); + engine_client +} + +/// An L1 reorg hands the engine the safe heads derivation produced, each with the L1 block it +/// was derived from, so the engine can keep one the reorg left canonical. The pipeline's own +/// origin was reorged out, so a safe head derived at it is not offered. +#[rstest] +#[case::derived_below_the_reorged_origin(DERIVED_FROM_L1 + 1, vec![derived_head()])] +#[case::derived_at_the_reorged_origin(DERIVED_FROM_L1, vec![])] +#[tokio::test] +async fn an_l1_reorg_hands_the_engine_the_recent_safe_heads( + #[case] reorged_origin: u64, + #[case] expected: Vec, +) { + let mut node = RecordingActor::new(engine_client_expecting_reorg_reset(expected)); + node.actor.pipeline.derived_from = l1_block(reorged_origin); + node.actor.pipeline.reorg_when_exhausted = true; + + node.derive_and_confirm_one_block().await; + + assert_eq!( + node.actor.derivation_state_machine.current_state(), + DerivationState::AwaitingSignal + ); +} + +/// A reset to a safe head the history did not record clears the history, so a later reorg +/// offers nothing that may depend on data below the reset. +#[tokio::test] +async fn a_reset_to_an_unrecorded_safe_head_clears_the_history() { + let mut node = RecordingActor::new(engine_client_accepting_attributes()); + node.derive_and_confirm_one_block().await; + assert_eq!(node.actor.safe_head_history.newest_first(), vec![derived_head()]); + + node.signal(Signal::Reset(kona_derive::ResetSignal { l2_safe_head: l2_block(RESUME_HEAD) })) + .await; + + assert_eq!(node.actor.safe_head_history.newest_first(), vec![]); +} + +/// At EL sync completion, the safe head's L1 data is unknown beyond the end of its sequencing +/// window, so a block derived below that is not recorded. +#[tokio::test] +async fn el_sync_completion_floors_the_history_at_the_sequencing_window() { + let mut node = RecordingActor::new(engine_client_expecting_reorg_reset(vec![])); + node.actor.pipeline.rollup_config = + Arc::new(RollupConfig { seq_window_size: DERIVED_FROM_L1, ..Default::default() }); + node.actor.pipeline.derived_from = l1_block(DERIVED_FROM_L1 + 1); + node.actor.pipeline.reorg_when_exhausted = true; + + node.derive_and_confirm_one_block().await; +} + +/// After a reset that keeps the confirmed safe head, derivation resumes from it and records the +/// next block it derives. +#[tokio::test] +async fn a_reset_to_the_confirmed_safe_head_records_the_next_block() { + let mut node = RecordingActor::new(engine_client_accepting_attributes()); + node.derive_and_confirm_one_block().await; + let next_l1 = DERIVED_FROM_L1 + 1; + node.actor.pipeline.attributes.push_back(attributes_after(DERIVED_HEAD, next_l1)); + + node.signal(Signal::Reset(kona_derive::ResetSignal { l2_safe_head: l2_block(DERIVED_HEAD) })) + .await; + + assert_eq!( + node.actor.derivation_state_machine.current_state(), + DerivationState::AwaitingSafeHeadConfirmation + ); + node.confirm_safe_head(DERIVED_HEAD + 1).await; + assert_eq!(node.db.last_entry().unwrap(), record(next_l1, DERIVED_HEAD + 1)); +} + +/// Attributes for the child of `parent`, derived from L1 block `derived_from`. +fn attributes_after(parent: u64, derived_from: u64) -> OpAttributesWithParent { + OpAttributesWithParent { + attributes: OpPayloadAttributes::default(), + parent: l2_block(parent), + derived_from: Some(l1_block(derived_from)), + is_last_in_span: true, + } +} + +/// After a reset, the pipeline replays from before the safe head's L1 origin and can produce the +/// next block from an L1 block below the one the safe head became safe at. That block became safe +/// only once its parent did, so the database records it at its parent's L1 block: updates stay +/// in ascending L1 order, and no query answers it for an L1 block where the parent was not safe. +#[tokio::test] +async fn replay_after_a_reset_records_no_safe_head_below_its_parents_l1_block() { + let mut node = RecordingActor::new(engine_client_accepting_attributes()); + node.derive_and_confirm_one_block().await; + let parent_l1 = DERIVED_FROM_L1 + 6; + node.actor.pipeline.attributes.push_back(attributes_after(DERIVED_HEAD, parent_l1)); + node.handle(DerivationActorRequest::ProcessL1HeadUpdateRequest(Box::default())).await; + node.confirm_safe_head(DERIVED_HEAD + 1).await; + assert_eq!(node.db.last_entry().unwrap(), record(parent_l1, DERIVED_HEAD + 1)); + + // A reset that keeps that safe head, then a replay that finds the next block's batch lower. + node.actor.pipeline.attributes.push_back(attributes_after(DERIVED_HEAD + 1, parent_l1 - 3)); + node.signal(Signal::Reset(kona_derive::ResetSignal { + l2_safe_head: l2_block(DERIVED_HEAD + 1), + })) + .await; + node.actor + .pipeline + .attributes + .retain(|attributes| attributes.parent == l2_block(DERIVED_HEAD + 1)); + node.confirm_safe_head(DERIVED_HEAD + 2).await; + + assert_eq!(node.db.last_entry().unwrap(), record(parent_l1, DERIVED_HEAD + 2)); + // Below the parent's L1 block, the safe head is still the grandparent. + assert_eq!( + node.db.safe_head_at_l1(parent_l1 - 1).unwrap(), + record(DERIVED_FROM_L1, DERIVED_HEAD) + ); + assert_eq!(node.db.l1_at_safe_head(DERIVED_HEAD + 1).unwrap().l1, l1_block(parent_l1).id()); +} + #[tokio::test] async fn a_confirmed_block_is_recorded_against_its_derivation_source() { let mut node = RecordingActor::new(engine_client_accepting_attributes()); diff --git a/rust/kona/crates/node/service/src/actors/derivation/engine_client.rs b/rust/kona/crates/node/service/src/actors/derivation/engine_client.rs index ad024f605ad..fa85c69f9f5 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/engine_client.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/engine_client.rs @@ -1,7 +1,7 @@ -use crate::{EngineActorRequest, EngineClientError, EngineClientResult, ResetRequest}; +use crate::{EngineActorRequest, EngineClientError, EngineClientResult, ResetKind, ResetRequest}; use async_trait::async_trait; use derive_more::Constructor; -use kona_engine::{ConsolidateInput, FinalizeBlockId}; +use kona_engine::{ConsolidateInput, DerivedSafeHead, FinalizeBlockId}; use std::fmt::Debug; use tokio::sync::mpsc; @@ -12,6 +12,14 @@ pub trait DerivationEngineClient: Debug + Send + Sync { /// Resets the engine's forkchoice. async fn reset_engine_forkchoice(&self) -> EngineClientResult<()>; + /// Resets the engine after derivation detected an L1 reorg. `recent_safe_heads` are the safe + /// heads derivation recently produced, newest first. The engine keeps the newest one derived + /// from L1 data the reorg left canonical, and falls back to a full reset otherwise. + async fn reset_engine_after_l1_reorg( + &self, + recent_safe_heads: Vec, + ) -> EngineClientResult<()>; + /// Sends a request to finalize the L2 block identified by the provided [`FinalizeBlockId`]. /// Note: This does not wait for the engine to process it. async fn send_finalized_l2_block(&self, block_id: FinalizeBlockId) -> EngineClientResult<()>; @@ -33,13 +41,12 @@ pub struct QueuedDerivationEngineClient { pub engine_actor_request_tx: mpsc::Sender, } -#[async_trait] -impl DerivationEngineClient for QueuedDerivationEngineClient { - async fn reset_engine_forkchoice(&self) -> EngineClientResult<()> { +impl QueuedDerivationEngineClient { + async fn reset(&self, kind: ResetKind) -> EngineClientResult<()> { let (result_tx, mut result_rx) = mpsc::channel(1); self.engine_actor_request_tx - .send(EngineActorRequest::Reset(Box::new(ResetRequest { result_tx }))) + .send(EngineActorRequest::Reset(Box::new(ResetRequest { result_tx, kind }))) .await .map_err(|_| EngineClientError::RequestError("request channel closed.".to_string()))?; @@ -48,6 +55,20 @@ impl DerivationEngineClient for QueuedDerivationEngineClient { EngineClientError::ResponseError("response channel closed.".to_string()) })? } +} + +#[async_trait] +impl DerivationEngineClient for QueuedDerivationEngineClient { + async fn reset_engine_forkchoice(&self) -> EngineClientResult<()> { + self.reset(ResetKind::Full).await + } + + async fn reset_engine_after_l1_reorg( + &self, + recent_safe_heads: Vec, + ) -> EngineClientResult<()> { + self.reset(ResetKind::AfterL1Reorg(recent_safe_heads)).await + } async fn send_finalized_l2_block(&self, block_id: FinalizeBlockId) -> EngineClientResult<()> { self.engine_actor_request_tx diff --git a/rust/kona/crates/node/service/src/actors/derivation/mod.rs b/rust/kona/crates/node/service/src/actors/derivation/mod.rs index c3dc57502c0..5ff0d0c3d93 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/mod.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/mod.rs @@ -10,6 +10,9 @@ pub use engine_client::{DerivationEngineClient, QueuedDerivationEngineClient}; mod finalizer; pub(crate) use finalizer::L2Finalizer; +mod safe_head_history; +use safe_head_history::SafeHeadHistory; + mod request; pub use request::{DerivationActorRequest, DerivationClientError, DerivationClientResult}; diff --git a/rust/kona/crates/node/service/src/actors/derivation/safe_head_history.rs b/rust/kona/crates/node/service/src/actors/derivation/safe_head_history.rs new file mode 100644 index 00000000000..5fc55c84dd6 --- /dev/null +++ b/rust/kona/crates/node/service/src/actors/derivation/safe_head_history.rs @@ -0,0 +1,216 @@ +//! The [`SafeHeadHistory`] for the derivation actor. + +use alloy_eips::BlockNumHash; +use kona_engine::DerivedSafeHead; +use kona_protocol::L2BlockInfo; +use std::collections::BTreeMap; + +/// How many safe heads are kept, one per L1 block that produced one. A reorg that reaches below +/// the oldest kept one falls back to a full engine reset. +const MAX_ENTRIES: usize = 64; + +/// The safe heads derivation produced from the most recent L1 blocks, each with an L1 block that +/// all of the L1 data it depends on is at or below. After an L1 reorg, the engine keeps the newest +/// one whose L1 block the reorg left canonical, instead of rewinding a full sequencing window; +/// see [`kona_engine::find_reorg_forkchoice`]. +/// +/// This is kept in memory whether or not the safe head database is enabled, and is bounded to +/// the depth an L1 reorg can plausibly reach. The database records each safe head at the L1 block +/// [`SafeHeadHistory::record`] returns, so the two never disagree. +/// +/// A safe head depends on the L1 data its own attributes were derived from, and on everything its +/// parent depends on. The pipeline's origin bounds only the former: after a reset the pipeline +/// replays from before the safe head's L1 origin, and can produce the next block from an L1 block +/// below the one its parent was recorded at, as when its batch landed first and was buffered. +/// So the history keeps a [`Floor`] that is at least the L1 block the current safe head depends +/// on, and records no safe head below it. +#[derive(Debug, Default)] +pub(crate) struct SafeHeadHistory { + /// L1 block number -> the highest safe head produced with the pipeline at that L1 block. + by_l1_block: BTreeMap, + /// The lowest L1 block a new record may claim. + floor: Floor, +} + +/// The lowest L1 block a new record may claim: at least the L1 block the current safe head +/// depends on. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Floor { + /// The L1 block the current safe head was recorded at. A safe head derived below it cannot + /// have become safe before its parent, so it is recorded there. + Recorded(BlockNumHash), + /// Only a bound on the L1 blocks the current safe head depends on, with no block to record + /// at. A safe head derived below it is not recorded. + Bound(u64), +} + +impl Default for Floor { + fn default() -> Self { + Self::Bound(0) + } +} + +impl SafeHeadHistory { + /// Records `safe_head`, confirmed from attributes the pipeline produced at `derived_from`, at + /// `derived_from` raised to the floor, and returns that L1 block. `None` when it is below a + /// floor known only as a bound, and so is not recorded. + /// + /// The returned block bounds the L1 data `safe_head` depends on, and is the earliest L1 block + /// it can have become safe at. Before Holocene that is where it became safe. After Holocene a + /// node that reset can derive it from a batch its first pass dropped as a future batch, and + /// the first pass took it from a later one. + pub(crate) fn record( + &mut self, + safe_head: L2BlockInfo, + derived_from: BlockNumHash, + ) -> Option { + let safe_at = match self.floor { + Floor::Recorded(floor) if derived_from.number < floor.number => floor, + Floor::Bound(bound) if derived_from.number < bound => return None, + _ => derived_from, + }; + + self.floor = Floor::Recorded(safe_at); + self.by_l1_block + .insert(safe_at.number, DerivedSafeHead { safe_head, derived_from: safe_at }); + while self.by_l1_block.len() > MAX_ENTRIES { + self.by_l1_block.pop_first(); + } + Some(safe_at) + } + + /// The recorded safe heads, newest first. + pub(crate) fn newest_first(&self) -> Vec { + self.by_l1_block.values().rev().copied().collect() + } + + /// Derivation restarts from `safe_head`: forgets the safe heads above it. + /// + /// If `safe_head` is recorded, the floor is the L1 block it was recorded at. Otherwise + /// nothing kept is known to lead to it, so the history is cleared and the floor is bounded by + /// the L1 block by which derivation must have decided `safe_head`: one past the end of its + /// epoch's sequencing window. + pub(crate) fn reset_to(&mut self, safe_head: L2BlockInfo, seq_window_size: u64) { + self.by_l1_block.retain(|_, derived| { + derived.safe_head.block_info.number <= safe_head.block_info.number + }); + + match self.by_l1_block.last_key_value() { + Some((_, derived)) if derived.safe_head == safe_head => { + self.floor = Floor::Recorded(derived.derived_from); + } + _ => { + self.by_l1_block.clear(); + self.floor = Floor::Bound(safe_head.l1_origin.number + seq_window_size + 1); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use alloy_primitives::B256; + use kona_protocol::BlockInfo; + + const SEQ_WINDOW_SIZE: u64 = 10; + + fn l2_block(number: u64) -> L2BlockInfo { + L2BlockInfo { + block_info: BlockInfo { + number, + hash: B256::with_last_byte(number as u8), + parent_hash: B256::with_last_byte(number.wrapping_sub(1) as u8), + timestamp: number, + }, + ..Default::default() + } + } + + fn l1_block(number: u64) -> BlockNumHash { + BlockNumHash { number, hash: B256::repeat_byte(number as u8) } + } + + fn record(history: &mut SafeHeadHistory, number: u64, derived_from: u64) -> Option { + history.record(l2_block(number), l1_block(derived_from)).map(|safe_at| safe_at.number) + } + + /// `(safe head number, L1 block number)` of each record, newest first. + fn recorded(history: &SafeHeadHistory) -> Vec<(u64, u64)> { + history + .newest_first() + .iter() + .map(|d| (d.safe_head.block_info.number, d.derived_from.number)) + .collect() + } + + #[test] + fn keeps_the_highest_safe_head_per_l1_block_newest_first() { + let mut history = SafeHeadHistory::default(); + record(&mut history, 11, 100); + record(&mut history, 12, 100); + record(&mut history, 13, 102); + + assert_eq!( + history.newest_first(), + vec![ + DerivedSafeHead { safe_head: l2_block(13), derived_from: l1_block(102) }, + DerivedSafeHead { safe_head: l2_block(12), derived_from: l1_block(100) }, + ] + ); + } + + #[test] + fn bounds_the_entries_kept() { + let mut history = SafeHeadHistory::default(); + for i in 0..(MAX_ENTRIES as u64 + 10) { + record(&mut history, 1 + i, 100 + i); + } + + let recorded = history.newest_first(); + assert_eq!(recorded.len(), MAX_ENTRIES); + assert_eq!(recorded.last().unwrap().derived_from.number, 110); + } + + /// After a reset to a recorded safe head, the pipeline replays from before its L1 origin and + /// can produce the next block from an L1 block below the one that safe head was recorded + /// with. That block depends on its parent, so it became safe at its parent's L1 block, where + /// it is recorded instead. + #[test] + fn replay_after_reset_records_at_the_kept_safe_heads_l1_block() { + let mut history = SafeHeadHistory::default(); + record(&mut history, 11, 100); + record(&mut history, 12, 106); + record(&mut history, 13, 107); + + history.reset_to(l2_block(12), SEQ_WINDOW_SIZE); + assert_eq!(record(&mut history, 13, 100), Some(106)); + assert_eq!(recorded(&history), vec![(13, 106), (11, 100)]); + + assert_eq!(record(&mut history, 14, 108), Some(108)); + assert_eq!(recorded(&history), vec![(14, 108), (13, 106), (11, 100)]); + } + + /// A reset to a safe head the history did not record, such as one from a full reset, leaves + /// that safe head's L1 dependency unknown beyond the end of its sequencing window. + #[test] + fn reset_to_an_unrecorded_safe_head_floors_at_its_sequencing_window() { + let mut history = SafeHeadHistory::default(); + record(&mut history, 11, 100); + record(&mut history, 12, 101); + + let mut safe_head = l2_block(11); + safe_head.l1_origin.number = 95; + safe_head.block_info.hash = B256::repeat_byte(0xaa); + history.reset_to(safe_head, SEQ_WINDOW_SIZE); + assert!(history.newest_first().is_empty()); + + // Derived below the end of the sequencing window: not recorded. + assert_eq!(record(&mut history, 12, 105), None); + assert!(history.newest_first().is_empty()); + + // From the floor on, recording resumes. + assert_eq!(record(&mut history, 13, 106), Some(106)); + assert_eq!(recorded(&history), vec![(13, 106)]); + } +} diff --git a/rust/kona/crates/node/service/src/actors/derivation/state_machine.rs b/rust/kona/crates/node/service/src/actors/derivation/state_machine.rs index 3b5cef5d88b..8b22b3cc383 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/state_machine.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/state_machine.rs @@ -197,7 +197,7 @@ impl DerivationStateMachine { if let DerivationStateUpdate::NewAttributesConfirmed(safe_head) = state_update && safe_head.block_info.hash == self.confirmed_safe_head.block_info.hash { - info!(target: "derivation", ?safe_head, "Re-received safe head. Skipping state transition."); + info!(target: "derivation", ?safe_head, "Re-received the confirmed safe head."); } info!(target: "derivation", state=?self.state, ?state_update, "Executing derivation state update."); diff --git a/rust/kona/crates/node/service/src/actors/engine/actor.rs b/rust/kona/crates/node/service/src/actors/engine/actor.rs index b45bfa62238..e3fd09a97fd 100644 --- a/rust/kona/crates/node/service/src/actors/engine/actor.rs +++ b/rust/kona/crates/node/service/src/actors/engine/actor.rs @@ -1,5 +1,6 @@ use crate::{ - BuildRequest, CanonicalizeRequest, EngineDerivationClient, EngineError, NodeActor, SealRequest, + BuildRequest, CanonicalizeRequest, EngineDerivationClient, EngineError, NodeActor, ResetKind, + ResetRequest, SealRequest, }; use async_trait::async_trait; use kona_derive::{ResetSignal, Signal}; @@ -55,7 +56,7 @@ where /// instead of fetching it back from the execution layer. block_sink: Arc, /// A reset is retained until successful; RPC failure must not lose the request. - reset_pending: bool, + pending_reset: Option, /// Callers waiting for the retained reset to finish. reset_waiters: Vec>>, /// Earliest next attempt after a dependency failure. @@ -89,7 +90,7 @@ where unsafe_head_tx, inbound_request_rx, block_sink, - reset_pending: false, + pending_reset: None, reset_waiters: Vec::new(), retry_at: Instant::now(), retry_delay: Duration::from_millis(100), @@ -113,19 +114,25 @@ where self.retry_delay = (self.retry_delay * 2).min(Duration::from_secs(5)); } - fn begin_reset(&mut self) { - if !self.reset_pending { - self.engine.clear(); - self.last_skipped_task_error = None; - self.reset_pending = true; - self.retry_at = Instant::now(); - } + /// Retains a reset of `kind`, combined with any reset already retained. + fn begin_reset(&mut self, kind: ResetKind) { + self.pending_reset = Some(match self.pending_reset.take() { + // A pending full reset stays full; a pending reset after an L1 reorg is replaced. + Some(ResetKind::Full) => ResetKind::Full, + Some(ResetKind::AfterL1Reorg(_)) => kind, + None => { + self.engine.clear(); + self.last_skipped_task_error = None; + self.retry_at = Instant::now(); + kind + } + }); } async fn attempt_reset(&mut self) -> Result<(), EngineError> { match self.reset().await { Ok(()) => { - self.reset_pending = false; + self.pending_reset = None; self.retry_delay = Duration::from_millis(100); if !self.el_sync_complete && self.engine.state().el_sync_finished { self.mark_el_sync_complete_and_notify_derivation_actor().await?; @@ -143,7 +150,18 @@ where /// Resets the inner [`Engine`] and propagates the reset to the derivation actor. async fn reset(&mut self) -> Result<(), EngineError> { // Reset the engine. - let l2_safe_head = self.engine.reset(self.client.clone(), self.rollup.clone()).await?; + let l2_safe_head = match &self.pending_reset { + Some(ResetKind::AfterL1Reorg(recent_safe_heads)) => { + self.engine + .reset_after_l1_reorg( + self.client.clone(), + self.rollup.clone(), + recent_safe_heads, + ) + .await? + } + _ => self.engine.reset(self.client.clone(), self.rollup.clone()).await?, + }; // The sequencer must observe the same reset head as RPC readers before its reset // acknowledgement can release a queued start command. self.publish_unsafe_head(); @@ -202,7 +220,7 @@ where } EngineTaskErrorSeverity::Reset => { warn!(target: "engine", ?err, "Received reset request"); - self.begin_reset(); + self.begin_reset(ResetKind::Full); return Ok(()); } EngineTaskErrorSeverity::Flush => { @@ -236,7 +254,7 @@ where if !self.el_sync_complete && self.engine.state().el_sync_finished { if self.engine.state().sync_state.established_finalized_head().is_none() { - self.begin_reset(); + self.begin_reset(ResetKind::Full); } else { self.mark_el_sync_complete_and_notify_derivation_actor().await?; } @@ -292,7 +310,7 @@ where async fn step(&mut self) -> Result<(), Self::Error> { if Instant::now() >= self.retry_at { - if self.reset_pending { + if self.pending_reset.is_some() { self.attempt_reset().await?; } else { self.drain().await?; @@ -311,7 +329,7 @@ where request.ok_or(EngineError::ChannelClosed)? } _ = time::sleep_until(self.retry_at), - if self.reset_pending || !self.engine.is_empty() => return Ok(()), + if self.pending_reset.is_some() || !self.engine.is_empty() => return Ok(()), }; match request { @@ -357,8 +375,9 @@ where EngineActorRequest::Reset(reset_request) => { warn!(target: "engine", "Received reset request"); - self.begin_reset(); - self.reset_waiters.push(reset_request.result_tx); + let ResetRequest { result_tx, kind } = *reset_request; + self.begin_reset(kind); + self.reset_waiters.push(result_tx); } EngineActorRequest::Canonicalize(request) => { let CanonicalizeRequest { payload, parent, result_tx } = *request; diff --git a/rust/kona/crates/node/service/src/actors/engine/mod.rs b/rust/kona/crates/node/service/src/actors/engine/mod.rs index ef8308ab059..69bacfc43a3 100644 --- a/rust/kona/crates/node/service/src/actors/engine/mod.rs +++ b/rust/kona/crates/node/service/src/actors/engine/mod.rs @@ -14,7 +14,7 @@ pub use error::EngineError; pub use kona_engine::{ BuildRequest, CanonicalizeRequest, EngineActorRequest, EngineRequestError as EngineClientError, - EngineRequestResult as EngineClientResult, ResetRequest, SealRequest, + EngineRequestResult as EngineClientResult, ResetKind, ResetRequest, SealRequest, }; #[cfg(test)] diff --git a/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs b/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs index c7399efc6e6..0d4959154b4 100644 --- a/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs +++ b/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs @@ -193,7 +193,7 @@ async fn delegated_poll_during_el_sync_moves_safe_labels_and_skips_finalization( assert!(matches!(actor.step().await, Err(EngineError::ChannelClosed))); assert!(actor.engine.is_empty()); - assert!(!actor.reset_pending); + assert!(actor.pending_reset.is_none()); assert_eq!(state_rx.borrow().sync_state, expected_sync_state); l1.assert_finished(); l2.assert_finished(); @@ -220,7 +220,10 @@ fn expect_reset_to_genesis( fn reset_request() -> (EngineActorRequest, mpsc::Receiver>) { let (result_tx, result_rx) = mpsc::channel(1); - (EngineActorRequest::Reset(Box::new(ResetRequest { result_tx })), result_rx) + ( + EngineActorRequest::Reset(Box::new(ResetRequest { result_tx, kind: ResetKind::Full })), + result_rx, + ) } /// How the finalization target is identified: delegated derivation pins the hash, and @@ -304,7 +307,7 @@ async fn finalization_target_survives_resets_below_it(#[case] target_kind: Targe .unwrap(); actor.step().await.unwrap(); assert_eq!(actor.engine.len(), 1, "only the unsafe import may remain queued"); - assert!(!actor.reset_pending); + assert!(actor.pending_reset.is_none()); l2.expect("engine_newPayloadV2", PayloadStatus::from_status(PayloadStatusEnum::Valid)); l2.expect_forkchoice_updated( forkchoice(unsafe_head, genesis, genesis), @@ -323,7 +326,7 @@ async fn finalization_target_survives_resets_below_it(#[case] target_kind: Targe requests.send(EngineActorRequest::ProcessSafeL2Signal(restored_safe.into())).await.unwrap(); actor.step().await.unwrap(); assert_eq!(actor.engine.len(), 1, "only the safe head update may remain queued"); - assert!(!actor.reset_pending); + assert!(actor.pending_reset.is_none()); assert_eq!(state_rx.borrow().sync_state.finalized_head(), genesis); // Once the safe head reaches the target, the repeated request finalizes it. @@ -385,7 +388,7 @@ async fn skipped_finalization_keeps_the_actor_running_until_a_reset_forgets_it() time::timeout(Duration::from_secs(1), actor.step()).await.unwrap().unwrap(); assert_eq!(actor.last_skipped_task_error, first_skip); - actor.begin_reset(); + actor.begin_reset(ResetKind::Full); assert_eq!(actor.last_skipped_task_error, None); l1.assert_finished(); l2.assert_finished(); diff --git a/rust/kona/crates/node/service/src/actors/engine/tests.rs b/rust/kona/crates/node/service/src/actors/engine/tests.rs index 0d0169c241f..d046a593c84 100644 --- a/rust/kona/crates/node/service/src/actors/engine/tests.rs +++ b/rust/kona/crates/node/service/src/actors/engine/tests.rs @@ -4,7 +4,9 @@ use alloy_primitives::B256; use alloy_rpc_types_engine::{ForkchoiceUpdated, PayloadStatus, PayloadStatusEnum}; use kona_engine::{ Engine, EngineActorRequest, EngineState, EngineSyncStateUpdate, FinalizeBlockId, NoopBlockSink, - test_utils::{TestAttributesBuilder, test_engine_client}, + test_utils::{ + TestAttributesBuilder, TestChain, expect_l1_block, expect_l1_head, test_engine_client, + }, }; use kona_genesis::RollupConfig; use kona_protocol::{BlockInfo, L2BlockInfo}; @@ -155,18 +157,15 @@ async fn admin_reset_recovers_and_completes_original_request( #[case] l1_failure: bool, #[case] read_delay: Duration, ) { - let block: ::BlockResponse = - Default::default(); - let mut config = RollupConfig::default(); - config.genesis.l2.hash = block.header.inner.hash_slow(); - config.genesis.l1.hash = alloy_consensus::Header::default().hash_slow(); - let config = Arc::new(config); + // Finalized, safe and unsafe heads 18, 20 and 26 of a real chain, so the reset checks the + // unsafe head's L1 origin. The sequencing window is long enough that the safe head search + // stops at the finalized head. + let chain = TestChain::new(); + let config = Arc::new(RollupConfig { seq_window_size: 1000, ..Default::default() }); let (client, l1, l2) = test_engine_client(config.clone()); let client = Arc::new(client); if l1_failure { - for tag in ["finalized", "safe", "latest"] { - l2.expect_params("eth_getBlockByNumber", serde_json::json!([tag, true]), &block); - } + chain.expect_heads(&l2, 18, 20, 26); l1.expect_error("eth_getBlockByNumber"); } else { l2.expect_error("eth_getBlockByNumber"); @@ -197,14 +196,13 @@ async fn admin_reset_recovers_and_completes_original_request( actor.step().await.unwrap(); // retain the reset actor.step().await.unwrap(); // failed attempt, then wait for backoff assert!(!reset.is_finished()); - for _ in 0..3 { - l2.expect_with_delay("eth_getBlockByNumber", &block, read_delay); + for number in [18, 20, 26] { + l2.expect_with_delay("eth_getBlockByNumber", chain.rpc(number), read_delay); } - // The L1 genesis, the L2 genesis's origin, is canonical and is also the L1 head. - let mut l1_genesis = alloy_rpc_types_eth::Block::::default(); - l1_genesis.header.hash = l1_genesis.header.inner.hash_slow(); - for number in ["latest", "0x0"] { - l1.expect_params("eth_getBlockByNumber", serde_json::json!([number, false]), &l1_genesis); + expect_l1_head(&l1, 106); + expect_l1_block(&l1, 105, 106); + for number in (18..26).rev() { + chain.expect_by_hash(&l2, number); } l2.expect( "engine_forkchoiceUpdatedV3", @@ -222,16 +220,13 @@ async fn admin_reset_recovers_and_completes_original_request( time::timeout(Duration::from_secs(20), reset).await.unwrap().unwrap().unwrap(); // Reset acknowledgement must make the recovered heads visible to RPC readers, even // when stopped sequencing leaves the engine task queue empty. - let published = *state_rx.borrow(); - for head in [ - published.sync_state.unsafe_head(), - published.sync_state.local_safe_head(), - published.sync_state.safe_head(), - published.sync_state.finalized_head(), - ] { - assert_eq!(head.hash(), block.header.inner.hash_slow()); + // The unsafe head keeps its canonical L1 origin; the safe heads walk back to finalized. + let published = state_rx.borrow().sync_state; + assert_eq!(published.unsafe_head(), chain.info(26)); + for head in [published.local_safe_head(), published.safe_head(), published.finalized_head()] { + assert_eq!(head, chain.info(18)); } - assert_eq!(unsafe_head_rx.borrow().hash(), block.header.inner.hash_slow()); + assert_eq!(*unsafe_head_rx.borrow(), chain.info(26)); assert!(matches!( derivation_rx.recv().await.unwrap(), DerivationActorRequest::ProcessL1HeadUpdateRequest(_) @@ -242,7 +237,7 @@ async fn admin_reset_recovers_and_completes_original_request( panic!("expected reset signal carrying the recovered safe head"); }; assert!(matches!(*signal, kona_derive::Signal::Reset(reset) - if reset.l2_safe_head == published.sync_state.local_safe_head())); + if reset.l2_safe_head == published.local_safe_head())); assert!(matches!( derivation_rx.recv().await.unwrap(), DerivationActorRequest::ProcessEngineSyncCompletionRequest(_) diff --git a/rust/kona/crates/node/service/src/actors/mod.rs b/rust/kona/crates/node/service/src/actors/mod.rs index cb221342aab..b4888331601 100644 --- a/rust/kona/crates/node/service/src/actors/mod.rs +++ b/rust/kona/crates/node/service/src/actors/mod.rs @@ -15,7 +15,7 @@ mod engine; pub use engine::{ BuildRequest, CanonicalizeRequest, EngineActor, EngineActorRequest, EngineClientError, EngineClientResult, EngineConfig, EngineDerivationClient, EngineError, - QueuedEngineDerivationClient, ResetRequest, SealRequest, + QueuedEngineDerivationClient, ResetKind, ResetRequest, SealRequest, }; mod rpc; diff --git a/rust/kona/crates/node/service/src/actors/sequencer/engine_client.rs b/rust/kona/crates/node/service/src/actors/sequencer/engine_client.rs index e949c90c95a..32261399c2d 100644 --- a/rust/kona/crates/node/service/src/actors/sequencer/engine_client.rs +++ b/rust/kona/crates/node/service/src/actors/sequencer/engine_client.rs @@ -1,7 +1,7 @@ use crate::{ EngineClientError, EngineClientResult, actors::engine::{ - BuildRequest, CanonicalizeRequest, EngineActorRequest, ResetRequest, SealRequest, + BuildRequest, CanonicalizeRequest, EngineActorRequest, ResetKind, ResetRequest, SealRequest, }, }; use alloy_rpc_types_engine::PayloadId; @@ -70,7 +70,10 @@ impl SequencerEngineClient for QueuedSequencerEngineClient { let (result_tx, mut result_rx) = mpsc::channel(1); self.engine_actor_request_tx - .send(EngineActorRequest::Reset(Box::new(ResetRequest { result_tx }))) + .send(EngineActorRequest::Reset(Box::new(ResetRequest { + result_tx, + kind: ResetKind::Full, + }))) .await .map_err(|_| EngineClientError::RequestError("request channel closed.".to_string()))?; diff --git a/rust/kona/crates/node/service/src/lib.rs b/rust/kona/crates/node/service/src/lib.rs index 73a28c6b803..2258b593f15 100644 --- a/rust/kona/crates/node/service/src/lib.rs +++ b/rust/kona/crates/node/service/src/lib.rs @@ -25,8 +25,8 @@ pub use actors::{ NetworkBuilderError, NetworkConfig, NetworkDriver, NetworkDriverError, NetworkEngineClient, NetworkHandler, NodeActor, OriginSelector, QueuedDerivationEngineClient, QueuedEngineDerivationClient, QueuedL1WatcherDerivationClient, QueuedNetworkEngineClient, - QueuedSequencerEngineClient, QueuedUnsafePayloadGossipClient, ResetRequest, RpcActor, - RpcActorError, RunningDiscovery, SealRequest, SequencerActor, SequencerActorError, + QueuedSequencerEngineClient, QueuedUnsafePayloadGossipClient, ResetKind, ResetRequest, + RpcActor, RpcActorError, RunningDiscovery, SealRequest, SequencerActor, SequencerActorError, SequencerConfig, SequencerEngineClient, SignedPayload, SignerActor, SignerActorError, SyncStatusSource, SystemClock, UnsafePayloadGossipClient, UnsafePayloadGossipClientError, };