diff --git a/rust/kona/crates/node/engine/src/sync/error.rs b/rust/kona/crates/node/engine/src/sync/error.rs index d98b69d642b..a23e2e9e845 100644 --- a/rust/kona/crates/node/engine/src/sync/error.rs +++ b/rust/kona/crates/node/engine/src/sync/error.rs @@ -2,6 +2,7 @@ use alloy_eips::BlockId; use alloy_primitives::B256; +use alloy_rpc_types_engine::PayloadStatus; use alloy_transport::{RpcError, TransportErrorKind}; use kona_protocol::FromBlockError; use thiserror::Error; @@ -9,6 +10,15 @@ use thiserror::Error; /// An error that can occur during the sync start process. #[derive(Error, Debug)] pub enum SyncStartError { + /// The EL is still syncing; retry without trusting the retained checkpoint's labels. + #[error("Execution layer has not validated the sync-start checkpoint")] + CheckpointNotValidated, + /// The EL rejected the checkpoint forkchoice or returned an unexpected status; do not retry. + #[error("Execution layer rejected the sync-start checkpoint: {0:?}")] + CheckpointRejected(PayloadStatus), + /// The canonical L1 view changed during recovery; retry with a fresh view. + #[error("Canonical chain changed during sync-start recovery")] + ChainViewChanged, /// An rpc error occurred #[error("An RPC error occurred: {0}")] RpcError(#[from] RpcError), @@ -29,6 +39,9 @@ pub enum SyncStartError { /// Finalized block mismatch #[error("Finalized block mismatch. Expected {0}, Got {1}")] MismatchedFinalizedBlock(B256, B256), + /// Recovery would rewind the unsafe origin by more than op-node's five lookback windows. + #[error("L1 reorg exceeds five sync lookback windows")] + ReorgTooDeep, /// L1 origin mismatch. #[error("L1 origin mismatch")] L1OriginMismatch, diff --git a/rust/kona/crates/node/engine/src/sync/mod.rs b/rust/kona/crates/node/engine/src/sync/mod.rs index cdf468ba554..689eddf2974 100644 --- a/rust/kona/crates/node/engine/src/sync/mod.rs +++ b/rust/kona/crates/node/engine/src/sync/mod.rs @@ -1,7 +1,11 @@ //! Sync start algorithm for the OP Stack rollup node. +use alloy_eips::BlockId; +use alloy_primitives::B256; +use alloy_provider::ext::EngineApi; +use alloy_rpc_types_engine::PayloadStatusEnum; use kona_genesis::RollupConfig; -use kona_protocol::L2BlockInfo; +use kona_protocol::{BlockInfo, L2BlockInfo}; mod forkchoice; pub use forkchoice::L2ForkchoiceState; @@ -13,21 +17,29 @@ use tracing::info; use crate::EngineClient; -/// Searches for the latest [`L2ForkchoiceState`] that we can use to start the sync process with. -/// -/// - The *unsafe L2 block*: This is the highest L2 block whose L1 origin is a *plausible* -/// extension of the canonical L1 chain (as known to the rollup node). -/// - The *safe L2 block*: This is the highest L2 block whose epoch's sequencing window is -/// complete within the canonical L1 chain (as known to the rollup node). -/// - The *finalized L2 block*: This is the L2 block which is known to be fully derived from -/// finalized L1 block data. -/// -/// Plausible: meaning that the blockhash of the L2 block's L1 origin -/// (as reported in the L1 Attributes deposit within the L2 block) is not canonical at another -/// height in the L1 chain, and the same holds for all its ancestors. +/// Searches for a sync starting point using op-node's epoch and sequence consistency rules. +/// Unsafe origins may be ahead of L1; conservative safe recovery stops at an epoch boundary +/// beyond the sync lookback, or at finality. pub async fn find_starting_forkchoice( cfg: &RollupConfig, engine_client: &EngineClient, +) -> Result { + find_starting_forkchoice_with_checkpoints(cfg, engine_client, &[]).await +} + +/// A completed safe block and an L1 block bounding all of its inherited derivation inputs. +#[derive(Debug, Clone, Copy)] +pub(crate) struct SafeCheckpoint { + pub safe: L2BlockInfo, + pub source: BlockInfo, + /// The retained parent's input, which replay may encounter on a different L1 branch. + pub replay_source: Option, +} + +pub(crate) async fn find_starting_forkchoice_with_checkpoints( + cfg: &RollupConfig, + engine_client: &EngineClient, + checkpoints: &[SafeCheckpoint], ) -> Result { let mut current_fc = L2ForkchoiceState::current(cfg, engine_client).await?; info!( @@ -38,81 +50,287 @@ pub async fn find_starting_forkchoice( "Loaded current L2 EL forkchoice state" ); - // Search for the highest `unsafe` block, relative to the initial `unsafe` block's L1 origin, + let previous_unsafe = current_fc.un_safe; + let previous_safe = current_fc.safe; + let mut cursor = previous_unsafe; + let mut unsafe_candidate = Some(cursor); + let mut highest_canonical: Option = None; + let mut walker = SyncWalker::new(cfg, engine_client); + let mut check_checkpoint = !checkpoints.is_empty(); + let lookback = sync_lookback(cfg); + + // Port of op-node's normal FindL2Heads walk. Keep plausible heads ahead of L1, discard + // replaced origins, and validate every traversed child/parent epoch and sequence relation. loop { - let l1_origin = - engine_client.get_l1_block(current_fc.un_safe.l1_origin.hash.into()).await?; - info!( - target: "sync_start", - l1_origin = %current_fc.un_safe.l1_origin.number, - l2_unsafe = %current_fc.un_safe.block_info.number, - "Searching for L2 unsafe block with canonical L1 origin" - ); - - match l1_origin { - Some(_) => { - // Unsafe block has existing L1 origin. Continue with this head. - info!( - target: "sync_start", - l2_unsafe = %current_fc.un_safe.block_info.number, - "Found L2 unsafe block with canonical L1 origin" - ); - break; + if cursor.block_info.number < current_fc.finalized.block_info.number || + (cursor.block_info.number == current_fc.finalized.block_info.number && + cursor.hash() != current_fc.finalized.hash()) + { + return Err(SyncStartError::MismatchedFinalizedBlock( + current_fc.finalized.hash(), + cursor.hash(), + )); + } + let origin = walker.origin(cursor).await?; + if cursor.block_info.number == cfg.genesis.l2.number { + if cursor.hash() != cfg.genesis.l2.hash { + return Err(SyncStartError::InvalidL2GenesisHash( + cfg.genesis.l2.hash, + cursor.hash(), + )); } - None => { - let l2_parent_hash = current_fc.un_safe.block_info.parent_hash.into(); - let l2_parent = engine_client - .get_l2_block(l2_parent_hash) - .full() - .await? - .ok_or(SyncStartError::BlockNotFound(l2_parent_hash))?; - - current_fc.un_safe = - L2BlockInfo::from_block_and_genesis(&l2_parent.into_consensus(), &cfg.genesis)?; + if let Some(l1) = walker.l1 && + l1.hash != cfg.genesis.l1.hash + { + return Err(SyncStartError::InvalidL1GenesisHash(cfg.genesis.l1.hash, l1.hash)); } } - } - - // Search for the highest `safe` block that's L1 origin is at least older than the sequencing - // window, relative to the L1 origin of the `unsafe` block. - let mut safe_cursor = current_fc.un_safe; - loop { - info!( - target: "sync_start", - l1_origin = %safe_cursor.l1_origin.number, - l2_safe = %safe_cursor.block_info.number, - "Searching for L2 safe block beyond sequencing window" - ); - - let is_behind_sequence_window = - current_fc.un_safe.l1_origin.number.saturating_sub(cfg.seq_window_size) > - safe_cursor.l1_origin.number; - let is_finalized = safe_cursor.block_info.hash == current_fc.finalized.block_info.hash; - let is_genesis = safe_cursor.block_info.hash == cfg.genesis.l2.hash; - if is_behind_sequence_window || is_finalized || is_genesis { - info!( - target: "sync_start", - l2_safe = %safe_cursor.block_info.number, - is_behind_sequence_window, - is_finalized, - is_genesis, - "Found suitable L2 safe block" - ); - current_fc.safe = safe_cursor; + if unsafe_candidate.is_none() { + unsafe_candidate = Some(cursor); + if cursor.l1_origin.number.saturating_add(5u64.saturating_mul(lookback)) < + previous_unsafe.l1_origin.number + { + return Err(SyncStartError::ReorgTooDeep); + } + } + match origin { + OriginStatus::Ahead => highest_canonical = None, + OriginStatus::Canonical => { + highest_canonical.get_or_insert(cursor); + } + OriginStatus::Replaced => { + unsafe_candidate = None; + highest_canonical = None; + check_checkpoint = !checkpoints.is_empty(); + } + } + if cursor.block_info.number == current_fc.finalized.block_info.number { + current_fc.un_safe = unsafe_candidate.ok_or(SyncStartError::L1OriginMismatch)?; + current_fc.safe = cursor; + break; + } + if check_checkpoint && origin != OriginStatus::Replaced { + check_checkpoint = false; + current_fc.un_safe = unsafe_candidate.ok_or(SyncStartError::L1OriginMismatch)?; + if let Some(safe) = + retained_safe_checkpoint(engine_client, ¤t_fc, checkpoints).await? + { + info!(target: "sync_start", safe = %safe, "Retaining completed safe checkpoint with canonical L1 input"); + current_fc.safe = safe; + return Ok(current_fc); + } + // Checkpoint validation may have observed a newer chain view. Restart this origin + // by number before doing the conservative walk in that view. + walker.l1 = None; + highest_canonical = None; + continue; + } + // Use the parent of an epoch boundary below both the previous safe head and a full + // canonical lookback window. Finality above is the lower bound. + let ready = cursor.block_info.number <= previous_safe.block_info.number && + highest_canonical.is_some_and(|high| { + cursor.l1_origin.number.saturating_add(lookback) < high.l1_origin.number + }) && + cursor.seq_num == 0; + cursor = walker.parent(cursor).await?; + if ready { + current_fc.un_safe = unsafe_candidate.ok_or(SyncStartError::L1OriginMismatch)?; + current_fc.safe = cursor; break; } - let block = engine_client - .get_l2_block(safe_cursor.block_info.parent_hash.into()) - .full() - .await? - .ok_or(SyncStartError::BlockNotFound(safe_cursor.block_info.parent_hash.into()))?; - safe_cursor = L2BlockInfo::from_block_and_genesis(&block.into_consensus(), &cfg.genesis)?; } // Leave the finalized block as-is, and return the current forkchoice. Ok(current_fc) } +/// Matches op-node Config.SyncLookback, including the longer Alt-DA challenge/resolve window. +pub(crate) fn sync_lookback(cfg: &RollupConfig) -> u64 { + let alt_da = cfg.alt_da_config.as_ref(); + alt_da.map_or(cfg.seq_window_size, |da| { + cfg.seq_window_size.max( + da.da_challenge_window + .unwrap_or_default() + .saturating_add(da.da_resolve_window.unwrap_or_default()), + ) + }) +} + +#[derive(Debug, PartialEq, Eq)] +enum OriginStatus { + Canonical, + Replaced, + Ahead, +} + +/// The same L1-origin cursor and child/parent checks used by op-node `FindL2Heads`. +struct SyncWalker<'a> { + cfg: &'a RollupConfig, + client: &'a EngineClient, + l1: Option, +} + +impl<'a> SyncWalker<'a> { + const fn new(cfg: &'a RollupConfig, client: &'a EngineClient) -> Self { + Self { cfg, client, l1: None } + } + + async fn origin(&mut self, head: L2BlockInfo) -> Result { + if self.l1.is_none_or(|l1| l1.hash != head.l1_origin.hash) { + // A by-hash read is canonical only when it follows the parent of a known L1 block. + let id = if self.l1.is_some_and(|l1| l1.parent_hash == head.l1_origin.hash) { + head.l1_origin.hash.into() + } else { + BlockId::number(head.l1_origin.number) + }; + let block = self.client.get_l1_block(id).await?; + if block.is_none() && matches!(id, BlockId::Hash(_)) { + return Err(SyncStartError::ChainViewChanged); + } + self.l1 = block.map(|block| BlockInfo { + hash: block.header.hash, + number: block.header.number, + parent_hash: block.header.parent_hash, + timestamp: block.header.timestamp, + }); + } + Ok(match self.l1 { + None => OriginStatus::Ahead, + Some(l1) if l1.hash == head.l1_origin.hash => OriginStatus::Canonical, + Some(_) => OriginStatus::Replaced, + }) + } + + async fn parent(&self, child: L2BlockInfo) -> Result { + let id = child.block_info.parent_hash.into(); + let block = + self.client.get_l2_block(id).full().await?.ok_or(SyncStartError::BlockNotFound(id))?; + let parent = + L2BlockInfo::from_block_and_genesis(&block.into_consensus(), &self.cfg.genesis)?; + let canonical_parent = + self.l1.filter(|l1| l1.hash == child.l1_origin.hash).map(|l1| l1.parent_hash); + check_epoch_and_sequence(child, parent, canonical_parent)?; + Ok(parent) + } +} + +// Ported from op-node/rollup/sync/start.go's L1-origin relation check. Compare full origin IDs: +// a same-height hash replacement is an epoch inconsistency, not a same-origin sequence step. +fn check_epoch_and_sequence( + child: L2BlockInfo, + parent: L2BlockInfo, + canonical_parent: Option, +) -> Result<(), SyncStartError> { + if parent.l1_origin != child.l1_origin { + if parent.l1_origin.number.checked_add(1) != Some(child.l1_origin.number) { + return Err(SyncStartError::L1OriginMismatch); + } + if child.seq_num != 0 { + return Err(SyncStartError::NonZeroSequenceNumber); + } + if canonical_parent.is_some_and(|hash| hash != parent.l1_origin.hash) { + return Err(SyncStartError::L1OriginMismatch); + } + } else if parent.seq_num.checked_add(1) != Some(child.seq_num) { + return Err(SyncStartError::InconsistentSequenceNumber); + } + Ok(()) +} + +/// A canonical origin alone does not prove safety: the batch may have been submitted later. +/// Retain only completed blocks whose actual L1 input and L2 ancestry are still canonical. +async fn retained_safe_checkpoint( + client: &EngineClient, + heads: &L2ForkchoiceState, + checkpoints: &[SafeCheckpoint], +) -> Result, SyncStartError> { + if checkpoints.is_empty() { + return Ok(None); + } + let anchor = + client.get_l1_block(BlockId::latest()).await?.ok_or(SyncStartError::ChainViewChanged)?; + // The L1 chain may have changed since the unsafe walk, before this anchor was read. + let mut walker = SyncWalker::new(client.cfg(), client); + if heads.un_safe != heads.finalized { + let status = walker.origin(heads.un_safe).await?; + if status == OriginStatus::Replaced || + (status == OriginStatus::Ahead && + heads.un_safe.l1_origin.number <= anchor.header.number) + { + return Err(SyncStartError::ChainViewChanged); + } + } + for checkpoint in checkpoints.iter().rev() { + if checkpoint.safe.block_info.number < heads.finalized.block_info.number || + checkpoint.safe.block_info.number > heads.un_safe.block_info.number || + checkpoint.source.number > anchor.header.number + { + continue; + } + let source = client.get_l1_block(checkpoint.source.number.into()).await?; + if source.is_none_or(|block| block.header.hash != checkpoint.source.hash) { + continue; + } + if let Some(parent_input) = + checkpoint.replay_source.filter(|input| *input != checkpoint.source) + { + let parent = client.get_l1_block(parent_input.number.into()).await?; + if parent.is_none_or(|block| block.header.hash != parent_input.hash) { + continue; + } + } + // Number lookups are on one canonical L2 chain. Recheck its unsafe anchor after the + // checkpoint lookup so an EL reorg cannot combine two different chain views. + let safe = client.get_l2_block(checkpoint.safe.block_info.number.into()).await?; + if safe.is_none_or(|block| block.header.hash != checkpoint.safe.hash()) { + continue; + } + let unsafe_anchor = client.get_l2_block(heads.un_safe.block_info.number.into()).await?; + if unsafe_anchor.is_none_or(|block| block.header.hash != heads.un_safe.hash()) { + return Err(SyncStartError::ChainViewChanged); + } + // Numeric ancestry alone does not establish epoch/sequence consistency. Apply the + // same edge checks as the conservative walk before accepting the retained checkpoint. + let mut cursor = heads.un_safe; + let mut ancestry = SyncWalker { cfg: walker.cfg, client, l1: walker.l1 }; + while cursor.block_info.number > checkpoint.safe.block_info.number { + cursor = ancestry.parent(cursor).await?; + if cursor.block_info.number > checkpoint.safe.block_info.number && + ancestry.origin(cursor).await? == OriginStatus::Replaced + { + return Ok(None); + } + } + if cursor != checkpoint.safe { + return Err(SyncStartError::ChainViewChanged); + } + // A successful consolidation can have received SYNCING. Numeric canonical lookups + // establish ancestry, but only a fresh VALID response establishes execution validity, + // including after an EL restart. Do not trust the permanent el_sync_finished latch. + let response = client + .fork_choice_updated_v3( + alloy_rpc_types_engine::ForkchoiceState { + head_block_hash: heads.un_safe.hash(), + safe_block_hash: checkpoint.safe.hash(), + finalized_block_hash: heads.finalized.hash(), + }, + None, + ) + .await?; + match &response.payload_status.status { + PayloadStatusEnum::Valid => {} + PayloadStatusEnum::Syncing => return Err(SyncStartError::CheckpointNotValidated), + _ => return Err(SyncStartError::CheckpointRejected(response.payload_status)), + } + let current_anchor = client.get_l1_block(anchor.header.number.into()).await?; + if current_anchor.is_none_or(|block| block.header.hash != anchor.header.hash) { + return Err(SyncStartError::ChainViewChanged); + } + return Ok(Some(checkpoint.safe)); + } + Ok(None) +} + #[cfg(test)] mod test { use alloy_provider::Network; @@ -146,3 +364,215 @@ mod test { assert_eq!(rpc_reported_hash, l2_block_info.block_info.hash); } } + +#[cfg(test)] +mod checkpoint_tests { + use super::*; + use crate::test_utils::{test_block_info, test_engine_client}; + use alloy_primitives::B256; + use alloy_rpc_types_engine::{ForkchoiceUpdated, PayloadStatus, PayloadStatusEnum}; + use serde_json::json; + use std::sync::Arc; + + #[derive(Debug, Clone, Copy)] + enum Scenario { + Retain, + InvalidInput, + InvalidL2, + ChangedL1, + ChangedL2, + NewerInvalid, + MissingHead, + SyncingAfterValid, + InvalidExecution, + AcceptedExecution, + CanonicalReplay, + ReorgedReplay, + ReorgedUnsafe, + } + + #[rstest::rstest] + #[case(Scenario::Retain)] + #[case(Scenario::InvalidInput)] + #[case(Scenario::InvalidL2)] + #[case(Scenario::ChangedL1)] + #[case(Scenario::ChangedL2)] + #[case(Scenario::NewerInvalid)] + #[case(Scenario::MissingHead)] + #[case(Scenario::SyncingAfterValid)] + #[case(Scenario::InvalidExecution)] + #[case(Scenario::AcceptedExecution)] + #[case(Scenario::CanonicalReplay)] + #[case(Scenario::ReorgedReplay)] + #[case(Scenario::ReorgedUnsafe)] + #[tokio::test] + async fn checkpoint_requires_canonical_batch_input_and_l2_ancestry(#[case] scenario: Scenario) { + let (client, l1, l2) = test_engine_client(Arc::new(RollupConfig::default())); + let safe = test_block_info(117029); + let heads = L2ForkchoiceState { safe, un_safe: safe, finalized: test_block_info(44302) }; + let mut source = alloy_rpc_types_eth::Block::::default(); + source.header.inner.number = 10679; + source.header.hash = B256::repeat_byte(1); + let mut checkpoint = SafeCheckpoint { + safe, + source: BlockInfo { number: 10679, hash: source.header.hash, ..Default::default() }, + replay_source: None, + }; + if matches!(scenario, Scenario::CanonicalReplay | Scenario::ReorgedReplay) { + checkpoint.replay_source = + Some(BlockInfo { number: 10678, hash: B256::repeat_byte(8), ..Default::default() }); + } + let mut anchor = source.clone(); + anchor.header.inner.number = 10690; + anchor.header.hash = B256::repeat_byte(2); + let mut safe_block = + alloy_rpc_types_eth::Block::::default(); + safe_block.header.hash = safe.hash(); + let mut unsafe_block = safe_block.clone(); + unsafe_block.header.hash = heads.un_safe.hash(); + let mut checkpoints = vec![checkpoint]; + if matches!(scenario, Scenario::NewerInvalid) { + checkpoints.push(SafeCheckpoint { + safe, + source: BlockInfo { number: 10680, ..checkpoint.source }, + replay_source: None, + }); + } + let attempts = if matches!(scenario, Scenario::SyncingAfterValid) { 2 } else { 1 }; + for attempt in 0..attempts { + l1.expect_params( + "eth_getBlockByNumber", + json!(["latest", false]), + (!matches!(scenario, Scenario::MissingHead)).then_some(&anchor), + ); + if matches!(scenario, Scenario::MissingHead) { + break; + } + let mut origin = source.clone(); + origin.header.hash = if matches!(scenario, Scenario::ReorgedUnsafe) { + B256::repeat_byte(9) + } else { + heads.un_safe.l1_origin.hash + }; + l1.expect_params("eth_getBlockByNumber", json!(["0x0", false]), origin); + if matches!(scenario, Scenario::ReorgedUnsafe) { + break; + } + if matches!(scenario, Scenario::NewerInvalid) { + l1.expect_params( + "eth_getBlockByNumber", + json!(["0x29b8", false]), + Option::::None, + ); + } + if matches!(scenario, Scenario::InvalidInput) { + source.header.hash = B256::ZERO; + } + l1.expect_params("eth_getBlockByNumber", json!(["0x29b7", false]), &source); + if matches!(scenario, Scenario::InvalidInput) { + break; + } + if let Some(parent_input) = checkpoint.replay_source { + let mut parent = source.clone(); + parent.header.hash = if matches!(scenario, Scenario::ReorgedReplay) { + B256::ZERO + } else { + parent_input.hash + }; + l1.expect_params("eth_getBlockByNumber", json!(["0x29b6", false]), parent); + if matches!(scenario, Scenario::ReorgedReplay) { + break; + } + } + if matches!(scenario, Scenario::InvalidL2) { + safe_block.header.hash = B256::ZERO; + } + l2.expect_params( + "eth_getBlockByNumber", + json!([format!("{:#x}", safe.block_info.number), false]), + &safe_block, + ); + if matches!(scenario, Scenario::InvalidL2) { + break; + } + if matches!(scenario, Scenario::ChangedL2) { + unsafe_block.header.hash = B256::ZERO; + } + l2.expect_params( + "eth_getBlockByNumber", + json!([format!("{:#x}", heads.un_safe.block_info.number), false]), + &unsafe_block, + ); + if matches!(scenario, Scenario::ChangedL2) { + break; + } + let status = match scenario { + Scenario::InvalidExecution => { + PayloadStatusEnum::Invalid { validation_error: "invalid state root".into() } + } + Scenario::AcceptedExecution => PayloadStatusEnum::Accepted, + _ if attempt == 1 => PayloadStatusEnum::Syncing, + _ => PayloadStatusEnum::Valid, + }; + let payload_status = + PayloadStatus::from_status(status).with_latest_valid_hash(B256::repeat_byte(9)); + l2.expect_params("engine_forkchoiceUpdatedV3", json!([{ + "headBlockHash": heads.un_safe.hash(), "safeBlockHash": safe.hash(), "finalizedBlockHash": heads.finalized.hash() + }, null]), ForkchoiceUpdated::new(payload_status)); + if attempt == 1 || + matches!(scenario, Scenario::InvalidExecution | Scenario::AcceptedExecution) + { + break; + } + let mut recheck = anchor.clone(); + if matches!(scenario, Scenario::ChangedL1) { + recheck.header.hash = B256::ZERO; + } + l1.expect_params("eth_getBlockByNumber", json!(["0x29c2", false]), recheck); + } + for attempt in 0..attempts { + let selected = retained_safe_checkpoint(&client, &heads, &checkpoints).await; + match scenario { + Scenario::SyncingAfterValid if attempt == 1 => { + let err = selected.unwrap_err(); + assert!(matches!(err, SyncStartError::CheckpointNotValidated)); + assert!(crate::EngineResetError::SyncStart(err).is_temporary()); + } + Scenario::InvalidExecution | Scenario::AcceptedExecution => { + let err = crate::EngineResetError::SyncStart(selected.unwrap_err()); + assert!(!err.is_temporary(), "execution rejection must not retry: {err}"); + let diagnostic = err.to_string(); + assert!(diagnostic.contains(&B256::repeat_byte(9).to_string())); + if matches!(scenario, Scenario::InvalidExecution) { + assert!(diagnostic.contains("invalid state root")); + } + } + Scenario::ChangedL1 | + Scenario::ChangedL2 | + Scenario::MissingHead | + Scenario::ReorgedUnsafe => { + assert!(matches!(selected, Err(SyncStartError::ChainViewChanged))) + } + Scenario::InvalidInput | Scenario::InvalidL2 | Scenario::ReorgedReplay => { + assert_eq!(selected.unwrap(), None) + } + _ => assert_eq!(selected.unwrap(), Some(safe)), + } + } + l1.assert_finished(); + l2.assert_finished(); + } + + #[tokio::test] + async fn startup_without_accepted_checkpoints_uses_conservative_walkback() { + let (client, l1, l2) = test_engine_client(Arc::new(RollupConfig::default())); + let block = test_block_info(1); + let heads = L2ForkchoiceState { safe: block, un_safe: block, finalized: block }; + assert_eq!(retained_safe_checkpoint(&client, &heads, &[]).await.unwrap(), None); + l1.assert_finished(); + l2.assert_finished(); + } +} + +#[cfg(test)] +mod tests; diff --git a/rust/kona/crates/node/engine/src/sync/tests.rs b/rust/kona/crates/node/engine/src/sync/tests.rs new file mode 100644 index 00000000000..1cb7013e29a --- /dev/null +++ b/rust/kona/crates/node/engine/src/sync/tests.rs @@ -0,0 +1,429 @@ +//! Regression vectors from op-node/rollup/sync/start_test.go, plus malformed traversal edges. + +use super::*; +use crate::test_utils::{RpcMock, test_engine_client, test_l2_block}; +use alloy_rpc_types_eth::Block; +use serde_json::json; +use std::sync::Arc; + +type L1Block = Block; +type L2Block = Block; + +fn l1_block(number: u64) -> L1Block { + let mut block = L1Block::default(); + block.header.inner.number = number; + block.header.hash = B256::with_last_byte(number as u8); + block.header.inner.parent_hash = B256::with_last_byte(number.saturating_sub(1) as u8); + block +} + +fn info(block: &L2Block, cfg: &RollupConfig) -> L2BlockInfo { + L2BlockInfo::from_block_and_genesis(&block.clone().into_consensus(), &cfg.genesis).unwrap() +} + +fn expect_heads(mock: &RpcMock, finalized: &L2Block, safe: &L2Block, head: &L2Block) { + for (tag, block) in [("finalized", finalized), ("safe", safe), ("latest", head)] { + mock.expect_params("eth_getBlockByNumber", json!([tag, true]), block); + } +} + +fn expect_parent(mock: &RpcMock, parent: &L2Block) { + mock.expect_params("eth_getBlockByHash", json!([parent.header.hash, true]), parent); +} + +// Same histories and expected labels as op-node's small-reorg, L1-ahead and L2-ahead vectors. +// The final case keeps the previous safe head below the lookback boundary. +#[rstest::rstest] +#[case::small_reorg(8, 8, Some(7), 1, 7, (6, 2), &[(7,false),(6,true),(5,true),(4,true),(3,true)])] +#[case::l1_ahead(5, 6, None, 0, 3, (4, 0), &[(4,false),(3,true),(2,true),(1,true)])] +#[case::l2_ahead_after_reorg(6, 4, None, 1, 3, (5, 1), &[(5,false),(4,false),(3,false),(2,true),(1,true)])] +#[case::prior_safe_limits_recovery(12, 12, None, 1, 3, (11, 2), &[(11,false),(10,true),(9,true),(8,true),(7,true),(6,true),(5,true),(4,true),(3,true)])] +#[tokio::test] +async fn matches_op_node_sync_start_vectors( + #[case] l2_len: usize, + #[case] l1_len: usize, + #[case] replaced: Option, + #[case] finalized: usize, + #[case] safe: usize, + #[case] expected: (usize, usize), + #[case] reads: &[(usize, bool)], +) { + let (expected_unsafe, expected_safe) = expected; + let mut blocks = vec![]; + for i in 0..l2_len { + let parent = blocks.last().map_or(B256::ZERO, |b: &L2Block| b.header.hash); + blocks.push(test_l2_block(i as u64, i as u64, 0, parent)); + } + let mut cfg = RollupConfig { seq_window_size: 2, ..Default::default() }; + cfg.genesis.l2.hash = blocks[0].header.hash; + let (client, l1, l2) = test_engine_client(Arc::new(cfg.clone())); + expect_heads(&l2, &blocks[finalized], &blocks[safe], blocks.last().unwrap()); + for &(number, by_hash) in reads { + let mut block = (number < l1_len).then(|| l1_block(number as u64)); + if replaced == Some(number) { + block.as_mut().unwrap().header.hash = B256::repeat_byte(0xff); + } + let (method, id) = if by_hash { + ("eth_getBlockByHash", json!(B256::with_last_byte(number as u8))) + } else { + ("eth_getBlockByNumber", json!(format!("{number:#x}"))) + }; + l1.expect_params(method, json!([id, false]), block); + } + for parent in (expected_safe..l2_len - 1).rev() { + expect_parent(&l2, &blocks[parent]); + } + let result = find_starting_forkchoice(&cfg, &client).await.unwrap(); + assert_eq!(result.un_safe, info(&blocks[expected_unsafe], &cfg)); + assert_eq!(result.safe, info(&blocks[expected_safe], &cfg)); + assert_eq!(result.finalized, info(&blocks[finalized], &cfg)); + l1.assert_finished(); + l2.assert_finished(); +} + +#[rstest::rstest] +#[case::sequencing_window(false, 1)] +#[case::alt_da_window(true, 0)] +#[tokio::test] +async fn safe_head_is_last_block_before_epoch_boundary( + #[case] alt_da: bool, + #[case] expected: usize, +) { + let mut blocks = vec![]; + for i in 0..8 { + let parent = blocks.last().map_or(B256::ZERO, |b: &L2Block| b.header.hash); + blocks.push(test_l2_block(i, i / 2, i % 2, parent)); + } + let mut cfg = RollupConfig { seq_window_size: 1, ..Default::default() }; + if alt_da { + cfg.alt_da_config = Some(kona_genesis::AltDAConfig { + da_challenge_window: Some(1), + da_resolve_window: Some(1), + ..Default::default() + }); + } + cfg.genesis.l2.hash = blocks[0].header.hash; + let (client, l1, l2) = test_engine_client(Arc::new(cfg.clone())); + expect_heads(&l2, &blocks[0], &blocks[7], &blocks[7]); + l1.expect_params("eth_getBlockByNumber", json!(["0x3", false]), l1_block(3)); + for number in (if alt_da { 0 } else { 1 }..=2).rev() { + l1.expect_params( + "eth_getBlockByHash", + json!([B256::with_last_byte(number), false]), + l1_block(number as u64), + ); + } + for parent in (expected..7).rev() { + expect_parent(&l2, &blocks[parent]); + } + let result = find_starting_forkchoice(&cfg, &client).await.unwrap(); + assert_eq!(result.un_safe, info(&blocks[7], &cfg)); + assert_eq!(result.safe, info(&blocks[expected], &cfg)); + l1.assert_finished(); + l2.assert_finished(); +} + +#[derive(Debug, Clone, Copy)] +enum Path { + ReplacedUnsafe, + Conservative, + Checkpoint, +} +#[derive(Debug, Clone, Copy)] +enum Invalid { + EpochSkip, + NonzeroEpoch, + Sequence, + ParentHash, +} + +// Each traversal calls the production RPC client and decodes real L1-info deposits. A check +// that exists only in the conservative path cannot protect the retained-checkpoint path. +#[rstest::rstest] +#[case(Path::ReplacedUnsafe, Invalid::EpochSkip)] +#[case(Path::ReplacedUnsafe, Invalid::NonzeroEpoch)] +#[case(Path::ReplacedUnsafe, Invalid::Sequence)] +#[case(Path::Conservative, Invalid::EpochSkip)] +#[case(Path::Conservative, Invalid::NonzeroEpoch)] +#[case(Path::Conservative, Invalid::Sequence)] +#[case(Path::Conservative, Invalid::ParentHash)] +#[case(Path::Checkpoint, Invalid::EpochSkip)] +#[case(Path::Checkpoint, Invalid::NonzeroEpoch)] +#[case(Path::Checkpoint, Invalid::Sequence)] +#[case(Path::Checkpoint, Invalid::ParentHash)] +#[tokio::test] +async fn rejects_inconsistent_edges_on_every_path(#[case] path: Path, #[case] invalid: Invalid) { + let cfg = RollupConfig::default(); + let finalized = test_l2_block(1, 99, 0, B256::ZERO); + let parent = test_l2_block(2, 100, 0, finalized.header.hash); + let (origin, seq) = match invalid { + Invalid::EpochSkip => (102, 0), + Invalid::NonzeroEpoch => (101, 1), + Invalid::Sequence => (100, 2), + Invalid::ParentHash => (101, 0), + }; + let child = test_l2_block(3, origin, seq, parent.header.hash); + let mut l1_origin = l1_block(origin); + if matches!(invalid, Invalid::ParentHash) { + l1_origin.header.inner.parent_hash = B256::repeat_byte(0xff); + } + if matches!(path, Path::ReplacedUnsafe) { + l1_origin.header.hash = B256::repeat_byte(0xff); + } + let (client, l1, l2) = test_engine_client(Arc::new(cfg.clone())); + expect_heads(&l2, &finalized, &parent, &child); + l1.expect_params("eth_getBlockByNumber", json!([format!("{origin:#x}"), false]), &l1_origin); + let checkpoint = SafeCheckpoint { + safe: info(&parent, &cfg), + source: BlockInfo { number: 103, hash: l1_block(103).header.hash, ..Default::default() }, + replay_source: None, + }; + if matches!(path, Path::Checkpoint) { + l1.expect_params("eth_getBlockByNumber", json!(["latest", false]), l1_block(105)); + l1.expect_params( + "eth_getBlockByNumber", + json!([format!("{origin:#x}"), false]), + &l1_origin, + ); + l1.expect_params("eth_getBlockByNumber", json!(["0x67", false]), l1_block(103)); + l2.expect_params("eth_getBlockByNumber", json!(["0x2", false]), &parent); + l2.expect_params("eth_getBlockByNumber", json!(["0x3", false]), &child); + } + expect_parent(&l2, &parent); + let checkpoints = if matches!(path, Path::Checkpoint) { vec![checkpoint] } else { vec![] }; + let err = + find_starting_forkchoice_with_checkpoints(&cfg, &client, &checkpoints).await.unwrap_err(); + assert!( + match invalid { + Invalid::EpochSkip | Invalid::ParentHash => + matches!(err, SyncStartError::L1OriginMismatch), + Invalid::NonzeroEpoch => matches!(err, SyncStartError::NonZeroSequenceNumber), + Invalid::Sequence => matches!(err, SyncStartError::InconsistentSequenceNumber), + }, + "unexpected error: {err}" + ); + l1.assert_finished(); + l2.assert_finished(); +} + +#[test] +fn epoch_checks_compare_full_ids_and_reject_overflow() { + let mut parent = L2BlockInfo::default(); + let mut child = parent; + child.l1_origin.hash = B256::repeat_byte(1); // Same height, different hash is not a new epoch. + assert!(matches!( + check_epoch_and_sequence(child, parent, None), + Err(SyncStartError::L1OriginMismatch) + )); + parent.l1_origin.number = u64::MAX; + assert!(matches!( + check_epoch_and_sequence(child, parent, None), + Err(SyncStartError::L1OriginMismatch) + )); + child = parent; + parent.seq_num = u64::MAX; + assert!(matches!( + check_epoch_and_sequence(child, parent, None), + Err(SyncStartError::InconsistentSequenceNumber) + )); +} + +#[test] +fn alt_da_extends_sync_lookback() { + let mut cfg = RollupConfig { seq_window_size: 2, ..Default::default() }; + cfg.alt_da_config = Some(kona_genesis::AltDAConfig { + da_challenge_address: Some(alloy_primitives::Address::repeat_byte(1)), + da_challenge_window: Some(3), + da_resolve_window: Some(4), + ..Default::default() + }); + assert_eq!(sync_lookback(&cfg), 7); + cfg.seq_window_size = 10; + assert_eq!(sync_lookback(&cfg), 10); + cfg.seq_window_size = 2; + cfg.alt_da_config = None; + assert_eq!(sync_lookback(&cfg), 2); +} + +#[tokio::test] +async fn missing_l1_parent_retries_from_a_fresh_chain_view() { + let cfg = RollupConfig::default(); + let (client, l1, l2) = test_engine_client(Arc::new(cfg.clone())); + let mut walker = SyncWalker::new(&cfg, &client); + let mut head = L2BlockInfo::default(); + head.l1_origin.number = 101; + head.l1_origin.hash = l1_block(101).header.hash; + l1.expect_params("eth_getBlockByNumber", json!(["0x65", false]), l1_block(101)); + assert_eq!(walker.origin(head).await.unwrap(), OriginStatus::Canonical); + head.l1_origin.number = 100; + head.l1_origin.hash = l1_block(100).header.hash; + l1.expect_params( + "eth_getBlockByHash", + json!([head.l1_origin.hash, false]), + Option::::None, + ); + let err = walker.origin(head).await.unwrap_err(); + assert!(matches!(err, SyncStartError::ChainViewChanged)); + assert!(crate::EngineResetError::SyncStart(err).is_temporary()); + l1.assert_finished(); + l2.assert_finished(); +} + +#[rstest::rstest] +#[case::ahead(false)] +#[case::disappeared(true)] +#[tokio::test] +async fn checkpoint_accepts_future_unsafe_origin_but_retries_a_disappeared_one( + #[case] disappeared: bool, +) { + let cfg = RollupConfig::default(); + let finalized = test_l2_block(1, 99, 0, B256::ZERO); + let safe = test_l2_block(2, 100, 0, finalized.header.hash); + let head = test_l2_block(3, 101, 0, safe.header.hash); + let (client, l1, l2) = test_engine_client(Arc::new(cfg.clone())); + expect_heads(&l2, &finalized, &safe, &head); + l1.expect_params("eth_getBlockByNumber", json!(["0x65", false]), Option::::None); + l1.expect_params( + "eth_getBlockByNumber", + json!(["latest", false]), + l1_block(if disappeared { 101 } else { 100 }), + ); + l1.expect_params("eth_getBlockByNumber", json!(["0x65", false]), Option::::None); + if !disappeared { + l1.expect_params("eth_getBlockByNumber", json!(["0x64", false]), l1_block(100)); + for (number, block) in [(2, &safe), (3, &head)] { + l2.expect_params("eth_getBlockByNumber", json!([format!("{number:#x}"), false]), block); + } + expect_parent(&l2, &safe); + l2.expect_params( + "engine_forkchoiceUpdatedV3", + json!([{ + "headBlockHash": head.header.hash, "safeBlockHash": safe.header.hash, + "finalizedBlockHash": finalized.header.hash + }, null]), + alloy_rpc_types_engine::ForkchoiceUpdated::new( + alloy_rpc_types_engine::PayloadStatus::from_status( + alloy_rpc_types_engine::PayloadStatusEnum::Valid, + ), + ), + ); + l1.expect_params("eth_getBlockByNumber", json!(["0x64", false]), l1_block(100)); + } + let checkpoint = SafeCheckpoint { + safe: info(&safe, &cfg), + source: BlockInfo { number: 100, hash: l1_block(100).header.hash, ..Default::default() }, + replay_source: None, + }; + let result = find_starting_forkchoice_with_checkpoints(&cfg, &client, &[checkpoint]).await; + if disappeared { + assert!(matches!(result, Err(SyncStartError::ChainViewChanged))); + } else { + let result = result.unwrap(); + assert_eq!(result.un_safe, info(&head, &cfg)); + assert_eq!(result.safe, checkpoint.safe); + } + l1.assert_finished(); + l2.assert_finished(); +} + +#[tokio::test] +async fn rejects_reorgs_deeper_than_five_lookback_windows() { + let mut blocks = vec![]; + for number in 0..9 { + let parent = blocks.last().map_or(B256::ZERO, |b: &L2Block| b.header.hash); + blocks.push(test_l2_block(number, number, 0, parent)); + } + let mut cfg = RollupConfig { seq_window_size: 1, ..Default::default() }; + cfg.genesis.l2.hash = blocks[0].header.hash; + let (client, l1, l2) = test_engine_client(Arc::new(cfg.clone())); + expect_heads(&l2, &blocks[0], &blocks[0], &blocks[8]); + for number in (2..=8).rev() { + let mut replacement = l1_block(number); + replacement.header.hash = B256::with_last_byte(number as u8 + 100); + replacement.header.inner.parent_hash = B256::with_last_byte(number as u8 + 99); + l1.expect_params( + "eth_getBlockByNumber", + json!([format!("{number:#x}"), false]), + replacement, + ); + } + for parent in (2..8).rev() { + expect_parent(&l2, &blocks[parent]); + } + let err = find_starting_forkchoice(&cfg, &client).await.unwrap_err(); + assert!(matches!(err, SyncStartError::ReorgTooDeep)); + assert!(!crate::EngineResetError::SyncStart(err).is_temporary()); + l1.assert_finished(); + l2.assert_finished(); +} + +#[tokio::test] +async fn checkpoint_is_retried_after_an_ahead_suffix_hides_a_replaced_origin() { + let cfg = RollupConfig { seq_window_size: 2, ..Default::default() }; + let mut blocks = vec![]; + for number in 1..=5 { + let parent = blocks.last().map_or(B256::ZERO, |b: &L2Block| b.header.hash); + blocks.push(test_l2_block(number, number + 97, 0, parent)); + } + let (client, l1, l2) = test_engine_client(Arc::new(cfg.clone())); + expect_heads(&l2, &blocks[0], &blocks[1], &blocks[4]); + let mut replacement = l1_block(100); + replacement.header.hash = B256::repeat_byte(0xff); + let missing = Option::::None; + l1.expect_params("eth_getBlockByNumber", json!(["0x66", false]), missing.as_ref()); + l1.expect_params("eth_getBlockByNumber", json!(["latest", false]), &replacement); + l1.expect_params("eth_getBlockByNumber", json!(["0x66", false]), missing.as_ref()); + l1.expect_params("eth_getBlockByNumber", json!(["0x63", false]), l1_block(99)); + for block in [&blocks[1], &blocks[4]] { + l2.expect_params( + "eth_getBlockByNumber", + json!([format!("{:#x}", block.header.number), false]), + block, + ); + } + // The first probe and then the conservative walk both encounter the hidden replacement. + for pass in 0..2 { + if pass == 1 { + l1.expect_params("eth_getBlockByNumber", json!(["0x66", false]), missing.as_ref()); + } + for parent in [&blocks[3], &blocks[2]] { + expect_parent(&l2, parent); + } + l1.expect_params("eth_getBlockByNumber", json!(["0x65", false]), missing.as_ref()); + l1.expect_params("eth_getBlockByNumber", json!(["0x64", false]), &replacement); + } + expect_parent(&l2, &blocks[1]); + l1.expect_params("eth_getBlockByHash", json!([l1_block(99).header.hash, false]), l1_block(99)); + l1.expect_params("eth_getBlockByNumber", json!(["latest", false]), &replacement); + for _ in 0..2 { + l1.expect_params("eth_getBlockByNumber", json!(["0x63", false]), l1_block(99)); + } + for _ in 0..2 { + l2.expect_params("eth_getBlockByNumber", json!(["0x2", false]), &blocks[1]); + } + l2.expect_params( + "engine_forkchoiceUpdatedV3", + json!([{ + "headBlockHash": blocks[1].header.hash, "safeBlockHash": blocks[1].header.hash, + "finalizedBlockHash": blocks[0].header.hash + }, null]), + alloy_rpc_types_engine::ForkchoiceUpdated::new( + alloy_rpc_types_engine::PayloadStatus::from_status( + alloy_rpc_types_engine::PayloadStatusEnum::Valid, + ), + ), + ); + l1.expect_params("eth_getBlockByNumber", json!(["0x64", false]), replacement); + let safe = info(&blocks[1], &cfg); + let checkpoint = SafeCheckpoint { + safe, + source: BlockInfo { number: 99, hash: l1_block(99).header.hash, ..Default::default() }, + replay_source: None, + }; + let result = + find_starting_forkchoice_with_checkpoints(&cfg, &client, &[checkpoint]).await.unwrap(); + assert_eq!(result.un_safe, safe); + assert_eq!(result.safe, safe); + l1.assert_finished(); + l2.assert_finished(); +} diff --git a/rust/kona/crates/node/engine/src/task_queue/core.rs b/rust/kona/crates/node/engine/src/task_queue/core.rs index 04cae800e78..e12508188e8 100644 --- a/rust/kona/crates/node/engine/src/task_queue/core.rs +++ b/rust/kona/crates/node/engine/src/task_queue/core.rs @@ -4,11 +4,11 @@ use super::EngineTaskExt; use crate::{ EngineClient, EngineState, EngineSyncStateUpdate, EngineTask, EngineTaskError, EngineTaskErrorSeverity, Metrics, SyncStartError, SynchronizeTaskError, - find_starting_forkchoice, + sync::{SafeCheckpoint, find_starting_forkchoice_with_checkpoints, sync_lookback}, task_queue::{EngineTaskErrors, synchronize}, }; use kona_genesis::RollupConfig; -use kona_protocol::L2BlockInfo; +use kona_protocol::{BlockInfo, L2BlockInfo, OpAttributesWithParent}; use std::{cmp::Ordering, collections::BinaryHeap, sync::Arc}; use thiserror::Error; use tokio::sync::watch::Sender; @@ -37,6 +37,69 @@ impl Ord for QueuedTask { } } +/// L1 dependencies are inherited across replay and incomplete spans. Only completed spans +/// become reset candidates; their source bounds every input their ancestors depend on. +#[derive(Debug, Default)] +struct SafeCheckpoints { + entries: Vec, + source: Option, + // Keep the retained parent's dependency even if replay crosses onto a new L1 branch. + replay_source: Option, +} + +impl SafeCheckpoints { + fn record( + &mut self, + attributes: &OpAttributesWithParent, + cfg: &RollupConfig, + safe: L2BlockInfo, + ) { + let Some(mut source) = attributes.derived_from else { + *self = Self::default(); + return; + }; + if let Some(parent_source) = self.source { + if parent_source.number > source.number { + source = parent_source; + } + } else if attributes.parent.block_info.id() != cfg.genesis.l2 && + source.number < + attributes + .parent + .l1_origin + .number + .saturating_add(sync_lookback(cfg)) + .saturating_add(1) + { + // The parent's exact input is unknown after startup or a conservative reset. + // Wait until derivation reaches an L1 block that bounds its sequencing window. + return; + } + self.source = Some(source); + if !attributes.is_last_in_span { + return; + } + self.entries.retain(|entry| { + entry.safe.block_info.number < safe.block_info.number && + entry.source.number < source.number + }); + self.entries.push(SafeCheckpoint { safe, source, replay_source: self.replay_source }); + if self.entries.len() > 4096 { + self.entries.remove(0); + } + } + + fn reset_to(&mut self, safe: L2BlockInfo) { + self.entries.retain(|entry| entry.safe.block_info.number <= safe.block_info.number); + self.source = + self.entries.last().filter(|entry| entry.safe == safe).map(|entry| entry.source); + self.replay_source = self.source; + if self.source.is_none() { + self.entries.clear(); + } + } +} + /// The [`Engine`] task queue. /// /// Tasks of a shared [`EngineTask`] variant are processed in FIFO order, providing synchronization @@ -63,6 +126,8 @@ pub struct Engine { /// Preserve the task being retried ahead of newly enqueued work: it may have already /// performed part of an Engine API operation. active: Option, + /// Recent accepted completed-safe checkpoints; no unverified EL labels are recorded. + safe_checkpoints: SafeCheckpoints, } impl Engine { @@ -73,6 +138,7 @@ impl Engine { state_sender, tasks: BinaryHeap::default(), active: None, + safe_checkpoints: SafeCheckpoints::default(), next_sequence: 0, } } @@ -106,14 +172,20 @@ impl Engine { } /// Resets the engine by finding a plausible sync starting point via - /// [`find_starting_forkchoice`]. The state will be updated to the starting point, and a + /// [`crate::find_starting_forkchoice`]. The state will be updated to the starting point, and a /// forkchoice update will be enqueued in order to reorg the execution layer. pub async fn reset( &mut self, client: Arc, config: Arc, ) -> Result { - let start = find_starting_forkchoice(&config, client.as_ref()).await?; + let previous_safe = self.state.sync_state.safe_head(); + let start = find_starting_forkchoice_with_checkpoints( + &config, + client.as_ref(), + &self.safe_checkpoints.entries, + ) + .await?; // One attempt. The actor retains the reset request and retries transient failures. synchronize( @@ -128,6 +200,14 @@ impl Engine { ) .await?; + self.safe_checkpoints.reset_to(start.safe); + self.state_sender.send_replace(self.state); + if start.safe.block_info.number < previous_safe.block_info.number { + warn!(target: "engine", old_safe = previous_safe.block_info.number, + new_safe = start.safe.block_info.number, + rollback_blocks = previous_safe.block_info.number - start.safe.block_info.number, + "Reset demoted completed safe head"); + } metrics::counter!(Metrics::ENGINE_RESET_COUNT).increment(1); Ok(start.safe) @@ -159,6 +239,25 @@ impl Engine { return result; } + if let EngineTask::Consolidate(task) = task { + match &task.input { + crate::ConsolidateInput::Attributes(attributes) + if self.state.sync_state.local_safe_head().block_info.number == + attributes.block_number() => + { + self.safe_checkpoints.record( + attributes, + &task.cfg, + self.state.sync_state.local_safe_head(), + ); + } + // Delegated derivation supplies no L1 input dependency. + crate::ConsolidateInput::BlockInfo(_) => { + self.safe_checkpoints = SafeCheckpoints::default() + } + _ => {} + } + } self.state_sender.send_replace(self.state); self.active = None; result?; @@ -183,7 +282,11 @@ impl EngineResetError { /// Whether retrying the reset can recover without changing the node configuration. pub fn is_temporary(&self) -> bool { match self { - Self::SyncStart(SyncStartError::RpcError(_)) => true, + Self::SyncStart( + SyncStartError::RpcError(_) | + SyncStartError::ChainViewChanged | + SyncStartError::CheckpointNotValidated, + ) => true, Self::SyncStart(_) => false, Self::Forkchoice(err) => err.severity() != EngineTaskErrorSeverity::Critical, } @@ -268,14 +371,20 @@ mod recovery_tests { // A flush means the task already replaced its block with a deposits-only block. Keeping it // would rebuild that replacement on every later drain. + #[rstest::rstest] + #[case(true)] + #[case(false)] #[tokio::test] - async fn flushed_task_is_dropped_not_retried() { + async fn flushed_task_is_dropped_not_retried(#[case] completed_span: bool) { let parent = test_block_info(10); let deposit = alloy_primitives::Bytes::from(vec![op_alloy_consensus::DEPOSIT_TX_TYPE_ID]); - let attributes = TestAttributesBuilder::new() + let mut attributes = TestAttributesBuilder::new() .with_parent(parent) .with_transactions(vec![deposit, alloy_primitives::Bytes::from(vec![0x02])]) .build(); + let source = kona_protocol::BlockInfo { number: 20, ..Default::default() }; + attributes.derived_from = Some(source); + attributes.is_last_in_span = completed_span; // The deposits-only block the engine builds. It is genesis, so its block info needs no // L1-info deposit. @@ -326,6 +435,7 @@ mod recovery_tests { let state = TestEngineStateBuilder::new().with_unsafe_head(parent).build(); let (state_tx, state_rx) = tokio::sync::watch::channel(EngineState::default()); let mut engine = Engine::new(state, state_tx); + engine.safe_checkpoints.source = Some(source); engine.enqueue(EngineTask::Consolidate(Box::new(ConsolidateTask::new( client.clone(), config, @@ -338,6 +448,11 @@ mod recovery_tests { l1.assert_finished(); l2.assert_finished(); assert!(engine.is_empty()); + assert_eq!(engine.safe_checkpoints.entries.len(), usize::from(completed_span)); + if completed_span { + assert_eq!(engine.safe_checkpoints.entries[0].source, source); + assert_eq!(engine.safe_checkpoints.entries[0].safe.hash(), imported.header.hash_slow()); + } // The task updates `engine.state()` itself; only drain publishes it. assert_eq!( state_rx.borrow().sync_state.safe_head().block_info.hash, @@ -345,3 +460,272 @@ mod recovery_tests { ); } } + +#[cfg(test)] +mod reset_tests { + use super::*; + use crate::test_utils::{TestEngineStateBuilder, test_engine_client, test_l2_block}; + use alloy_primitives::B256; + use alloy_rpc_types_engine::{ForkchoiceUpdated, PayloadStatus, PayloadStatusEnum}; + use kona_protocol::BlockInfo; + + #[rstest::rstest] + #[case::canonical_origin(false)] + #[case::replaced_origin(true)] + #[tokio::test] + async fn reset_retains_recent_completed_safe_instead_of_walking_seventy_thousand_blocks( + #[case] reorged_unsafe: bool, + ) { + let config = Arc::new(RollupConfig { seq_window_size: 3600, ..Default::default() }); + let safe = test_l2_block(117029, 10635, 0, B256::ZERO); + let unsafe_block = test_l2_block( + 117030, + if reorged_unsafe { 10636 } else { 10635 }, + if reorged_unsafe { 0 } else { 1 }, + safe.header.hash, + ); + let finalized = test_l2_block(44302, 7084, 0, B256::ZERO); + let safe_info = + L2BlockInfo::from_block_and_genesis(&safe.clone().into_consensus(), &config.genesis) + .unwrap(); + let unsafe_info = L2BlockInfo::from_block_and_genesis( + &unsafe_block.clone().into_consensus(), + &config.genesis, + ) + .unwrap(); + let mut head: ::BlockResponse = + Default::default(); + head.header.inner.number = 10690; + head.header.hash = B256::repeat_byte(2); // A replacement head, not the old branch's hash. + let mut source: ::BlockResponse = + Default::default(); + source.header.inner.number = 10679; + source.header.hash = B256::repeat_byte(3); + let checkpoint = SafeCheckpoint { + safe: safe_info, + source: BlockInfo { + number: source.header.number, + hash: source.header.hash, + ..Default::default() + }, + replay_source: None, + }; + let (client, l1, l2) = test_engine_client(config.clone()); + for (tag, block) in [("finalized", &finalized), ("safe", &safe), ("latest", &unsafe_block)] + { + l2.expect_params("eth_getBlockByNumber", serde_json::json!([tag, true]), block); + } + let mut origin = head.clone(); + origin.header.hash = if reorged_unsafe { B256::ZERO } else { unsafe_info.l1_origin.hash }; + l1.expect_params( + "eth_getBlockByNumber", + serde_json::json!([format!("{:#x}", unsafe_info.l1_origin.number), false]), + origin, + ); + if reorged_unsafe { + l2.expect_params( + "eth_getBlockByHash", + serde_json::json!([safe_info.hash(), true]), + &safe, + ); + let mut origin = source.clone(); + origin.header.inner.number = safe_info.l1_origin.number; + origin.header.hash = safe_info.l1_origin.hash; + l1.expect_params( + "eth_getBlockByNumber", + serde_json::json!([format!("{:#x}", safe_info.l1_origin.number), false]), + origin, + ); + } + let kept_unsafe = if reorged_unsafe { safe_info } else { unsafe_info }; + l1.expect_params("eth_getBlockByNumber", serde_json::json!(["latest", false]), &head); + let mut origin = head.clone(); + origin.header.inner.number = kept_unsafe.l1_origin.number; + origin.header.hash = kept_unsafe.l1_origin.hash; + l1.expect_params( + "eth_getBlockByNumber", + serde_json::json!([format!("{:#x}", kept_unsafe.l1_origin.number), false]), + origin, + ); + l1.expect_params("eth_getBlockByNumber", serde_json::json!(["0x29b7", false]), source); + l2.expect_params( + "eth_getBlockByNumber", + serde_json::json!([format!("{:#x}", safe_info.block_info.number), false]), + &safe, + ); + + l2.expect_params( + "eth_getBlockByNumber", + serde_json::json!([format!("{:#x}", kept_unsafe.block_info.number), false]), + if reorged_unsafe { &safe } else { &unsafe_block }, + ); + if !reorged_unsafe { + l2.expect_params( + "eth_getBlockByHash", + serde_json::json!([safe_info.hash(), true]), + &safe, + ); + } + let forkchoice = alloy_rpc_types_engine::ForkchoiceState { + head_block_hash: kept_unsafe.hash(), + safe_block_hash: safe_info.hash(), + finalized_block_hash: finalized.header.hash, + }; + l2.expect_params( + "engine_forkchoiceUpdatedV3", + serde_json::json!([forkchoice, null]), + ForkchoiceUpdated::new(PayloadStatus::from_status(PayloadStatusEnum::Valid)), + ); + l1.expect_params("eth_getBlockByNumber", serde_json::json!(["0x29c2", false]), head); + l2.expect_params( + "engine_forkchoiceUpdatedV3", + serde_json::json!([forkchoice, null]), + ForkchoiceUpdated::new(PayloadStatus::from_status(PayloadStatusEnum::Valid)), + ); + let state = TestEngineStateBuilder::new() + .with_safe_head(safe_info) + .with_unsafe_head(unsafe_info) + .build(); + let (sender, receiver) = tokio::sync::watch::channel(state); + let mut engine = Engine::new(state, sender); + engine.safe_checkpoints.entries.push(checkpoint); + // No old L2 ancestors are supplied: the historical full-window walk fails this fixture. + assert_eq!(engine.reset(Arc::new(client), config).await.unwrap(), safe_info); + assert_eq!(engine.state().sync_state.safe_head(), safe_info); + assert_eq!(receiver.borrow().sync_state.safe_head(), safe_info); + assert_eq!(engine.state().sync_state.unsafe_head(), kept_unsafe); + l1.assert_finished(); + l2.assert_finished(); + } + + #[rstest::rstest] + #[case::walk_stops_at_finality(22, false)] + #[case::already_finalized(20, false)] + #[case::conflicting_finalized(20, true)] + #[case::below_finalized(18, false)] + #[tokio::test] + async fn unsafe_walk_never_passes_finality(#[case] number: u64, #[case] conflict: bool) { + let config = Arc::new(RollupConfig { seq_window_size: 3600, ..Default::default() }); + let (client, l1, l2) = test_engine_client(config.clone()); + let finalized = test_l2_block(20, 99, 0, B256::ZERO); + let parent = test_l2_block(21, 100, 0, finalized.header.hash); + let mut head = if number == 20 { + finalized.clone() + } else { + test_l2_block(number, 100, 1, parent.header.hash) + }; + if number == 22 { + head.header.inner.parent_hash = parent.header.hash; + } + if conflict { + head.header.inner.timestamp += 1; + } + head.header.hash = head.header.inner.hash_slow(); + for (tag, block) in [("finalized", &finalized), ("safe", &finalized), ("latest", &head)] { + l2.expect_params("eth_getBlockByNumber", serde_json::json!([tag, true]), block); + } + if number == 22 { + let mut replaced = + alloy_rpc_types_eth::Block::::default(); + replaced.header.hash = B256::ZERO; + for _ in 0..2 { + l1.expect_params( + "eth_getBlockByNumber", + serde_json::json!(["0x64", false]), + &replaced, + ); + } + for block in [&parent, &finalized] { + l2.expect_params( + "eth_getBlockByHash", + serde_json::json!([block.header.hash, true]), + block, + ); + } + } + if number >= 20 && !conflict { + let mut origin = + alloy_rpc_types_eth::Block::::default(); + origin.header.hash = B256::with_last_byte(99); + origin.header.inner.number = 99; + l1.expect_params("eth_getBlockByNumber", serde_json::json!(["0x63", false]), origin); + } + let result = crate::find_starting_forkchoice(&config, &client).await; + if number < 20 || conflict { + let err = result.unwrap_err(); + assert!(matches!(err, SyncStartError::MismatchedFinalizedBlock(..))); + assert!(!EngineResetError::SyncStart(err).is_temporary()); + } else { + let heads = result.unwrap(); + assert_eq!(heads.un_safe, heads.finalized); + assert_eq!(heads.safe, heads.finalized); + } + l1.assert_finished(); + l2.assert_finished(); + } +} + +#[cfg(test)] +mod checkpoint_history_tests { + use super::*; + use crate::test_utils::{TestAttributesBuilder, test_block_info}; + + #[rstest::rstest] + #[case::completed_parent(true)] + #[case::incomplete_span(false)] + fn replay_inherits_parent_inputs_even_when_child_source_is_earlier(#[case] completed: bool) { + let cfg = RollupConfig { seq_window_size: 10, ..Default::default() }; + let parent = test_block_info(20); + let source = BlockInfo { number: 1000, ..Default::default() }; + let mut history = SafeCheckpoints { source: Some(source), ..Default::default() }; + let mut attrs = TestAttributesBuilder::new().with_parent(parent).build(); + attrs.derived_from = Some(source); + attrs.is_last_in_span = completed; + let child = test_block_info(21); + history.record(&attrs, &cfg, child); + assert_eq!(history.entries.len(), usize::from(completed)); + if completed { + history.reset_to(child); + } + attrs.parent = child; + attrs.derived_from.as_mut().unwrap().number = 900; + attrs.is_last_in_span = true; + let grandchild = test_block_info(22); + history.record(&attrs, &cfg, grandchild); + assert_eq!(history.entries.len(), 1); + assert_eq!(history.entries[0].source, source); + assert_eq!(history.entries[0].replay_source, completed.then_some(source)); + assert_eq!(history.entries[0].safe, grandchild); + } + + #[rstest::rstest] + #[case::sequencing_window(false)] + #[case::alt_da_window(true)] + fn unknown_parent_waits_for_its_dependency_bound_and_resets_forget_unrelated_inputs( + #[case] alt_da: bool, + ) { + let mut cfg = RollupConfig { seq_window_size: 10, ..Default::default() }; + if alt_da { + cfg.alt_da_config = Some(kona_genesis::AltDAConfig { + da_challenge_window: Some(7), + da_resolve_window: Some(8), + ..Default::default() + }); + } + let parent = test_block_info(20); + let mut attrs = TestAttributesBuilder::new().with_parent(parent).build(); + let bound = parent.l1_origin.number + if alt_da { 15 } else { 10 } + 1; + attrs.derived_from = Some(BlockInfo { number: bound - 1, ..Default::default() }); + attrs.is_last_in_span = true; + let child = test_block_info(21); + let mut history = SafeCheckpoints::default(); + history.record(&attrs, &cfg, child); + assert!(history.entries.is_empty()); + attrs.derived_from.as_mut().unwrap().number = bound; + history.record(&attrs, &cfg, child); + assert_eq!(history.entries.len(), 1); + history.reset_to(parent); + assert!(history.entries.is_empty()); + assert!(history.source.is_none()); + } +} diff --git a/rust/kona/crates/node/engine/src/test_utils/misc.rs b/rust/kona/crates/node/engine/src/test_utils/misc.rs index fd77bf7ccf4..fb303a39b09 100644 --- a/rust/kona/crates/node/engine/src/test_utils/misc.rs +++ b/rust/kona/crates/node/engine/src/test_utils/misc.rs @@ -1,6 +1,6 @@ use alloy_eips::BlockNumHash; -use alloy_primitives::B256; -use kona_protocol::{BlockInfo, L2BlockInfo}; +use alloy_primitives::{B256, Sealed}; +use kona_protocol::{BlockInfo, L1BlockInfoBedrock, L2BlockInfo}; /// Helper to create a test `L2BlockInfo` at a specific block number pub fn test_block_info(number: u64) -> L2BlockInfo { @@ -15,3 +15,39 @@ pub fn test_block_info(number: u64) -> L2BlockInfo { seq_num: 0, } } + +/// Builds a full L2 block with a real L1-info deposit for reset traversal tests. +pub fn test_l2_block( + number: u64, + origin: u64, + sequence: u64, + parent: B256, +) -> alloy_rpc_types_eth::Block { + let data = L1BlockInfoBedrock::new( + origin, + 0, + 0, + B256::with_last_byte(origin as u8), + sequence, + Default::default(), + Default::default(), + Default::default(), + ) + .encode_calldata(); + let transaction = + op_alloy_consensus::OpTxEnvelope::Deposit(Sealed::new(op_alloy_consensus::TxDeposit { + input: data, + ..Default::default() + })); + let transaction = op_alloy_rpc_types::Transaction::from_transaction( + alloy_consensus::transaction::Recovered::new_unchecked(transaction, Default::default()), + Default::default(), + ); + let mut block: alloy_rpc_types_eth::Block = Default::default(); + block.header.inner.number = number; + block.header.inner.parent_hash = parent; + block.header.inner.timestamp = number * 2; + block.header.hash = block.header.inner.hash_slow(); + block.transactions = alloy_rpc_types_eth::BlockTransactions::Full(vec![transaction]); + block +} diff --git a/rust/kona/crates/node/engine/src/test_utils/mod.rs b/rust/kona/crates/node/engine/src/test_utils/mod.rs index a5575963a67..8487d07fc78 100644 --- a/rust/kona/crates/node/engine/src/test_utils/mod.rs +++ b/rust/kona/crates/node/engine/src/test_utils/mod.rs @@ -8,4 +8,4 @@ mod engine_state; pub use engine_state::TestEngineStateBuilder; mod misc; -pub use misc::test_block_info; +pub use misc::{test_block_info, test_l2_block}; diff --git a/rust/kona/crates/node/service/src/actors/derivation/actor.rs b/rust/kona/crates/node/service/src/actors/derivation/actor.rs index 73297ab8c4f..5aaca75058b 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/actor.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/actor.rs @@ -217,6 +217,21 @@ where self.derivation_state_machine .update(&DerivationStateUpdate::SignalProcessed)?; } + // The engine only publishes changed safe heads. Resume a reset that retains + // the already-confirmed head without waiting for another L1 block. + if let Signal::Reset(reset) = *signal && + reset.l2_safe_head == + self.derivation_state_machine.last_confirmed_safe_head() && + self.derivation_state_machine.current_state() == + DerivationState::AwaitingUpdateAfterSignal + { + self.derivation_state_machine.update( + &DerivationStateUpdate::NewAttributesConfirmed(Box::new( + reset.l2_safe_head, + )), + )?; + self.attempt_derivation().await?; + } } DerivationActorRequest::ProcessFinalizedL1Block(finalized_l1_block) => { // Attempt to finalize the block. If successful, notify engine. diff --git a/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs b/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs index 67d7d153581..0f14bf3a086 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs @@ -540,3 +540,21 @@ async fn a_block_is_queued_for_finalization_only_once_it_is_recorded() { mod database; mod safedb_tests; + +#[tokio::test] +async fn reset_retaining_confirmed_head_resumes_without_a_new_l1_head() { + let mut client = MockDerivationEngineClient::new(); + client.expect_send_safe_l2_signal().times(2).returning(|_| Ok(())); + let mut node = RecordingActor::new(client); + node.complete_el_sync_at(RESUME_HEAD).await; + node.signal(Signal::Reset(kona_derive::ResetSignal { l2_safe_head: l2_block(RESUME_HEAD) })) + .await; + assert_eq!( + node.actor.derivation_state_machine.current_state(), + DerivationState::AwaitingSafeHeadConfirmation + ); + assert_eq!( + node.actor.attributes_awaiting_confirmation.as_ref().unwrap().parent, + l2_block(RESUME_HEAD) + ); +} diff --git a/rust/kona/crates/node/service/src/actors/engine/tests.rs b/rust/kona/crates/node/service/src/actors/engine/tests.rs index eded3d472ca..1f4245cc15e 100644 --- a/rust/kona/crates/node/service/src/actors/engine/tests.rs +++ b/rust/kona/crates/node/service/src/actors/engine/tests.rs @@ -30,14 +30,47 @@ async fn admin_reset_recovers_and_completes_original_request( Default::default(); let mut config = RollupConfig::default(); config.genesis.l2.hash = block.header.inner.hash_slow(); + // Keep an unsafe block above genesis so both the L1 origin check and the safe walk run. + let mut unsafe_block = block.clone(); + unsafe_block.header.inner.number = 1; + unsafe_block.header.inner.parent_hash = config.genesis.l2.hash; + unsafe_block.header.hash = unsafe_block.header.inner.hash_slow(); + type RpcTransaction = + ::TransactionResponse; + let deposit = op_alloy_consensus::OpTxEnvelope::Deposit(alloy_primitives::Sealed::new( + op_alloy_consensus::TxDeposit { + input: kona_protocol::L1BlockInfoBedrock::new( + 0, + 0, + 0, + B256::ZERO, + 1, + Default::default(), + Default::default(), + Default::default(), + ) + .encode_calldata(), + ..Default::default() + }, + )); + unsafe_block.transactions = + alloy_rpc_types_eth::BlockTransactions::Full(vec![RpcTransaction::from_transaction( + alloy_consensus::transaction::Recovered::new_unchecked(deposit, Default::default()), + Default::default(), + )]); + let config = Arc::new(config); let (client, l1, l2) = test_engine_client(config.clone()); let client = Arc::new(client); if l1_failure { for tag in ["finalized", "safe", "latest"] { - l2.expect_params("eth_getBlockByNumber", serde_json::json!([tag, true]), &block); + l2.expect_params( + "eth_getBlockByNumber", + serde_json::json!([tag, true]), + if tag == "latest" { &unsafe_block } else { &block }, + ); } - l1.expect_error("eth_getBlockByHash"); + l1.expect_error("eth_getBlockByNumber"); } else { l2.expect_error("eth_getBlockByNumber"); } @@ -52,7 +85,7 @@ async fn admin_reset_recovers_and_completes_original_request( let (state_tx, _state_rx) = tokio::sync::watch::channel(EngineState::default()); let mut actor = EngineActor::new( client, - config, + config.clone(), derivation, Engine::new(EngineState::default(), state_tx), None, @@ -65,11 +98,16 @@ async fn admin_reset_recovers_and_completes_original_request( actor.step().await.unwrap(); // retain the reset actor.step().await.unwrap(); // failed attempt, then wait for backoff assert!(!reset.is_finished()); - for _ in 0..3 { - l2.expect_with_delay("eth_getBlockByNumber", &block, read_delay); + for response in [&block, &block, &unsafe_block] { + l2.expect_with_delay("eth_getBlockByNumber", response, read_delay); } - l1.expect( + l2.expect_params( "eth_getBlockByHash", + serde_json::json!([config.genesis.l2.hash, true]), + &block, + ); + l1.expect( + "eth_getBlockByNumber", alloy_rpc_types_eth::Block::::default(), ); l2.expect(