diff --git a/op-acceptance-tests/tests/sync/elsync/reorg/sync_test.go b/op-acceptance-tests/tests/sync/elsync/reorg/sync_test.go index 3dad7ddbfcc..09644136d2a 100644 --- a/op-acceptance-tests/tests/sync/elsync/reorg/sync_test.go +++ b/op-acceptance-tests/tests/sync/elsync/reorg/sync_test.go @@ -115,15 +115,6 @@ func TestUnsafeGapFillAfterSafeReorg(gt *testing.T) { // 4. Verifier then backfills and closes the unsafe gap once reconnected via CLP2P. func TestUnsafeGapFillAfterUnsafeReorg_RestartL2CL(gt *testing.T) { t := devtest.ParallelT(gt) - // Example error with kona-node: - // - // assertions.go:387: ERROR[03-30|22:17:07.231] - // assertions.go:387: Error Trace: /optimism/op-devstack/dsl/l2_el.go:204 - // assertions.go:387: /optimism/op-acceptance-tests/tests/sync/elsync/reorg/sync_test.go:211 - // assertions.go:387: Error: Received unexpected error: - // assertions.go:387: operation failed permanently after 30 attempts: expected head to reorg 0x893d77533b0ff9b37a92090679bf256d987b4535f06186ec71f29e68ddccd9a5:14, but got 0x893d77533b0ff9b37a92090679bf256d987b4535f06186ec71f29e68ddccd9a5:14 - // assertions.go:387: Test: TestUnsafeGapFillAfterUnsafeReorg_RestartL2CL - sysgo.SkipOnKonaNode(t, "not supported (timeout)") sys := newReorgSystem(t) require := t.Require() logger := t.Logger() @@ -259,15 +250,6 @@ func TestUnsafeGapFillAfterUnsafeReorg_RestartL2CL(gt *testing.T) { // 4. CLP2P is restored Verifier, the verifier backfills and the unsafe gap is closed. func TestUnsafeGapFillAfterUnsafeReorg_RestartCLP2P(gt *testing.T) { t := devtest.ParallelT(gt) - // Example error with kona-node: - // - // assertions.go:387: ERROR[03-31|11:15:39.398] - // assertions.go:387: Error Trace: /optimism/op-devstack/dsl/l2_el.go:204 - // assertions.go:387: /optimism/op-acceptance-tests/tests/sync/elsync/reorg/sync_test.go:356 - // assertions.go:387: Error: Received unexpected error: - // assertions.go:387: operation failed permanently after 30 attempts: expected head to reorg 0x166970054ad16ad090210e5d1045538eeccd2afd88ea991b010de026d0106870:18, but got 0x166970054ad16ad090210e5d1045538eeccd2afd88ea991b010de026d0106870:18 - // assertions.go:387: Test: TestUnsafeGapFillAfterUnsafeReorg_RestartCLP2P - sysgo.SkipOnKonaNode(t, "not supported (timeout)") sys := newReorgSystem(t) require := t.Require() logger := t.Logger() diff --git a/rust/Cargo.lock b/rust/Cargo.lock index adddd373f53..50ac5ac901f 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -7609,6 +7609,7 @@ dependencies = [ "dirs 6.0.0", "discv5 0.12.0", "enr 0.14.0", + "humantime", "kona-cli", "kona-derive", "kona-disc", diff --git a/rust/kona/bin/node/Cargo.toml b/rust/kona/bin/node/Cargo.toml index 4746aece14d..b63326a690d 100644 --- a/rust/kona/bin/node/Cargo.toml +++ b/rust/kona/bin/node/Cargo.toml @@ -54,6 +54,7 @@ op-alloy-network.workspace = true # general url.workspace = true dirs.workspace = true +humantime.workspace = true strum.workspace = true discv5.workspace = true enr.workspace = true diff --git a/rust/kona/bin/node/src/commands/node/command.rs b/rust/kona/bin/node/src/commands/node/command.rs index 8b5ba53775f..b32ab0a59d9 100644 --- a/rust/kona/bin/node/src/commands/node/command.rs +++ b/rust/kona/bin/node/src/commands/node/command.rs @@ -12,7 +12,7 @@ use anyhow::{Context as _, Result, bail}; use backon::{ExponentialBuilder, Retryable}; use clap::Parser; use kona_cli::{LogConfig, MetricsArgs}; -use kona_engine::EngineClient; +use kona_engine::{DEFAULT_EL_SYNC_SAFE_OFFSET, EngineClient}; use kona_genesis::{L1ChainConfig, RollupConfig}; use kona_interop::DependencySet; use kona_node_service::{EngineConfig, L1ConfigBuilder, NodeMode, RollupNodeBuilder}; @@ -25,6 +25,7 @@ use std::{ io::{Read, Write}, path::PathBuf, sync::Arc, + time::Duration, }; use strum::IntoEnumIterator; use tracing::{debug, error, info}; @@ -126,6 +127,16 @@ pub struct NodeCommand { /// Reorgs are supported, but may be slow to recover from. #[arg(long = "verifier.l1-confs", env = "KONA_NODE_VERIFIER_L1_CONFS", default_value_t = 0)] pub verifier_l1_confs: u64, + /// After execution-layer sync, or when the execution layer's forkchoice labels are behind its + /// chain, set the safe and finalized heads this far behind the synced tip (converted to L2 + /// blocks with ceiling division). Mirrors op-node's `--syncmode.offset-el-safe`. + #[arg( + long = "syncmode.offset-el-safe", + env = "KONA_NODE_SYNCMODE_OFFSET_EL_SAFE", + default_value = "12h", + value_parser = humantime::parse_duration, + )] + pub el_sync_safe_offset: Duration, /// P2P CLI arguments. #[command(flatten)] pub p2p_flags: P2PArgs, @@ -148,6 +159,7 @@ impl Default for NodeCommand { interop_dependency_set: None, safedb_path: None, verifier_l1_confs: 0, + el_sync_safe_offset: DEFAULT_EL_SYNC_SAFE_OFFSET, node_mode: NodeMode::Validator, p2p_flags: P2PArgs::default(), rpc_flags: RpcArgs::default(), @@ -349,6 +361,7 @@ impl NodeCommand { .with_dependency_set(dependency_set) .with_safe_db(safe_db) .with_verifier_l1_confs(self.verifier_l1_confs) + .with_el_sync_safe_offset(self.el_sync_safe_offset) .build() .start() .await diff --git a/rust/kona/bin/node/src/commands/node/tests.rs b/rust/kona/bin/node/src/commands/node/tests.rs index 0cfb440c62b..c2c295b3ff6 100644 --- a/rust/kona/bin/node/src/commands/node/tests.rs +++ b/rust/kona/bin/node/src/commands/node/tests.rs @@ -59,6 +59,33 @@ fn verifier_confirmation_depth_has_the_expected_environment_variable() { assert_eq!(arg.get_env().unwrap(), "KONA_NODE_VERIFIER_L1_CONFS"); } +/// Like op-node, the EL-sync safe offset defaults to 12h and takes durations such as `30m`. +#[rstest::rstest] +#[case::default(None, 12 * 60 * 60)] +#[case::minutes(Some("30m"), 30 * 60)] +#[case::disabled(Some("0s"), 0)] +fn el_sync_safe_offset_parses_durations(#[case] flag: Option<&str>, #[case] seconds: u64) { + let flag = flag.map(|value| ["--syncmode.offset-el-safe", value]); + let args = NodeCommand::parse_from( + std::iter::once("node") + .chain(default_flags().iter().copied()) + .chain(flag.into_iter().flatten()), + ); + assert_eq!(args.el_sync_safe_offset, std::time::Duration::from_secs(seconds)); +} + +#[test] +fn el_sync_safe_offset_default_matches_the_engine() { + assert_eq!( + NodeCommand::default().el_sync_safe_offset, + kona_engine::DEFAULT_EL_SYNC_SAFE_OFFSET + ); + use clap::CommandFactory; + let command = NodeCommand::command(); + let arg = command.get_arguments().find(|arg| arg.get_id() == "el_sync_safe_offset").unwrap(); + assert_eq!(arg.get_env().unwrap(), "KONA_NODE_SYNCMODE_OFFSET_EL_SAFE"); +} + #[test] fn test_node_cli_missing_l1_eth_rpc() { let err = NodeCommand::try_parse_from(["node"]).unwrap_err(); diff --git a/rust/kona/crates/node/engine/src/lib.rs b/rust/kona/crates/node/engine/src/lib.rs index c1f955096bb..f4a5cadeb22 100644 --- a/rust/kona/crates/node/engine/src/lib.rs +++ b/rust/kona/crates/node/engine/src/lib.rs @@ -74,7 +74,9 @@ mod metrics; pub use metrics::Metrics; mod sync; -pub use sync::{L2ForkchoiceState, SyncStartError, find_starting_forkchoice}; +pub use sync::{ + DEFAULT_EL_SYNC_SAFE_OFFSET, L2ForkchoiceState, SyncStartError, find_starting_forkchoice, +}; #[cfg(any(test, feature = "test-utils"))] /// Utilities that are useful when creating unit tests using structs within this library. diff --git a/rust/kona/crates/node/engine/src/sync/error.rs b/rust/kona/crates/node/engine/src/sync/error.rs index d98b69d642b..feec542dd40 100644 --- a/rust/kona/crates/node/engine/src/sync/error.rs +++ b/rust/kona/crates/node/engine/src/sync/error.rs @@ -9,6 +9,9 @@ use thiserror::Error; /// An error that can occur during the sync start process. #[derive(Error, Debug)] pub enum SyncStartError { + /// The canonical L1 view changed during recovery; retry with a fresh view. + #[error("Canonical chain changed during sync-start recovery")] + ChainViewChanged, /// An rpc error occurred #[error("An RPC error occurred: {0}")] RpcError(#[from] RpcError), @@ -29,6 +32,12 @@ pub enum SyncStartError { /// Finalized block mismatch #[error("Finalized block mismatch. Expected {0}, Got {1}")] MismatchedFinalizedBlock(B256, B256), + /// Recovery would rewind the unsafe origin by more than op-node's five lookback windows. + #[error("L1 reorg exceeds five sync lookback windows")] + ReorgTooDeep, + /// The finalized L2 block's L1 origin is no longer canonical, so L1 reorged below finality. + #[error("Finalized L2 block's L1 origin is no longer canonical")] + FinalizedL1OriginReplaced, /// L1 origin mismatch. #[error("L1 origin mismatch")] L1OriginMismatch, diff --git a/rust/kona/crates/node/engine/src/sync/mod.rs b/rust/kona/crates/node/engine/src/sync/mod.rs index cdf468ba554..614475806cc 100644 --- a/rust/kona/crates/node/engine/src/sync/mod.rs +++ b/rust/kona/crates/node/engine/src/sync/mod.rs @@ -1,7 +1,9 @@ //! Sync start algorithm for the OP Stack rollup node. +use alloy_eips::BlockId; +use alloy_primitives::B256; use kona_genesis::RollupConfig; -use kona_protocol::L2BlockInfo; +use kona_protocol::{BlockInfo, L2BlockInfo}; mod forkchoice; pub use forkchoice::L2ForkchoiceState; @@ -9,25 +11,42 @@ pub use forkchoice::L2ForkchoiceState; mod error; pub use error::SyncStartError; -use tracing::info; +use std::time::Duration; +use tracing::{info, warn}; use crate::EngineClient; -/// Searches for the latest [`L2ForkchoiceState`] that we can use to start the sync process with. +/// Matches op-node's threshold for recovering EL-sync forkchoice labels left at genesis. +const RECOVER_MIN_SEQ_WINDOWS: u64 = 14; + +/// Matches op-node's `MaxReorgSeqWindows`: how many sync lookback windows the unsafe head's L1 +/// origin may be rewound before recovery gives up. +const MAX_REORG_SEQ_WINDOWS: u64 = 5; + +/// How many L2 blocks the walk traverses between progress logs. +const PROGRESS_LOG_INTERVAL: u64 = 10_000; + +/// Matches op-node's default `--syncmode.offset-el-safe`: how far behind an EL-synced tip the +/// recovered safe and finalized heads are placed. The tip arrived over P2P and may not be +/// batched to L1 yet, so it must not be labelled safe or finalized. +pub const DEFAULT_EL_SYNC_SAFE_OFFSET: Duration = Duration::from_secs(12 * 60 * 60); + +/// Searches for a sync starting point using op-node's `FindL2Heads` rules, and returns: /// -/// - The *unsafe L2 block*: This is the highest L2 block whose L1 origin is a *plausible* -/// extension of the canonical L1 chain (as known to the rollup node). -/// - The *safe L2 block*: This is the highest L2 block whose epoch's sequencing window is -/// complete within the canonical L1 chain (as known to the rollup node). -/// - The *finalized L2 block*: This is the L2 block which is known to be fully derived from -/// finalized L1 block data. +/// - The *unsafe* head: the highest L2 block whose L1 origin, and every ancestor's, is a plausible +/// extension of the canonical L1 chain. An origin not yet visible on L1 is plausible. +/// - The *safe* head: the highest L2 block whose epoch's sequencing window is complete within the +/// canonical L1 chain. Recovery stops at an epoch boundary below both the previous safe head and +/// a full lookback window, or at finality. +/// - The *finalized* head: unchanged, as it is fully derived from finalized L1 data. /// -/// Plausible: meaning that the blockhash of the L2 block's L1 origin -/// (as reported in the L1 Attributes deposit within the L2 block) is not canonical at another -/// height in the L1 chain, and the same holds for all its ancestors. +/// When the execution layer lost its safe and finalized labels after EL sync, or was rewound +/// below them, the unsafe head stays at the tip and safe and finalized are placed +/// `el_sync_safe_offset` behind it, as op-node's `L2HeadsForELSyncWithOffset` does. pub async fn find_starting_forkchoice( cfg: &RollupConfig, engine_client: &EngineClient, + el_sync_safe_offset: Duration, ) -> Result { let mut current_fc = L2ForkchoiceState::current(cfg, engine_client).await?; info!( @@ -38,79 +57,240 @@ pub async fn find_starting_forkchoice( "Loaded current L2 EL forkchoice state" ); - // Search for the highest `unsafe` block, relative to the initial `unsafe` block's L1 origin, + let missing_finality = current_fc.finalized.hash() == cfg.genesis.l2.hash && + current_fc.safe.hash() == cfg.genesis.l2.hash && + current_fc.un_safe.block_info.number > cfg.genesis.l2.number && + current_fc.un_safe.l1_origin.number > + cfg.genesis + .l1 + .number + .saturating_add(RECOVER_MIN_SEQ_WINDOWS.saturating_mul(cfg.seq_window_size)); + let rewound = current_fc.un_safe.block_info.number < current_fc.safe.block_info.number || + current_fc.un_safe.block_info.number < current_fc.finalized.block_info.number; + if missing_finality || rewound { + // Handle this before the normal walk: stale labels are not a reorg finality boundary. + warn!(target: "sync_start", ?current_fc, missing_finality, rewound, ?el_sync_safe_offset, "Recovering missing or stale execution-layer forkchoice labels"); + return heads_for_el_sync(cfg, engine_client, el_sync_safe_offset, current_fc.un_safe) + .await; + } + + let previous_unsafe = current_fc.un_safe; + let previous_safe = current_fc.safe; + let mut cursor = previous_unsafe; + let mut unsafe_candidate = Some(cursor); + let mut highest_canonical: Option = None; + let mut walker = SyncWalker::new(cfg, engine_client); + let lookback = cfg.seq_window_size; + let mut walked = 0u64; + + // Port of op-node's normal FindL2Heads walk. Keep plausible heads ahead of L1, discard + // replaced origins, and validate every traversed child/parent epoch and sequence relation. loop { - let l1_origin = - engine_client.get_l1_block(current_fc.un_safe.l1_origin.hash.into()).await?; - info!( - target: "sync_start", - l1_origin = %current_fc.un_safe.l1_origin.number, - l2_unsafe = %current_fc.un_safe.block_info.number, - "Searching for L2 unsafe block with canonical L1 origin" - ); - - match l1_origin { - Some(_) => { - // Unsafe block has existing L1 origin. Continue with this head. - info!( - target: "sync_start", - l2_unsafe = %current_fc.un_safe.block_info.number, - "Found L2 unsafe block with canonical L1 origin" - ); - break; + if cursor.block_info.number < current_fc.finalized.block_info.number || + (cursor.block_info.number == current_fc.finalized.block_info.number && + cursor.hash() != current_fc.finalized.hash()) + { + return Err(SyncStartError::MismatchedFinalizedBlock( + current_fc.finalized.hash(), + cursor.hash(), + )); + } + let origin = walker.origin(cursor).await?; + if cursor.block_info.number == cfg.genesis.l2.number { + if cursor.hash() != cfg.genesis.l2.hash { + return Err(SyncStartError::InvalidL2GenesisHash( + cfg.genesis.l2.hash, + cursor.hash(), + )); + } + if let Some(l1) = walker.l1 && + l1.hash != cfg.genesis.l1.hash + { + return Err(SyncStartError::InvalidL1GenesisHash(cfg.genesis.l1.hash, l1.hash)); } + } + let unsafe_head = match unsafe_candidate { + Some(head) => head, None => { - let l2_parent_hash = current_fc.un_safe.block_info.parent_hash.into(); - let l2_parent = engine_client - .get_l2_block(l2_parent_hash) - .full() - .await? - .ok_or(SyncStartError::BlockNotFound(l2_parent_hash))?; - - current_fc.un_safe = - L2BlockInfo::from_block_and_genesis(&l2_parent.into_consensus(), &cfg.genesis)?; + if cursor + .l1_origin + .number + .saturating_add(MAX_REORG_SEQ_WINDOWS.saturating_mul(lookback)) < + previous_unsafe.l1_origin.number + { + return Err(SyncStartError::ReorgTooDeep); + } + unsafe_candidate = Some(cursor); + cursor + } + }; + match origin { + OriginStatus::Ahead => highest_canonical = None, + OriginStatus::Canonical => { + highest_canonical.get_or_insert(cursor); + } + OriginStatus::Replaced => { + unsafe_candidate = None; + highest_canonical = None; } } - } - - // Search for the highest `safe` block that's L1 origin is at least older than the sequencing - // window, relative to the L1 origin of the `unsafe` block. - let mut safe_cursor = current_fc.un_safe; - loop { - info!( - target: "sync_start", - l1_origin = %safe_cursor.l1_origin.number, - l2_safe = %safe_cursor.block_info.number, - "Searching for L2 safe block beyond sequencing window" - ); - - let is_behind_sequence_window = - current_fc.un_safe.l1_origin.number.saturating_sub(cfg.seq_window_size) > - safe_cursor.l1_origin.number; - let is_finalized = safe_cursor.block_info.hash == current_fc.finalized.block_info.hash; - let is_genesis = safe_cursor.block_info.hash == cfg.genesis.l2.hash; - if is_behind_sequence_window || is_finalized || is_genesis { + if cursor.block_info.number == current_fc.finalized.block_info.number { + // op-node returns an empty unsafe head here. Refuse instead: L1 reorged below the + // finalized block's origin, which finality rules out. + current_fc.un_safe = + unsafe_candidate.ok_or(SyncStartError::FinalizedL1OriginReplaced)?; + current_fc.safe = cursor; + break; + } + // Use the parent of an epoch boundary below both the previous safe head and a full + // canonical lookback window. Finality above is the lower bound. + let ready = cursor.block_info.number <= previous_safe.block_info.number && + highest_canonical.is_some_and(|high| { + cursor.l1_origin.number.saturating_add(lookback) < high.l1_origin.number + }) && + cursor.seq_num == 0; + cursor = walker.parent(cursor).await?; + if ready { + // A ready block's origin is canonical, so this iteration kept its unsafe head. + current_fc.un_safe = unsafe_head; + current_fc.safe = cursor; + break; + } + walked += 1; + if walked.is_multiple_of(PROGRESS_LOG_INTERVAL) { info!( target: "sync_start", - l2_safe = %safe_cursor.block_info.number, - is_behind_sequence_window, - is_finalized, - is_genesis, - "Found suitable L2 safe block" + walked, + cursor = cursor.block_info.number, + finalized = current_fc.finalized.block_info.number, + "Walking back to find the sync start" ); - current_fc.safe = safe_cursor; - break; } + } + info!(target: "sync_start", walked, %current_fc, "Found sync start"); + + // Leave the finalized block as-is, and return the current forkchoice. + Ok(current_fc) +} + +/// Port of op-node's `L2HeadsForELSyncWithOffset`: the unsafe head stays at the EL-synced `tip`, +/// and safe and finalized retract by `offset`, converted to blocks with ceiling division and +/// clamped at genesis. +async fn heads_for_el_sync( + cfg: &RollupConfig, + engine_client: &EngineClient, + offset: Duration, + tip: L2BlockInfo, +) -> Result { + let target = + offset_block_number(offset, cfg.block_time, tip.block_info.number, cfg.genesis.l2.number); + let derived = if target < tip.block_info.number { + let id = BlockId::number(target); let block = engine_client - .get_l2_block(safe_cursor.block_info.parent_hash.into()) + .get_l2_block(id) .full() .await? - .ok_or(SyncStartError::BlockNotFound(safe_cursor.block_info.parent_hash.into()))?; - safe_cursor = L2BlockInfo::from_block_and_genesis(&block.into_consensus(), &cfg.genesis)?; + .ok_or(SyncStartError::BlockNotFound(id))?; + L2BlockInfo::from_block_and_genesis(&block.into_consensus(), &cfg.genesis)? + } else { + tip + }; + Ok(L2ForkchoiceState { un_safe: tip, safe: derived, finalized: derived }) +} + +/// Port of op-node's `OffsetBlockNum`: the block number `ceil(offset / block_time)` blocks behind +/// `head`, never below `genesis`. +pub(crate) fn offset_block_number( + offset: Duration, + block_time: u64, + head: u64, + genesis: u64, +) -> u64 { + if block_time == 0 || head <= genesis { + return head; } + head - offset.as_secs().div_ceil(block_time).min(head - genesis) +} - // Leave the finalized block as-is, and return the current forkchoice. - Ok(current_fc) +#[derive(Debug, PartialEq, Eq)] +enum OriginStatus { + Canonical, + Replaced, + Ahead, +} + +/// The same L1-origin cursor and child/parent checks used by op-node `FindL2Heads`. +struct SyncWalker<'a> { + cfg: &'a RollupConfig, + client: &'a EngineClient, + l1: Option, +} + +impl<'a> SyncWalker<'a> { + const fn new(cfg: &'a RollupConfig, client: &'a EngineClient) -> Self { + Self { cfg, client, l1: None } + } + + async fn origin(&mut self, head: L2BlockInfo) -> Result { + if self.l1.is_none_or(|l1| l1.hash != head.l1_origin.hash) { + // A by-hash read is canonical only when it follows the parent of a known L1 block. + let id = if self.l1.is_some_and(|l1| l1.parent_hash == head.l1_origin.hash) { + head.l1_origin.hash.into() + } else { + BlockId::number(head.l1_origin.number) + }; + let block = self.client.get_l1_block(id).await?; + if block.is_none() && matches!(id, BlockId::Hash(_)) { + return Err(SyncStartError::ChainViewChanged); + } + self.l1 = block.map(|block| BlockInfo { + hash: block.header.hash, + number: block.header.number, + parent_hash: block.header.parent_hash, + timestamp: block.header.timestamp, + }); + } + Ok(match self.l1 { + None => OriginStatus::Ahead, + Some(l1) if l1.hash == head.l1_origin.hash => OriginStatus::Canonical, + Some(_) => OriginStatus::Replaced, + }) + } + + async fn parent(&self, child: L2BlockInfo) -> Result { + let id = child.block_info.parent_hash.into(); + let block = + self.client.get_l2_block(id).full().await?.ok_or(SyncStartError::BlockNotFound(id))?; + let parent = + L2BlockInfo::from_block_and_genesis(&block.into_consensus(), &self.cfg.genesis)?; + let canonical_parent = + self.l1.filter(|l1| l1.hash == child.l1_origin.hash).map(|l1| l1.parent_hash); + check_epoch_and_sequence(child, parent, canonical_parent)?; + Ok(parent) + } +} + +// Ported from op-node/rollup/sync/start.go's L1-origin relation check. Compare full origin IDs: +// a same-height hash replacement is an epoch inconsistency, not a same-origin sequence step. +fn check_epoch_and_sequence( + child: L2BlockInfo, + parent: L2BlockInfo, + canonical_parent: Option, +) -> Result<(), SyncStartError> { + if parent.l1_origin != child.l1_origin { + if parent.l1_origin.number.checked_add(1) != Some(child.l1_origin.number) { + return Err(SyncStartError::L1OriginMismatch); + } + if child.seq_num != 0 { + return Err(SyncStartError::NonZeroSequenceNumber); + } + if canonical_parent.is_some_and(|hash| hash != parent.l1_origin.hash) { + return Err(SyncStartError::L1OriginMismatch); + } + } else if parent.seq_num.checked_add(1) != Some(child.seq_num) { + return Err(SyncStartError::InconsistentSequenceNumber); + } + Ok(()) } #[cfg(test)] @@ -146,3 +326,6 @@ mod test { assert_eq!(rpc_reported_hash, l2_block_info.block_info.hash); } } + +#[cfg(test)] +mod tests; diff --git a/rust/kona/crates/node/engine/src/sync/tests.rs b/rust/kona/crates/node/engine/src/sync/tests.rs new file mode 100644 index 00000000000..44357e49021 --- /dev/null +++ b/rust/kona/crates/node/engine/src/sync/tests.rs @@ -0,0 +1,551 @@ +//! Regression vectors from op-node/rollup/sync/start_test.go, plus malformed traversal edges. + +use super::*; +use crate::test_utils::{RpcMock, test_engine_client, test_l1_hash, test_l2_block}; +use alloy_rpc_types_eth::Block; +use serde_json::json; +use std::sync::Arc; + +type L1Block = Block; +type L2Block = Block; + +fn l1_block(number: u64) -> L1Block { + let mut block = L1Block::default(); + block.header.inner.number = number; + block.header.hash = test_l1_hash(number); + block.header.inner.parent_hash = test_l1_hash(number.saturating_sub(1)); + block +} + +fn info(block: &L2Block, cfg: &RollupConfig) -> L2BlockInfo { + L2BlockInfo::from_block_and_genesis(&block.clone().into_consensus(), &cfg.genesis).unwrap() +} + +fn expect_heads(mock: &RpcMock, finalized: &L2Block, safe: &L2Block, head: &L2Block) { + for (tag, block) in [("finalized", finalized), ("safe", safe), ("latest", head)] { + mock.expect_params("eth_getBlockByNumber", json!([tag, true]), block); + } +} + +fn expect_parent(mock: &RpcMock, parent: &L2Block) { + mock.expect_params("eth_getBlockByHash", json!([parent.header.hash, true]), parent); +} + +/// Two-second blocks, so op-node's default 12h offset is 21,600 blocks. +const BLOCK_TIME: u64 = 2; +const DEFAULT_OFFSET_BLOCKS: u64 = 21_600; + +// Old chains with missing labels must recover in constant time, with safe and finalized placed +// the EL-sync offset behind the tip; recent chains still derive from genesis. Non-zero genesis +// heights exercise the threshold relative to chain genesis. +#[rstest::rstest] +#[case::genesis(0, false)] +#[case::recent_chain(1, false)] +#[case::at_recovery_threshold(28, false)] +#[case::past_recovery_threshold(29, false)] +#[case::distant_unsafe_head(100_000_000, false)] +#[case::missing_labels(100_000_000, true)] +#[tokio::test] +async fn recovers_el_sync_with_genesis_labels(#[case] distance: u64, #[case] missing_labels: bool) { + let genesis = test_l2_block(100, 50, 0, B256::ZERO); + let mut cfg = RollupConfig { seq_window_size: 2, block_time: BLOCK_TIME, ..Default::default() }; + cfg.genesis.l1 = alloy_eips::BlockNumHash::new(50, l1_block(50).header.hash); + cfg.genesis.l2 = alloy_eips::BlockNumHash::new(100, genesis.header.hash); + let mut blocks = vec![genesis.clone()]; + if distance <= 28 { + for i in 1..=distance { + blocks.push(test_l2_block(100 + i, 50 + i, 0, blocks.last().unwrap().header.hash)); + } + } else { + blocks.push(test_l2_block(100 + distance, 50 + distance, 0, B256::ZERO)); + } + let head = blocks.last().unwrap(); + let (client, l1, l2) = test_engine_client(Arc::new(cfg.clone())); + if missing_labels { + l2.expect_params( + "eth_getBlockByNumber", + json!(["finalized", true]), + Option::::None, + ); + l2.expect_params("eth_getBlockByNumber", json!(["0x64", true]), &genesis); + l2.expect_params("eth_getBlockByNumber", json!(["safe", true]), Option::::None); + l2.expect_params("eth_getBlockByNumber", json!(["latest", true]), head); + } else { + expect_heads(&l2, &genesis, &genesis, head); + } + let derived = if distance <= 28 { + l1.expect_params( + "eth_getBlockByNumber", + json!([format!("{:#x}", 50 + distance), false]), + l1_block(50 + distance), + ); + for i in (0..distance).rev() { + expect_parent(&l2, &blocks[i as usize]); + l1.expect_params( + "eth_getBlockByHash", + json!([l1_block(50 + i).header.hash, false]), + l1_block(50 + i), + ); + } + genesis.clone() + } else { + // A chain younger than the offset clamps at genesis. + let number = (100 + distance).saturating_sub(DEFAULT_OFFSET_BLOCKS).max(100); + let offset_head = if number == 100 { + genesis.clone() + } else { + test_l2_block(number, number - 50, 0, B256::ZERO) + }; + l2.expect_params( + "eth_getBlockByNumber", + json!([format!("{number:#x}"), true]), + &offset_head, + ); + offset_head + }; + let result = + find_starting_forkchoice(&cfg, &client, DEFAULT_EL_SYNC_SAFE_OFFSET).await.unwrap(); + assert_eq!(result.un_safe, info(head, &cfg)); + assert_eq!(result.safe, info(&derived, &cfg)); + assert_eq!(result.finalized, info(&derived, &cfg)); + l1.assert_finished(); + l2.assert_finished(); +} + +// Like op-node, an execution layer rewound below its safe or finalized label keeps its unsafe +// tip, and safe and finalized are placed the EL-sync offset behind that tip. That can move +// finalized up when it was below the tip, as in `behind_safe`. +#[rstest::rstest] +#[case::behind_safe(0, 20, 10, Some(13))] +#[case::behind_finalized(20, 17, 10, Some(13))] +#[case::behind_finalized_and_safe(20, 21, 10, Some(13))] +#[case::offset_clamped_at_genesis(20, 21, 3_600, Some(0))] +#[case::zero_offset(20, 21, 0, None)] +#[tokio::test] +async fn recovers_rewound_el_labels( + #[case] finalized_number: u64, + #[case] safe_number: u64, + #[case] offset_seconds: u64, + #[case] expected_number: Option, +) { + let genesis = test_l2_block(0, 0, 0, B256::ZERO); + let mut cfg = RollupConfig { block_time: BLOCK_TIME, ..Default::default() }; + cfg.genesis.l2.hash = genesis.header.hash; + let finalized = if finalized_number == 0 { + genesis.clone() + } else { + test_l2_block(finalized_number, 100, 0, B256::ZERO) + }; + let safe = test_l2_block(safe_number, 101, 0, B256::ZERO); + let head = test_l2_block(18, 99, 0, B256::ZERO); + let (client, l1, l2) = test_engine_client(Arc::new(cfg.clone())); + expect_heads(&l2, &finalized, &safe, &head); + let expected = match expected_number { + Some(0) => genesis.clone(), + Some(number) => test_l2_block(number, 98, 0, B256::ZERO), + None => head.clone(), + }; + if let Some(number) = expected_number { + l2.expect_params("eth_getBlockByNumber", json!([format!("{number:#x}"), true]), &expected); + } + let result = + find_starting_forkchoice(&cfg, &client, Duration::from_secs(offset_seconds)).await.unwrap(); + let expected = info(&expected, &cfg); + assert_eq!( + result, + L2ForkchoiceState { un_safe: info(&head, &cfg), safe: expected, finalized: expected } + ); + l1.assert_finished(); + l2.assert_finished(); +} + +// Mirrors op-node's OffsetBlockNum: ceiling division by the block time, clamped at genesis. +#[rstest::rstest] +#[case::zero_offset(0, 2, 100, 0, 100)] +#[case::rounds_up(3_000, 2, 100, 0, 98)] +#[case::whole_blocks(4_000, 2, 100, 0, 98)] +#[case::truncates_sub_second(1_500, 2, 100, 0, 99)] +#[case::clamped_at_genesis(1_000_000, 2, 105, 100, 100)] +#[case::head_at_genesis(10_000, 2, 100, 100, 100)] +#[case::zero_block_time(10_000, 0, 100, 0, 100)] +fn offset_block_number_matches_op_node( + #[case] offset_millis: u64, + #[case] block_time: u64, + #[case] head: u64, + #[case] genesis: u64, + #[case] expected: u64, +) { + let offset = Duration::from_millis(offset_millis); + assert_eq!(offset_block_number(offset, block_time, head, genesis), expected); +} + +#[rstest::rstest] +#[case::only_safe_established(0)] +#[case::finality_established(1)] +#[tokio::test] +async fn established_labels_keep_the_normal_recovery_walk(#[case] finalized: usize) { + let mut blocks = vec![]; + for number in 0..30 { + let parent = blocks.last().map_or(B256::ZERO, |block: &L2Block| block.header.hash); + blocks.push(test_l2_block(number, number, 0, parent)); + } + let mut cfg = RollupConfig { seq_window_size: 2, ..Default::default() }; + cfg.genesis.l2.hash = blocks[0].header.hash; + let (client, l1, l2) = test_engine_client(Arc::new(cfg.clone())); + expect_heads(&l2, &blocks[finalized], &blocks[1], &blocks[29]); + l1.expect_params("eth_getBlockByNumber", json!(["0x1d", false]), l1_block(29)); + for number in (1..29).rev() { + l1.expect_params( + "eth_getBlockByHash", + json!([l1_block(number).header.hash, false]), + l1_block(number), + ); + } + for parent in (finalized..29).rev() { + expect_parent(&l2, &blocks[parent]); + } + let result = + find_starting_forkchoice(&cfg, &client, DEFAULT_EL_SYNC_SAFE_OFFSET).await.unwrap(); + assert_eq!(result.un_safe, info(&blocks[29], &cfg)); + assert_eq!(result.safe, info(&blocks[finalized], &cfg)); + assert_eq!(result.finalized, info(&blocks[finalized], &cfg)); + l1.assert_finished(); + l2.assert_finished(); +} + +// The walk stops at the finalized head and never passes it, even when every newer origin was +// replaced, and a conflicting finalized block is fatal rather than retried. +#[rstest::rstest] +#[case::walk_stops_at_finality(22, false)] +#[case::already_finalized(20, false)] +#[case::conflicting_finalized(20, true)] +#[tokio::test] +async fn unsafe_walk_never_passes_finality(#[case] number: u64, #[case] conflict: bool) { + let config = Arc::new(RollupConfig { seq_window_size: 3600, ..Default::default() }); + let (client, l1, l2) = test_engine_client(config.clone()); + let finalized = test_l2_block(20, 99, 0, B256::ZERO); + let parent = test_l2_block(21, 100, 0, finalized.header.hash); + let mut head = if number == 20 { + finalized.clone() + } else { + test_l2_block(number, 100, 1, parent.header.hash) + }; + if number == 22 { + head.header.inner.parent_hash = parent.header.hash; + } + if conflict { + head.header.inner.timestamp += 1; + } + head.header.hash = head.header.inner.hash_slow(); + for (tag, block) in [("finalized", &finalized), ("safe", &finalized), ("latest", &head)] { + l2.expect_params("eth_getBlockByNumber", serde_json::json!([tag, true]), block); + } + if number == 22 { + let mut replaced = + alloy_rpc_types_eth::Block::::default(); + replaced.header.hash = B256::ZERO; + for _ in 0..2 { + l1.expect_params("eth_getBlockByNumber", serde_json::json!(["0x64", false]), &replaced); + } + for block in [&parent, &finalized] { + l2.expect_params( + "eth_getBlockByHash", + serde_json::json!([block.header.hash, true]), + block, + ); + } + } + if number >= 20 && !conflict { + let mut origin = alloy_rpc_types_eth::Block::::default(); + origin.header.hash = test_l1_hash(99); + origin.header.inner.number = 99; + l1.expect_params("eth_getBlockByNumber", serde_json::json!(["0x63", false]), origin); + } + let result = find_starting_forkchoice(&config, &client, DEFAULT_EL_SYNC_SAFE_OFFSET).await; + if conflict { + let err = result.unwrap_err(); + assert!(matches!(err, SyncStartError::MismatchedFinalizedBlock(..))); + assert!(!crate::EngineResetError::SyncStart(err).is_temporary()); + } else { + let heads = result.unwrap(); + assert_eq!(heads.un_safe, heads.finalized); + assert_eq!(heads.safe, heads.finalized); + } + l1.assert_finished(); + l2.assert_finished(); +} + +// An L1 reorg below the finalized block's origin cannot be recovered from: it is reported as +// such, not as a generic origin mismatch, and is not retried. +#[tokio::test] +async fn replaced_finalized_origin_is_fatal() { + let cfg = RollupConfig { seq_window_size: 3600, ..Default::default() }; + let finalized = test_l2_block(1, 99, 0, B256::ZERO); + let head = test_l2_block(2, 100, 0, finalized.header.hash); + let (client, l1, l2) = test_engine_client(Arc::new(cfg.clone())); + expect_heads(&l2, &finalized, &finalized, &head); + // Replaced blocks, so the walk reads each origin by number. + for number in [100u64, 99] { + let mut replaced = l1_block(number); + replaced.header.hash = B256::repeat_byte(0xff); + replaced.header.inner.parent_hash = B256::repeat_byte(0xee); + l1.expect_params("eth_getBlockByNumber", json!([format!("{number:#x}"), false]), replaced); + } + expect_parent(&l2, &finalized); + let err = + find_starting_forkchoice(&cfg, &client, DEFAULT_EL_SYNC_SAFE_OFFSET).await.unwrap_err(); + assert!(matches!(err, SyncStartError::FinalizedL1OriginReplaced), "{err:?}"); + assert!(!crate::EngineResetError::SyncStart(err).is_temporary()); + l1.assert_finished(); + l2.assert_finished(); +} + +#[tokio::test] +async fn missing_l2_parent_can_be_retried() { + let cfg = RollupConfig::default(); + let finalized = test_l2_block(1, 99, 0, B256::ZERO); + let head = test_l2_block(2, 100, 0, finalized.header.hash); + let (client, l1, l2) = test_engine_client(Arc::new(cfg.clone())); + expect_heads(&l2, &finalized, &finalized, &head); + l1.expect_params("eth_getBlockByNumber", json!(["0x64", false]), l1_block(100)); + l2.expect_params( + "eth_getBlockByHash", + json!([finalized.header.hash, true]), + Option::::None, + ); + let err = + find_starting_forkchoice(&cfg, &client, DEFAULT_EL_SYNC_SAFE_OFFSET).await.unwrap_err(); + assert!(matches!(err, SyncStartError::BlockNotFound(_))); + assert!(crate::EngineResetError::SyncStart(err).is_temporary()); + + expect_heads(&l2, &finalized, &finalized, &head); + l1.expect_params("eth_getBlockByNumber", json!(["0x64", false]), l1_block(100)); + expect_parent(&l2, &finalized); + l1.expect_params("eth_getBlockByHash", json!([l1_block(99).header.hash, false]), l1_block(99)); + let result = + find_starting_forkchoice(&cfg, &client, DEFAULT_EL_SYNC_SAFE_OFFSET).await.unwrap(); + assert_eq!(result.un_safe, info(&head, &cfg)); + assert_eq!(result.safe, info(&finalized, &cfg)); + assert_eq!(result.finalized, info(&finalized, &cfg)); + l1.assert_finished(); + l2.assert_finished(); +} + +// Same histories and expected labels as op-node's small-reorg, L1-ahead and L2-ahead vectors. +// The final case keeps the previous safe head below the lookback boundary. +#[rstest::rstest] +#[case::small_reorg(8, 8, Some(7), 1, 7, (6, 2), &[(7,false),(6,true),(5,true),(4,true),(3,true)])] +#[case::l1_ahead(5, 6, None, 0, 3, (4, 0), &[(4,false),(3,true),(2,true),(1,true)])] +#[case::l2_ahead_after_reorg(6, 4, None, 1, 3, (5, 1), &[(5,false),(4,false),(3,false),(2,true),(1,true)])] +#[case::prior_safe_limits_recovery(12, 12, None, 1, 3, (11, 2), &[(11,false),(10,true),(9,true),(8,true),(7,true),(6,true),(5,true),(4,true),(3,true)])] +#[tokio::test] +async fn matches_op_node_sync_start_vectors( + #[case] l2_len: usize, + #[case] l1_len: usize, + #[case] replaced: Option, + #[case] finalized: usize, + #[case] safe: usize, + #[case] expected: (usize, usize), + #[case] reads: &[(usize, bool)], +) { + let (expected_unsafe, expected_safe) = expected; + let mut blocks = vec![]; + for i in 0..l2_len { + let parent = blocks.last().map_or(B256::ZERO, |b: &L2Block| b.header.hash); + blocks.push(test_l2_block(i as u64, i as u64, 0, parent)); + } + let mut cfg = RollupConfig { seq_window_size: 2, ..Default::default() }; + cfg.genesis.l2.hash = blocks[0].header.hash; + let (client, l1, l2) = test_engine_client(Arc::new(cfg.clone())); + expect_heads(&l2, &blocks[finalized], &blocks[safe], blocks.last().unwrap()); + for &(number, by_hash) in reads { + let mut block = (number < l1_len).then(|| l1_block(number as u64)); + if replaced == Some(number) { + block.as_mut().unwrap().header.hash = B256::repeat_byte(0xff); + } + let (method, id) = if by_hash { + ("eth_getBlockByHash", json!(test_l1_hash(number as u64))) + } else { + ("eth_getBlockByNumber", json!(format!("{number:#x}"))) + }; + l1.expect_params(method, json!([id, false]), block); + } + for parent in (expected_safe..l2_len - 1).rev() { + expect_parent(&l2, &blocks[parent]); + } + let result = + find_starting_forkchoice(&cfg, &client, DEFAULT_EL_SYNC_SAFE_OFFSET).await.unwrap(); + assert_eq!(result.un_safe, info(&blocks[expected_unsafe], &cfg)); + assert_eq!(result.safe, info(&blocks[expected_safe], &cfg)); + assert_eq!(result.finalized, info(&blocks[finalized], &cfg)); + l1.assert_finished(); + l2.assert_finished(); +} + +#[tokio::test] +async fn safe_head_is_last_block_before_epoch_boundary() { + let mut blocks = vec![]; + for i in 0..8 { + let parent = blocks.last().map_or(B256::ZERO, |b: &L2Block| b.header.hash); + blocks.push(test_l2_block(i, i / 2, i % 2, parent)); + } + let mut cfg = RollupConfig { seq_window_size: 1, ..Default::default() }; + cfg.genesis.l2.hash = blocks[0].header.hash; + let (client, l1, l2) = test_engine_client(Arc::new(cfg.clone())); + expect_heads(&l2, &blocks[0], &blocks[7], &blocks[7]); + l1.expect_params("eth_getBlockByNumber", json!(["0x3", false]), l1_block(3)); + for number in (1..=2).rev() { + l1.expect_params( + "eth_getBlockByHash", + json!([test_l1_hash(number), false]), + l1_block(number), + ); + } + for parent in (1..7).rev() { + expect_parent(&l2, &blocks[parent]); + } + let result = + find_starting_forkchoice(&cfg, &client, DEFAULT_EL_SYNC_SAFE_OFFSET).await.unwrap(); + assert_eq!(result.un_safe, info(&blocks[7], &cfg)); + assert_eq!(result.safe, info(&blocks[1], &cfg)); + l1.assert_finished(); + l2.assert_finished(); +} + +#[derive(Debug, Clone, Copy)] +enum Path { + ReplacedUnsafe, + Conservative, +} +#[derive(Debug, Clone, Copy)] +enum Invalid { + EpochSkip, + NonzeroEpoch, + Sequence, + ParentHash, +} + +// Both traversal paths call the production RPC client and decode real L1-info deposits. +#[rstest::rstest] +#[case(Path::ReplacedUnsafe, Invalid::EpochSkip)] +#[case(Path::ReplacedUnsafe, Invalid::NonzeroEpoch)] +#[case(Path::ReplacedUnsafe, Invalid::Sequence)] +#[case(Path::Conservative, Invalid::EpochSkip)] +#[case(Path::Conservative, Invalid::NonzeroEpoch)] +#[case(Path::Conservative, Invalid::Sequence)] +#[case(Path::Conservative, Invalid::ParentHash)] +#[tokio::test] +async fn rejects_inconsistent_edges_on_every_path(#[case] path: Path, #[case] invalid: Invalid) { + let cfg = RollupConfig::default(); + let finalized = test_l2_block(1, 99, 0, B256::ZERO); + let parent = test_l2_block(2, 100, 0, finalized.header.hash); + let (origin, seq) = match invalid { + Invalid::EpochSkip => (102, 0), + Invalid::NonzeroEpoch => (101, 1), + Invalid::Sequence => (100, 2), + Invalid::ParentHash => (101, 0), + }; + let child = test_l2_block(3, origin, seq, parent.header.hash); + let mut l1_origin = l1_block(origin); + if matches!(invalid, Invalid::ParentHash) { + l1_origin.header.inner.parent_hash = B256::repeat_byte(0xff); + } + if matches!(path, Path::ReplacedUnsafe) { + l1_origin.header.hash = B256::repeat_byte(0xff); + } + let (client, l1, l2) = test_engine_client(Arc::new(cfg.clone())); + expect_heads(&l2, &finalized, &parent, &child); + l1.expect_params("eth_getBlockByNumber", json!([format!("{origin:#x}"), false]), &l1_origin); + expect_parent(&l2, &parent); + let err = + find_starting_forkchoice(&cfg, &client, DEFAULT_EL_SYNC_SAFE_OFFSET).await.unwrap_err(); + assert!( + match invalid { + Invalid::EpochSkip | Invalid::ParentHash => + matches!(err, SyncStartError::L1OriginMismatch), + Invalid::NonzeroEpoch => matches!(err, SyncStartError::NonZeroSequenceNumber), + Invalid::Sequence => matches!(err, SyncStartError::InconsistentSequenceNumber), + }, + "unexpected error: {err}" + ); + l1.assert_finished(); + l2.assert_finished(); +} + +#[test] +fn epoch_checks_compare_full_ids_and_reject_overflow() { + let mut parent = L2BlockInfo::default(); + let mut child = parent; + child.l1_origin.hash = B256::repeat_byte(1); // Same height, different hash is not a new epoch. + assert!(matches!( + check_epoch_and_sequence(child, parent, None), + Err(SyncStartError::L1OriginMismatch) + )); + parent.l1_origin.number = u64::MAX; + assert!(matches!( + check_epoch_and_sequence(child, parent, None), + Err(SyncStartError::L1OriginMismatch) + )); + child = parent; + parent.seq_num = u64::MAX; + assert!(matches!( + check_epoch_and_sequence(child, parent, None), + Err(SyncStartError::InconsistentSequenceNumber) + )); +} + +#[tokio::test] +async fn missing_l1_parent_retries_from_a_fresh_chain_view() { + let cfg = RollupConfig::default(); + let (client, l1, l2) = test_engine_client(Arc::new(cfg.clone())); + let mut walker = SyncWalker::new(&cfg, &client); + let mut head = L2BlockInfo::default(); + head.l1_origin.number = 101; + head.l1_origin.hash = l1_block(101).header.hash; + l1.expect_params("eth_getBlockByNumber", json!(["0x65", false]), l1_block(101)); + assert_eq!(walker.origin(head).await.unwrap(), OriginStatus::Canonical); + head.l1_origin.number = 100; + head.l1_origin.hash = l1_block(100).header.hash; + l1.expect_params( + "eth_getBlockByHash", + json!([head.l1_origin.hash, false]), + Option::::None, + ); + let err = walker.origin(head).await.unwrap_err(); + assert!(matches!(err, SyncStartError::ChainViewChanged)); + assert!(crate::EngineResetError::SyncStart(err).is_temporary()); + l1.assert_finished(); + l2.assert_finished(); +} + +#[tokio::test] +async fn rejects_reorgs_deeper_than_five_lookback_windows() { + let mut blocks = vec![]; + for number in 0..9 { + let parent = blocks.last().map_or(B256::ZERO, |b: &L2Block| b.header.hash); + blocks.push(test_l2_block(number, number, 0, parent)); + } + let mut cfg = RollupConfig { seq_window_size: 1, ..Default::default() }; + cfg.genesis.l2.hash = blocks[0].header.hash; + let (client, l1, l2) = test_engine_client(Arc::new(cfg.clone())); + expect_heads(&l2, &blocks[0], &blocks[0], &blocks[8]); + for number in (2..=8).rev() { + let mut replacement = l1_block(number); + replacement.header.hash = B256::with_last_byte(number as u8 + 100); + replacement.header.inner.parent_hash = B256::with_last_byte(number as u8 + 99); + l1.expect_params( + "eth_getBlockByNumber", + json!([format!("{number:#x}"), false]), + replacement, + ); + } + for parent in (2..8).rev() { + expect_parent(&l2, &blocks[parent]); + } + let err = + find_starting_forkchoice(&cfg, &client, DEFAULT_EL_SYNC_SAFE_OFFSET).await.unwrap_err(); + assert!(matches!(err, SyncStartError::ReorgTooDeep)); + assert!(!crate::EngineResetError::SyncStart(err).is_temporary()); + l1.assert_finished(); + l2.assert_finished(); +} diff --git a/rust/kona/crates/node/engine/src/task_queue/core.rs b/rust/kona/crates/node/engine/src/task_queue/core.rs index 816c8d0519e..f3f5369d6a9 100644 --- a/rust/kona/crates/node/engine/src/task_queue/core.rs +++ b/rust/kona/crates/node/engine/src/task_queue/core.rs @@ -2,14 +2,14 @@ use super::ExecutableEngineTask; use crate::{ - EngineClient, EngineState, EngineSyncStateUpdate, EngineTask, EngineTaskError, - EngineTaskErrorSeverity, Metrics, SyncStartError, SynchronizeTaskError, + DEFAULT_EL_SYNC_SAFE_OFFSET, EngineClient, EngineState, EngineSyncStateUpdate, EngineTask, + EngineTaskError, EngineTaskErrorSeverity, Metrics, SyncStartError, SynchronizeTaskError, find_starting_forkchoice, task_queue::{EngineTaskErrors, synchronize}, }; use kona_genesis::RollupConfig; use kona_protocol::L2BlockInfo; -use std::{cmp::Ordering, collections::BinaryHeap, sync::Arc}; +use std::{cmp::Ordering, collections::BinaryHeap, sync::Arc, time::Duration}; use thiserror::Error; use tokio::sync::watch::Sender; @@ -63,6 +63,9 @@ pub struct Engine { /// Preserve the task being retried ahead of newly enqueued work: it may have already /// performed part of an Engine API operation. active: Option, + /// How far behind an EL-synced tip a reset places safe and finalized when the execution + /// layer's labels are missing or stale. + el_sync_safe_offset: Duration, } impl Engine { @@ -74,9 +77,17 @@ impl Engine { tasks: BinaryHeap::default(), active: None, next_sequence: 0, + el_sync_safe_offset: DEFAULT_EL_SYNC_SAFE_OFFSET, } } + /// Sets how far behind an EL-synced tip a reset places safe and finalized when the execution + /// layer's labels are missing or stale. Defaults to [`DEFAULT_EL_SYNC_SAFE_OFFSET`]. + pub const fn with_el_sync_safe_offset(mut self, el_sync_safe_offset: Duration) -> Self { + self.el_sync_safe_offset = el_sync_safe_offset; + self + } + /// Returns a reference to the inner [`EngineState`]. pub const fn state(&self) -> &EngineState { &self.state @@ -113,7 +124,8 @@ impl Engine { client: Arc, config: Arc, ) -> Result { - let start = find_starting_forkchoice(&config, client.as_ref()).await?; + let start = + find_starting_forkchoice(&config, client.as_ref(), self.el_sync_safe_offset).await?; // One attempt. The actor retains the reset request and retries transient failures. synchronize( @@ -184,7 +196,11 @@ impl EngineResetError { /// Whether retrying the reset can recover without changing the node configuration. pub fn is_temporary(&self) -> bool { match self { - Self::SyncStart(SyncStartError::RpcError(_)) => true, + Self::SyncStart( + SyncStartError::RpcError(_) | + SyncStartError::ChainViewChanged | + SyncStartError::BlockNotFound(_), + ) => true, Self::SyncStart(_) => false, Self::Forkchoice(err) => err.severity() != EngineTaskErrorSeverity::Critical, } diff --git a/rust/kona/crates/node/engine/src/test_utils/misc.rs b/rust/kona/crates/node/engine/src/test_utils/misc.rs index fd77bf7ccf4..c506503aca5 100644 --- a/rust/kona/crates/node/engine/src/test_utils/misc.rs +++ b/rust/kona/crates/node/engine/src/test_utils/misc.rs @@ -1,6 +1,6 @@ use alloy_eips::BlockNumHash; -use alloy_primitives::B256; -use kona_protocol::{BlockInfo, L2BlockInfo}; +use alloy_primitives::{B256, Sealed, U256}; +use kona_protocol::{BlockInfo, L1BlockInfoBedrock, L2BlockInfo}; /// Helper to create a test `L2BlockInfo` at a specific block number pub fn test_block_info(number: u64) -> L2BlockInfo { @@ -15,3 +15,45 @@ pub fn test_block_info(number: u64) -> L2BlockInfo { seq_num: 0, } } + +/// The hash that reset traversal tests give the L1 block at `number`. Unlike a single byte, it is +/// distinct for every height. +pub fn test_l1_hash(number: u64) -> B256 { + B256::from(U256::from(number)) +} + +/// Builds a full L2 block with a real L1-info deposit for reset traversal tests. +pub fn test_l2_block( + number: u64, + origin: u64, + sequence: u64, + parent: B256, +) -> alloy_rpc_types_eth::Block { + let data = L1BlockInfoBedrock::new( + origin, + 0, + 0, + test_l1_hash(origin), + sequence, + Default::default(), + Default::default(), + Default::default(), + ) + .encode_calldata(); + let transaction = + op_alloy_consensus::OpTxEnvelope::Deposit(Sealed::new(op_alloy_consensus::TxDeposit { + input: data, + ..Default::default() + })); + let transaction = op_alloy_rpc_types::Transaction::from_transaction( + alloy_consensus::transaction::Recovered::new_unchecked(transaction, Default::default()), + Default::default(), + ); + let mut block: alloy_rpc_types_eth::Block = Default::default(); + block.header.inner.number = number; + block.header.inner.parent_hash = parent; + block.header.inner.timestamp = number * 2; + block.header.hash = block.header.inner.hash_slow(); + block.transactions = alloy_rpc_types_eth::BlockTransactions::Full(vec![transaction]); + block +} diff --git a/rust/kona/crates/node/engine/src/test_utils/mod.rs b/rust/kona/crates/node/engine/src/test_utils/mod.rs index a5575963a67..04e3018e7e7 100644 --- a/rust/kona/crates/node/engine/src/test_utils/mod.rs +++ b/rust/kona/crates/node/engine/src/test_utils/mod.rs @@ -8,4 +8,4 @@ mod engine_state; pub use engine_state::TestEngineStateBuilder; mod misc; -pub use misc::test_block_info; +pub use misc::{test_block_info, test_l1_hash, test_l2_block}; diff --git a/rust/kona/crates/node/engine/src/test_utils/rpc.rs b/rust/kona/crates/node/engine/src/test_utils/rpc.rs index be93a9816fe..7d14f1f0927 100644 --- a/rust/kona/crates/node/engine/src/test_utils/rpc.rs +++ b/rust/kona/crates/node/engine/src/test_utils/rpc.rs @@ -21,7 +21,9 @@ use std::{ use tower::Service; /// Queues replies and checks the method and optional parameters of every outgoing RPC request. -/// Unexpected requests panic rather than silently satisfying a different operation's reply. +/// A request takes the first queued reply whose method and parameters match, so tests pin which +/// requests are made without pinning the order of unrelated ones. Unexpected requests panic +/// rather than silently satisfying a different operation's reply. #[derive(Clone, Debug, Default)] pub struct RpcMock { requests: Arc>>, @@ -141,18 +143,22 @@ impl Service for RpcMock { let RequestPacket::Single(ref rpc) = request else { panic!("engine tests expect individual RPC requests"); }; - let expected = self - .requests - .lock() - .unwrap() - .pop_front() - .unwrap_or_else(|| panic!("unexpected RPC request: {}", rpc.method())); - assert_eq!(rpc.method(), expected.method, "wrong RPC method"); - if let Some(params) = expected.params { - let actual: Value = - serde_json::from_str(rpc.params().expect("request has parameters").get()).unwrap(); - assert_eq!(actual, params, "wrong RPC parameters for {}", expected.method); - } + let actual: Option = + rpc.params().map(|params| serde_json::from_str(params.get()).unwrap()); + let expected = { + let mut requests = self.requests.lock().unwrap(); + let index = requests + .iter() + .position(|expected| { + expected.method == rpc.method() && + expected + .params + .as_ref() + .is_none_or(|params| actual.as_ref() == Some(params)) + }) + .unwrap_or_else(|| panic!("unexpected RPC request: {} {actual:?}", rpc.method())); + requests.remove(index).expect("index is in bounds") + }; let replies = Asserter::new(); let Some(reply) = expected.response else { return Box::pin(async { diff --git a/rust/kona/crates/node/service/src/actors/engine/tests.rs b/rust/kona/crates/node/service/src/actors/engine/tests.rs index b61712d95b5..a013ec06928 100644 --- a/rust/kona/crates/node/service/src/actors/engine/tests.rs +++ b/rust/kona/crates/node/service/src/actors/engine/tests.rs @@ -4,7 +4,7 @@ use alloy_primitives::B256; use alloy_rpc_types_engine::{ForkchoiceUpdated, PayloadStatus, PayloadStatusEnum}; use kona_engine::{ Engine, EngineActorRequest, EngineState, EngineSyncStateUpdate, FinalizeBlockId, NoopBlockSink, - test_utils::{TestAttributesBuilder, test_engine_client}, + test_utils::{TestAttributesBuilder, test_engine_client, test_l2_block}, }; use kona_genesis::RollupConfig; use kona_protocol::{BlockInfo, L2BlockInfo}; @@ -86,7 +86,7 @@ async fn invalid_forkchoice_resets_and_resumes(#[case] rebuild: bool, #[case] ho l2.expect_params("eth_getBlockByNumber", serde_json::json!([tag, true]), &block); } l1.expect( - "eth_getBlockByHash", + "eth_getBlockByNumber", alloy_rpc_types_eth::Block::::default(), ); l2.expect( @@ -141,29 +141,55 @@ async fn invalid_forkchoice_resets_and_resumes(#[case] rebuild: bool, #[case] ho tasks.abort_all(); } +#[derive(Debug, Clone, Copy)] +enum ResetFailure { + L1, + L2, + L2Parent, +} + /// Both L1 and L2 outages during a reset must retain the original request and finish it /// after recovery, without a new reset request or L1 head to wake the actor. #[rstest::rstest] -#[case(true, Duration::ZERO)] -#[case(false, Duration::ZERO)] -#[case(false, Duration::from_secs(4))] +#[case::l1(ResetFailure::L1, Duration::ZERO)] +#[case::l2(ResetFailure::L2, Duration::ZERO)] +#[case::slow_l2(ResetFailure::L2, Duration::from_secs(4))] +#[case::missing_l2_parent(ResetFailure::L2Parent, Duration::ZERO)] #[tokio::test(start_paused = true)] async fn admin_reset_recovers_and_completes_original_request( - #[case] l1_failure: bool, + #[case] failure: ResetFailure, #[case] read_delay: Duration, ) { - let block: ::BlockResponse = - Default::default(); + let block = test_l2_block(0, 0, 0, B256::ZERO); let mut config = RollupConfig::default(); - config.genesis.l2.hash = block.header.inner.hash_slow(); + config.genesis.l2.hash = block.header.hash; + // Keep an unsafe block above genesis so both the L1 origin check and the safe walk run. + let unsafe_block = test_l2_block(1, 0, 1, config.genesis.l2.hash); + let config = Arc::new(config); let (client, l1, l2) = test_engine_client(config.clone()); let client = Arc::new(client); - if l1_failure { + if matches!(failure, ResetFailure::L1 | ResetFailure::L2Parent) { for tag in ["finalized", "safe", "latest"] { - l2.expect_params("eth_getBlockByNumber", serde_json::json!([tag, true]), &block); + l2.expect_params( + "eth_getBlockByNumber", + serde_json::json!([tag, true]), + if tag == "latest" { &unsafe_block } else { &block }, + ); + } + if matches!(failure, ResetFailure::L1) { + l1.expect_error("eth_getBlockByNumber"); + } else { + l1.expect( + "eth_getBlockByNumber", + alloy_rpc_types_eth::Block::::default(), + ); + l2.expect_params( + "eth_getBlockByHash", + serde_json::json!([block.header.hash, true]), + serde_json::Value::Null, + ); } - l1.expect_error("eth_getBlockByHash"); } else { l2.expect_error("eth_getBlockByNumber"); } @@ -180,7 +206,7 @@ async fn admin_reset_recovers_and_completes_original_request( tokio::sync::watch::channel(kona_protocol::L2BlockInfo::default()); let mut actor = EngineActor::new( client, - config, + config.clone(), derivation, Engine::new(EngineState::default(), state_tx), Some(unsafe_head_tx), @@ -193,11 +219,16 @@ async fn admin_reset_recovers_and_completes_original_request( actor.step().await.unwrap(); // retain the reset actor.step().await.unwrap(); // failed attempt, then wait for backoff assert!(!reset.is_finished()); - for _ in 0..3 { - l2.expect_with_delay("eth_getBlockByNumber", &block, read_delay); + for response in [&block, &block, &unsafe_block] { + l2.expect_with_delay("eth_getBlockByNumber", response, read_delay); } - l1.expect( + l2.expect_params( "eth_getBlockByHash", + serde_json::json!([config.genesis.l2.hash, true]), + &block, + ); + l1.expect( + "eth_getBlockByNumber", alloy_rpc_types_eth::Block::::default(), ); l2.expect( @@ -218,14 +249,14 @@ async fn admin_reset_recovers_and_completes_original_request( // when stopped sequencing leaves the engine task queue empty. let published = *state_rx.borrow(); for head in [ - published.sync_state.unsafe_head(), published.sync_state.local_safe_head(), published.sync_state.safe_head(), published.sync_state.finalized_head(), ] { assert_eq!(head.hash(), block.header.inner.hash_slow()); } - assert_eq!(unsafe_head_rx.borrow().hash(), block.header.inner.hash_slow()); + assert_eq!(published.sync_state.unsafe_head().hash(), unsafe_block.header.hash); + assert_eq!(unsafe_head_rx.borrow().hash(), unsafe_block.header.hash); assert!(matches!( derivation_rx.recv().await.unwrap(), DerivationActorRequest::ProcessL1HeadUpdateRequest(_) @@ -250,6 +281,95 @@ async fn admin_reset_recovers_and_completes_original_request( l2.assert_finished(); } +/// EL-sync completion must bootstrap derivation without reading the synced chain's ancestry. Like +/// op-node, safe and finalized are placed the EL-sync offset behind the tip, which arrived over +/// P2P and may not be on L1 yet. +#[tokio::test(start_paused = true)] +async fn el_sync_completion_recovers_genesis_labels_without_a_historical_walk() { + let genesis = test_l2_block(0, 0, 0, B256::ZERO); + let mut config = RollupConfig { block_time: 2, ..Default::default() }; + config.genesis.l2.hash = genesis.header.hash; + let tip = test_l2_block(100_000_000, 14 * config.seq_window_size + 1, 0, B256::ZERO); + let tip_info = + L2BlockInfo::from_block_and_genesis(&tip.clone().into_consensus(), &config.genesis) + .unwrap(); + // The default 12h offset is 21,600 two-second blocks. + let offset_number = 100_000_000 - 21_600; + let offset_head = test_l2_block(offset_number, 14 * config.seq_window_size, 0, B256::ZERO); + let offset_info = + L2BlockInfo::from_block_and_genesis(&offset_head.clone().into_consensus(), &config.genesis) + .unwrap(); + let config = Arc::new(config); + let (client, l1, l2) = test_engine_client(config.clone()); + let state = EngineState { + sync_state: EngineState::default().sync_state.apply_update(EngineSyncStateUpdate { + unsafe_head: Some(tip_info), + ..Default::default() + }), + el_sync_finished: true, + ..Default::default() + }; + let (state_tx, state_rx) = tokio::sync::watch::channel(state); + let (requests, request_rx) = mpsc::channel(1); + let (derivation_tx, mut derivation_rx) = mpsc::channel(2); + let mut actor = EngineActor::new( + Arc::new(client), + config, + QueuedEngineDerivationClient { derivation_actor_request_tx: derivation_tx }, + Engine::new(state, state_tx), + None, + request_rx, + Arc::new(NoopBlockSink), + ); + // The completed EL sync with an unset finalized head schedules its own reset. + actor.step().await.unwrap(); + for (tag, block) in [("finalized", &genesis), ("safe", &genesis), ("latest", &tip)] { + l2.expect_params("eth_getBlockByNumber", serde_json::json!([tag, true]), block); + } + l2.expect_params( + "eth_getBlockByNumber", + serde_json::json!([format!("{offset_number:#x}"), true]), + &offset_head, + ); + l2.expect_params( + "engine_forkchoiceUpdatedV3", + serde_json::json!([ + alloy_rpc_types_engine::ForkchoiceState { + head_block_hash: tip_info.hash(), + safe_block_hash: offset_info.hash(), + finalized_block_hash: offset_info.hash(), + }, + null + ]), + ForkchoiceUpdated::new(PayloadStatus::from_status(PayloadStatusEnum::Valid)), + ); + // Closing input lets the step return after recovery and its notifications have completed. + drop(requests); + assert!(matches!(actor.step().await, Err(crate::EngineError::ChannelClosed))); + let published = *state_rx.borrow(); + assert_eq!(published.sync_state.unsafe_head(), tip_info); + assert_eq!(published.sync_state.local_safe_head(), offset_info); + assert_eq!(published.sync_state.safe_head(), offset_info); + assert_eq!(published.sync_state.finalized_head(), offset_info); + let DerivationActorRequest::ProcessEngineSignalRequest(signal) = + derivation_rx.recv().await.unwrap() + else { + panic!("expected the reset head"); + }; + assert!( + matches!(*signal, kona_derive::Signal::Reset(reset) if reset.l2_safe_head == offset_info) + ); + let DerivationActorRequest::ProcessEngineSyncCompletionRequest(head) = + derivation_rx.recv().await.unwrap() + else { + panic!("expected EL-sync completion after reset"); + }; + assert_eq!(*head, offset_info); + assert!(derivation_rx.try_recv().is_err()); + l1.assert_finished(); + l2.assert_finished(); +} + fn l2_block(number: u64) -> L2BlockInfo { L2BlockInfo { block_info: BlockInfo { diff --git a/rust/kona/crates/node/service/src/service/builder.rs b/rust/kona/crates/node/service/src/service/builder.rs index 8400aec1f6f..3bd4a4fb514 100644 --- a/rust/kona/crates/node/service/src/service/builder.rs +++ b/rust/kona/crates/node/service/src/service/builder.rs @@ -13,10 +13,11 @@ use alloy_transport_http::{ }; use http_body_util::Full; use op_alloy_network::Optimism; -use std::sync::Arc; +use std::{sync::Arc, time::Duration}; use tower::ServiceBuilder; use url::Url; +use kona_engine::DEFAULT_EL_SYNC_SAFE_OFFSET; use kona_genesis::{L1ChainConfig, RollupConfig}; use kona_interop::DependencySet; use kona_providers_alloy::OnlineBeaconClient; @@ -82,6 +83,9 @@ pub struct RollupNodeBuilder { pub safe_db: Option>, /// Number of L1 confirmations required before deriving L2 blocks. pub verifier_l1_confs: u64, + /// How far behind an EL-synced tip a reset places safe and finalized when the execution + /// layer's labels are missing or stale. + pub el_sync_safe_offset: Duration, } impl RollupNodeBuilder { @@ -108,6 +112,7 @@ impl RollupNodeBuilder { dependency_set: None, safe_db: None, verifier_l1_confs: 0, + el_sync_safe_offset: DEFAULT_EL_SYNC_SAFE_OFFSET, } } @@ -130,6 +135,12 @@ impl RollupNodeBuilder { Self { verifier_l1_confs, ..self } } + /// Sets how far behind an EL-synced tip a reset places safe and finalized when the execution + /// layer's labels are missing or stale. + pub fn with_el_sync_safe_offset(self, el_sync_safe_offset: Duration) -> Self { + Self { el_sync_safe_offset, ..self } + } + /// Sets the [`EngineConfig`] on the [`RollupNodeBuilder`]. pub fn with_engine_config(self, engine_config: EngineConfig) -> Self { Self { engine_config, ..self } @@ -222,6 +233,7 @@ impl RollupNodeBuilder { dependency_set: self.dependency_set, safe_db: self.safe_db, verifier_l1_confs: self.verifier_l1_confs, + el_sync_safe_offset: self.el_sync_safe_offset, } } } diff --git a/rust/kona/crates/node/service/src/service/node/rollup.rs b/rust/kona/crates/node/service/src/service/node/rollup.rs index d1b5fb0a12e..786852a991b 100644 --- a/rust/kona/crates/node/service/src/service/node/rollup.rs +++ b/rust/kona/crates/node/service/src/service/node/rollup.rs @@ -119,6 +119,8 @@ pub struct RollupNode { pub(crate) safe_db: Option>, /// L1 confirmation depth for online derivation, independent of sequencing. pub(crate) verifier_l1_confs: u64, + /// How far behind an EL-synced tip a reset places safe and finalized. + pub(crate) el_sync_safe_offset: Duration, } /// A RollupNode-level derivation actor wrapper. @@ -262,7 +264,8 @@ impl RollupNode { // Share engine state with RPC without routing reads through the actor. let engine_state = EngineState::default(); let (engine_state_tx, engine_state_rx) = watch::channel(engine_state); - let engine = Engine::new(engine_state, engine_state_tx); + let engine = Engine::new(engine_state, engine_state_tx) + .with_el_sync_safe_offset(self.el_sync_safe_offset); let engine_client = Arc::new(self.engine_config.clone().build_engine_client());