From 5bdce9163d85f18dc5f365f7f7817f65b6baa62b Mon Sep 17 00:00:00 2001 From: Josh Klopfenstein Date: Mon, 2 Nov 2026 19:18:03 -0800 Subject: [PATCH 1/2] kona-engine,kona-node!: align reset recovery with op-node kona's sync-start search could keep unsafe blocks built on replaced L1 origins, and chose different safe boundaries from op-node. After EL sync it also walked every L2 block back to genesis, one RPC at a time, before deriving again from genesis. Port op-node's FindL2Heads: - the walk's plausible-origin, epoch and sequence checks, previous-safe and epoch-boundary limits, finality floor and maximum reorg depth; - the early recovery when EL sync leaves safe and finalized at genesis on a chain older than fourteen sequencing windows, or the execution layer was rewound below its safe or finalized label. Like op-node's L2HeadsForELSyncWithOffset, the unsafe head stays at the synced tip and safe and finalized are placed an offset behind it. The tip arrived over P2P and may not be batched yet, so it is never labelled safe or final. The offset is the new --syncmode.offset-el-safe flag (KONA_NODE_SYNCMODE_OFFSET_EL_SAFE), which defaults to 12h like op-node's. Retry when a canonical L1 parent vanishes during the walk, and when an L2 block is missing while the execution layer catches up. Log progress during long walks. A replaced L1 origin at the finalized block is reported as its own error. Related to ethereum-optimism/protocol-team#328. BREAKING CHANGE: After EL sync, or when the execution layer's safe or finalized label is ahead of its chain, kona-node now sets safe and finalized 12h of L2 blocks behind the synced tip. Before, finalized stayed at genesis and safe was found by walking back from the tip. Set --syncmode.offset-el-safe to change the offset. find_starting_forkchoice takes the offset as a new argument; DEFAULT_EL_SYNC_SAFE_OFFSET matches op-node's default. Exhaustive SyncStartError matches must handle ChainViewChanged (retry with a fresh L1 view), ReorgTooDeep (recovery exceeds five sync lookback windows) and FinalizedL1OriginReplaced. BlockNotFound is now treated as retryable. Co-Authored-By: Claude Opus 5.5 --- rust/Cargo.lock | 1 + rust/kona/bin/node/Cargo.toml | 1 + .../bin/node/src/commands/node/command.rs | 15 +- rust/kona/bin/node/src/commands/node/tests.rs | 27 + rust/kona/crates/node/engine/src/lib.rs | 4 +- .../kona/crates/node/engine/src/sync/error.rs | 9 + rust/kona/crates/node/engine/src/sync/mod.rs | 325 ++++++++--- .../kona/crates/node/engine/src/sync/tests.rs | 551 ++++++++++++++++++ .../crates/node/engine/src/task_queue/core.rs | 26 +- .../crates/node/engine/src/test_utils/misc.rs | 46 +- .../crates/node/engine/src/test_utils/mod.rs | 2 +- .../crates/node/engine/src/test_utils/rpc.rs | 32 +- .../node/service/src/actors/engine/tests.rs | 156 ++++- .../node/service/src/service/builder.rs | 14 +- .../node/service/src/service/node/rollup.rs | 5 +- 15 files changed, 1100 insertions(+), 114 deletions(-) create mode 100644 rust/kona/crates/node/engine/src/sync/tests.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index adddd373f53..50ac5ac901f 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -7609,6 +7609,7 @@ dependencies = [ "dirs 6.0.0", "discv5 0.12.0", "enr 0.14.0", + "humantime", "kona-cli", "kona-derive", "kona-disc", diff --git a/rust/kona/bin/node/Cargo.toml b/rust/kona/bin/node/Cargo.toml index 4746aece14d..b63326a690d 100644 --- a/rust/kona/bin/node/Cargo.toml +++ b/rust/kona/bin/node/Cargo.toml @@ -54,6 +54,7 @@ op-alloy-network.workspace = true # general url.workspace = true dirs.workspace = true +humantime.workspace = true strum.workspace = true discv5.workspace = true enr.workspace = true diff --git a/rust/kona/bin/node/src/commands/node/command.rs b/rust/kona/bin/node/src/commands/node/command.rs index 8b5ba53775f..b32ab0a59d9 100644 --- a/rust/kona/bin/node/src/commands/node/command.rs +++ b/rust/kona/bin/node/src/commands/node/command.rs @@ -12,7 +12,7 @@ use anyhow::{Context as _, Result, bail}; use backon::{ExponentialBuilder, Retryable}; use clap::Parser; use kona_cli::{LogConfig, MetricsArgs}; -use kona_engine::EngineClient; +use kona_engine::{DEFAULT_EL_SYNC_SAFE_OFFSET, EngineClient}; use kona_genesis::{L1ChainConfig, RollupConfig}; use kona_interop::DependencySet; use kona_node_service::{EngineConfig, L1ConfigBuilder, NodeMode, RollupNodeBuilder}; @@ -25,6 +25,7 @@ use std::{ io::{Read, Write}, path::PathBuf, sync::Arc, + time::Duration, }; use strum::IntoEnumIterator; use tracing::{debug, error, info}; @@ -126,6 +127,16 @@ pub struct NodeCommand { /// Reorgs are supported, but may be slow to recover from. #[arg(long = "verifier.l1-confs", env = "KONA_NODE_VERIFIER_L1_CONFS", default_value_t = 0)] pub verifier_l1_confs: u64, + /// After execution-layer sync, or when the execution layer's forkchoice labels are behind its + /// chain, set the safe and finalized heads this far behind the synced tip (converted to L2 + /// blocks with ceiling division). Mirrors op-node's `--syncmode.offset-el-safe`. + #[arg( + long = "syncmode.offset-el-safe", + env = "KONA_NODE_SYNCMODE_OFFSET_EL_SAFE", + default_value = "12h", + value_parser = humantime::parse_duration, + )] + pub el_sync_safe_offset: Duration, /// P2P CLI arguments. #[command(flatten)] pub p2p_flags: P2PArgs, @@ -148,6 +159,7 @@ impl Default for NodeCommand { interop_dependency_set: None, safedb_path: None, verifier_l1_confs: 0, + el_sync_safe_offset: DEFAULT_EL_SYNC_SAFE_OFFSET, node_mode: NodeMode::Validator, p2p_flags: P2PArgs::default(), rpc_flags: RpcArgs::default(), @@ -349,6 +361,7 @@ impl NodeCommand { .with_dependency_set(dependency_set) .with_safe_db(safe_db) .with_verifier_l1_confs(self.verifier_l1_confs) + .with_el_sync_safe_offset(self.el_sync_safe_offset) .build() .start() .await diff --git a/rust/kona/bin/node/src/commands/node/tests.rs b/rust/kona/bin/node/src/commands/node/tests.rs index 0cfb440c62b..c2c295b3ff6 100644 --- a/rust/kona/bin/node/src/commands/node/tests.rs +++ b/rust/kona/bin/node/src/commands/node/tests.rs @@ -59,6 +59,33 @@ fn verifier_confirmation_depth_has_the_expected_environment_variable() { assert_eq!(arg.get_env().unwrap(), "KONA_NODE_VERIFIER_L1_CONFS"); } +/// Like op-node, the EL-sync safe offset defaults to 12h and takes durations such as `30m`. +#[rstest::rstest] +#[case::default(None, 12 * 60 * 60)] +#[case::minutes(Some("30m"), 30 * 60)] +#[case::disabled(Some("0s"), 0)] +fn el_sync_safe_offset_parses_durations(#[case] flag: Option<&str>, #[case] seconds: u64) { + let flag = flag.map(|value| ["--syncmode.offset-el-safe", value]); + let args = NodeCommand::parse_from( + std::iter::once("node") + .chain(default_flags().iter().copied()) + .chain(flag.into_iter().flatten()), + ); + assert_eq!(args.el_sync_safe_offset, std::time::Duration::from_secs(seconds)); +} + +#[test] +fn el_sync_safe_offset_default_matches_the_engine() { + assert_eq!( + NodeCommand::default().el_sync_safe_offset, + kona_engine::DEFAULT_EL_SYNC_SAFE_OFFSET + ); + use clap::CommandFactory; + let command = NodeCommand::command(); + let arg = command.get_arguments().find(|arg| arg.get_id() == "el_sync_safe_offset").unwrap(); + assert_eq!(arg.get_env().unwrap(), "KONA_NODE_SYNCMODE_OFFSET_EL_SAFE"); +} + #[test] fn test_node_cli_missing_l1_eth_rpc() { let err = NodeCommand::try_parse_from(["node"]).unwrap_err(); diff --git a/rust/kona/crates/node/engine/src/lib.rs b/rust/kona/crates/node/engine/src/lib.rs index c1f955096bb..f4a5cadeb22 100644 --- a/rust/kona/crates/node/engine/src/lib.rs +++ b/rust/kona/crates/node/engine/src/lib.rs @@ -74,7 +74,9 @@ mod metrics; pub use metrics::Metrics; mod sync; -pub use sync::{L2ForkchoiceState, SyncStartError, find_starting_forkchoice}; +pub use sync::{ + DEFAULT_EL_SYNC_SAFE_OFFSET, L2ForkchoiceState, SyncStartError, find_starting_forkchoice, +}; #[cfg(any(test, feature = "test-utils"))] /// Utilities that are useful when creating unit tests using structs within this library. diff --git a/rust/kona/crates/node/engine/src/sync/error.rs b/rust/kona/crates/node/engine/src/sync/error.rs index d98b69d642b..feec542dd40 100644 --- a/rust/kona/crates/node/engine/src/sync/error.rs +++ b/rust/kona/crates/node/engine/src/sync/error.rs @@ -9,6 +9,9 @@ use thiserror::Error; /// An error that can occur during the sync start process. #[derive(Error, Debug)] pub enum SyncStartError { + /// The canonical L1 view changed during recovery; retry with a fresh view. + #[error("Canonical chain changed during sync-start recovery")] + ChainViewChanged, /// An rpc error occurred #[error("An RPC error occurred: {0}")] RpcError(#[from] RpcError), @@ -29,6 +32,12 @@ pub enum SyncStartError { /// Finalized block mismatch #[error("Finalized block mismatch. Expected {0}, Got {1}")] MismatchedFinalizedBlock(B256, B256), + /// Recovery would rewind the unsafe origin by more than op-node's five lookback windows. + #[error("L1 reorg exceeds five sync lookback windows")] + ReorgTooDeep, + /// The finalized L2 block's L1 origin is no longer canonical, so L1 reorged below finality. + #[error("Finalized L2 block's L1 origin is no longer canonical")] + FinalizedL1OriginReplaced, /// L1 origin mismatch. #[error("L1 origin mismatch")] L1OriginMismatch, diff --git a/rust/kona/crates/node/engine/src/sync/mod.rs b/rust/kona/crates/node/engine/src/sync/mod.rs index cdf468ba554..614475806cc 100644 --- a/rust/kona/crates/node/engine/src/sync/mod.rs +++ b/rust/kona/crates/node/engine/src/sync/mod.rs @@ -1,7 +1,9 @@ //! Sync start algorithm for the OP Stack rollup node. +use alloy_eips::BlockId; +use alloy_primitives::B256; use kona_genesis::RollupConfig; -use kona_protocol::L2BlockInfo; +use kona_protocol::{BlockInfo, L2BlockInfo}; mod forkchoice; pub use forkchoice::L2ForkchoiceState; @@ -9,25 +11,42 @@ pub use forkchoice::L2ForkchoiceState; mod error; pub use error::SyncStartError; -use tracing::info; +use std::time::Duration; +use tracing::{info, warn}; use crate::EngineClient; -/// Searches for the latest [`L2ForkchoiceState`] that we can use to start the sync process with. +/// Matches op-node's threshold for recovering EL-sync forkchoice labels left at genesis. +const RECOVER_MIN_SEQ_WINDOWS: u64 = 14; + +/// Matches op-node's `MaxReorgSeqWindows`: how many sync lookback windows the unsafe head's L1 +/// origin may be rewound before recovery gives up. +const MAX_REORG_SEQ_WINDOWS: u64 = 5; + +/// How many L2 blocks the walk traverses between progress logs. +const PROGRESS_LOG_INTERVAL: u64 = 10_000; + +/// Matches op-node's default `--syncmode.offset-el-safe`: how far behind an EL-synced tip the +/// recovered safe and finalized heads are placed. The tip arrived over P2P and may not be +/// batched to L1 yet, so it must not be labelled safe or finalized. +pub const DEFAULT_EL_SYNC_SAFE_OFFSET: Duration = Duration::from_secs(12 * 60 * 60); + +/// Searches for a sync starting point using op-node's `FindL2Heads` rules, and returns: /// -/// - The *unsafe L2 block*: This is the highest L2 block whose L1 origin is a *plausible* -/// extension of the canonical L1 chain (as known to the rollup node). -/// - The *safe L2 block*: This is the highest L2 block whose epoch's sequencing window is -/// complete within the canonical L1 chain (as known to the rollup node). -/// - The *finalized L2 block*: This is the L2 block which is known to be fully derived from -/// finalized L1 block data. +/// - The *unsafe* head: the highest L2 block whose L1 origin, and every ancestor's, is a plausible +/// extension of the canonical L1 chain. An origin not yet visible on L1 is plausible. +/// - The *safe* head: the highest L2 block whose epoch's sequencing window is complete within the +/// canonical L1 chain. Recovery stops at an epoch boundary below both the previous safe head and +/// a full lookback window, or at finality. +/// - The *finalized* head: unchanged, as it is fully derived from finalized L1 data. /// -/// Plausible: meaning that the blockhash of the L2 block's L1 origin -/// (as reported in the L1 Attributes deposit within the L2 block) is not canonical at another -/// height in the L1 chain, and the same holds for all its ancestors. +/// When the execution layer lost its safe and finalized labels after EL sync, or was rewound +/// below them, the unsafe head stays at the tip and safe and finalized are placed +/// `el_sync_safe_offset` behind it, as op-node's `L2HeadsForELSyncWithOffset` does. pub async fn find_starting_forkchoice( cfg: &RollupConfig, engine_client: &EngineClient, + el_sync_safe_offset: Duration, ) -> Result { let mut current_fc = L2ForkchoiceState::current(cfg, engine_client).await?; info!( @@ -38,79 +57,240 @@ pub async fn find_starting_forkchoice( "Loaded current L2 EL forkchoice state" ); - // Search for the highest `unsafe` block, relative to the initial `unsafe` block's L1 origin, + let missing_finality = current_fc.finalized.hash() == cfg.genesis.l2.hash && + current_fc.safe.hash() == cfg.genesis.l2.hash && + current_fc.un_safe.block_info.number > cfg.genesis.l2.number && + current_fc.un_safe.l1_origin.number > + cfg.genesis + .l1 + .number + .saturating_add(RECOVER_MIN_SEQ_WINDOWS.saturating_mul(cfg.seq_window_size)); + let rewound = current_fc.un_safe.block_info.number < current_fc.safe.block_info.number || + current_fc.un_safe.block_info.number < current_fc.finalized.block_info.number; + if missing_finality || rewound { + // Handle this before the normal walk: stale labels are not a reorg finality boundary. + warn!(target: "sync_start", ?current_fc, missing_finality, rewound, ?el_sync_safe_offset, "Recovering missing or stale execution-layer forkchoice labels"); + return heads_for_el_sync(cfg, engine_client, el_sync_safe_offset, current_fc.un_safe) + .await; + } + + let previous_unsafe = current_fc.un_safe; + let previous_safe = current_fc.safe; + let mut cursor = previous_unsafe; + let mut unsafe_candidate = Some(cursor); + let mut highest_canonical: Option = None; + let mut walker = SyncWalker::new(cfg, engine_client); + let lookback = cfg.seq_window_size; + let mut walked = 0u64; + + // Port of op-node's normal FindL2Heads walk. Keep plausible heads ahead of L1, discard + // replaced origins, and validate every traversed child/parent epoch and sequence relation. loop { - let l1_origin = - engine_client.get_l1_block(current_fc.un_safe.l1_origin.hash.into()).await?; - info!( - target: "sync_start", - l1_origin = %current_fc.un_safe.l1_origin.number, - l2_unsafe = %current_fc.un_safe.block_info.number, - "Searching for L2 unsafe block with canonical L1 origin" - ); - - match l1_origin { - Some(_) => { - // Unsafe block has existing L1 origin. Continue with this head. - info!( - target: "sync_start", - l2_unsafe = %current_fc.un_safe.block_info.number, - "Found L2 unsafe block with canonical L1 origin" - ); - break; + if cursor.block_info.number < current_fc.finalized.block_info.number || + (cursor.block_info.number == current_fc.finalized.block_info.number && + cursor.hash() != current_fc.finalized.hash()) + { + return Err(SyncStartError::MismatchedFinalizedBlock( + current_fc.finalized.hash(), + cursor.hash(), + )); + } + let origin = walker.origin(cursor).await?; + if cursor.block_info.number == cfg.genesis.l2.number { + if cursor.hash() != cfg.genesis.l2.hash { + return Err(SyncStartError::InvalidL2GenesisHash( + cfg.genesis.l2.hash, + cursor.hash(), + )); + } + if let Some(l1) = walker.l1 && + l1.hash != cfg.genesis.l1.hash + { + return Err(SyncStartError::InvalidL1GenesisHash(cfg.genesis.l1.hash, l1.hash)); } + } + let unsafe_head = match unsafe_candidate { + Some(head) => head, None => { - let l2_parent_hash = current_fc.un_safe.block_info.parent_hash.into(); - let l2_parent = engine_client - .get_l2_block(l2_parent_hash) - .full() - .await? - .ok_or(SyncStartError::BlockNotFound(l2_parent_hash))?; - - current_fc.un_safe = - L2BlockInfo::from_block_and_genesis(&l2_parent.into_consensus(), &cfg.genesis)?; + if cursor + .l1_origin + .number + .saturating_add(MAX_REORG_SEQ_WINDOWS.saturating_mul(lookback)) < + previous_unsafe.l1_origin.number + { + return Err(SyncStartError::ReorgTooDeep); + } + unsafe_candidate = Some(cursor); + cursor + } + }; + match origin { + OriginStatus::Ahead => highest_canonical = None, + OriginStatus::Canonical => { + highest_canonical.get_or_insert(cursor); + } + OriginStatus::Replaced => { + unsafe_candidate = None; + highest_canonical = None; } } - } - - // Search for the highest `safe` block that's L1 origin is at least older than the sequencing - // window, relative to the L1 origin of the `unsafe` block. - let mut safe_cursor = current_fc.un_safe; - loop { - info!( - target: "sync_start", - l1_origin = %safe_cursor.l1_origin.number, - l2_safe = %safe_cursor.block_info.number, - "Searching for L2 safe block beyond sequencing window" - ); - - let is_behind_sequence_window = - current_fc.un_safe.l1_origin.number.saturating_sub(cfg.seq_window_size) > - safe_cursor.l1_origin.number; - let is_finalized = safe_cursor.block_info.hash == current_fc.finalized.block_info.hash; - let is_genesis = safe_cursor.block_info.hash == cfg.genesis.l2.hash; - if is_behind_sequence_window || is_finalized || is_genesis { + if cursor.block_info.number == current_fc.finalized.block_info.number { + // op-node returns an empty unsafe head here. Refuse instead: L1 reorged below the + // finalized block's origin, which finality rules out. + current_fc.un_safe = + unsafe_candidate.ok_or(SyncStartError::FinalizedL1OriginReplaced)?; + current_fc.safe = cursor; + break; + } + // Use the parent of an epoch boundary below both the previous safe head and a full + // canonical lookback window. Finality above is the lower bound. + let ready = cursor.block_info.number <= previous_safe.block_info.number && + highest_canonical.is_some_and(|high| { + cursor.l1_origin.number.saturating_add(lookback) < high.l1_origin.number + }) && + cursor.seq_num == 0; + cursor = walker.parent(cursor).await?; + if ready { + // A ready block's origin is canonical, so this iteration kept its unsafe head. + current_fc.un_safe = unsafe_head; + current_fc.safe = cursor; + break; + } + walked += 1; + if walked.is_multiple_of(PROGRESS_LOG_INTERVAL) { info!( target: "sync_start", - l2_safe = %safe_cursor.block_info.number, - is_behind_sequence_window, - is_finalized, - is_genesis, - "Found suitable L2 safe block" + walked, + cursor = cursor.block_info.number, + finalized = current_fc.finalized.block_info.number, + "Walking back to find the sync start" ); - current_fc.safe = safe_cursor; - break; } + } + info!(target: "sync_start", walked, %current_fc, "Found sync start"); + + // Leave the finalized block as-is, and return the current forkchoice. + Ok(current_fc) +} + +/// Port of op-node's `L2HeadsForELSyncWithOffset`: the unsafe head stays at the EL-synced `tip`, +/// and safe and finalized retract by `offset`, converted to blocks with ceiling division and +/// clamped at genesis. +async fn heads_for_el_sync( + cfg: &RollupConfig, + engine_client: &EngineClient, + offset: Duration, + tip: L2BlockInfo, +) -> Result { + let target = + offset_block_number(offset, cfg.block_time, tip.block_info.number, cfg.genesis.l2.number); + let derived = if target < tip.block_info.number { + let id = BlockId::number(target); let block = engine_client - .get_l2_block(safe_cursor.block_info.parent_hash.into()) + .get_l2_block(id) .full() .await? - .ok_or(SyncStartError::BlockNotFound(safe_cursor.block_info.parent_hash.into()))?; - safe_cursor = L2BlockInfo::from_block_and_genesis(&block.into_consensus(), &cfg.genesis)?; + .ok_or(SyncStartError::BlockNotFound(id))?; + L2BlockInfo::from_block_and_genesis(&block.into_consensus(), &cfg.genesis)? + } else { + tip + }; + Ok(L2ForkchoiceState { un_safe: tip, safe: derived, finalized: derived }) +} + +/// Port of op-node's `OffsetBlockNum`: the block number `ceil(offset / block_time)` blocks behind +/// `head`, never below `genesis`. +pub(crate) fn offset_block_number( + offset: Duration, + block_time: u64, + head: u64, + genesis: u64, +) -> u64 { + if block_time == 0 || head <= genesis { + return head; } + head - offset.as_secs().div_ceil(block_time).min(head - genesis) +} - // Leave the finalized block as-is, and return the current forkchoice. - Ok(current_fc) +#[derive(Debug, PartialEq, Eq)] +enum OriginStatus { + Canonical, + Replaced, + Ahead, +} + +/// The same L1-origin cursor and child/parent checks used by op-node `FindL2Heads`. +struct SyncWalker<'a> { + cfg: &'a RollupConfig, + client: &'a EngineClient, + l1: Option, +} + +impl<'a> SyncWalker<'a> { + const fn new(cfg: &'a RollupConfig, client: &'a EngineClient) -> Self { + Self { cfg, client, l1: None } + } + + async fn origin(&mut self, head: L2BlockInfo) -> Result { + if self.l1.is_none_or(|l1| l1.hash != head.l1_origin.hash) { + // A by-hash read is canonical only when it follows the parent of a known L1 block. + let id = if self.l1.is_some_and(|l1| l1.parent_hash == head.l1_origin.hash) { + head.l1_origin.hash.into() + } else { + BlockId::number(head.l1_origin.number) + }; + let block = self.client.get_l1_block(id).await?; + if block.is_none() && matches!(id, BlockId::Hash(_)) { + return Err(SyncStartError::ChainViewChanged); + } + self.l1 = block.map(|block| BlockInfo { + hash: block.header.hash, + number: block.header.number, + parent_hash: block.header.parent_hash, + timestamp: block.header.timestamp, + }); + } + Ok(match self.l1 { + None => OriginStatus::Ahead, + Some(l1) if l1.hash == head.l1_origin.hash => OriginStatus::Canonical, + Some(_) => OriginStatus::Replaced, + }) + } + + async fn parent(&self, child: L2BlockInfo) -> Result { + let id = child.block_info.parent_hash.into(); + let block = + self.client.get_l2_block(id).full().await?.ok_or(SyncStartError::BlockNotFound(id))?; + let parent = + L2BlockInfo::from_block_and_genesis(&block.into_consensus(), &self.cfg.genesis)?; + let canonical_parent = + self.l1.filter(|l1| l1.hash == child.l1_origin.hash).map(|l1| l1.parent_hash); + check_epoch_and_sequence(child, parent, canonical_parent)?; + Ok(parent) + } +} + +// Ported from op-node/rollup/sync/start.go's L1-origin relation check. Compare full origin IDs: +// a same-height hash replacement is an epoch inconsistency, not a same-origin sequence step. +fn check_epoch_and_sequence( + child: L2BlockInfo, + parent: L2BlockInfo, + canonical_parent: Option, +) -> Result<(), SyncStartError> { + if parent.l1_origin != child.l1_origin { + if parent.l1_origin.number.checked_add(1) != Some(child.l1_origin.number) { + return Err(SyncStartError::L1OriginMismatch); + } + if child.seq_num != 0 { + return Err(SyncStartError::NonZeroSequenceNumber); + } + if canonical_parent.is_some_and(|hash| hash != parent.l1_origin.hash) { + return Err(SyncStartError::L1OriginMismatch); + } + } else if parent.seq_num.checked_add(1) != Some(child.seq_num) { + return Err(SyncStartError::InconsistentSequenceNumber); + } + Ok(()) } #[cfg(test)] @@ -146,3 +326,6 @@ mod test { assert_eq!(rpc_reported_hash, l2_block_info.block_info.hash); } } + +#[cfg(test)] +mod tests; diff --git a/rust/kona/crates/node/engine/src/sync/tests.rs b/rust/kona/crates/node/engine/src/sync/tests.rs new file mode 100644 index 00000000000..44357e49021 --- /dev/null +++ b/rust/kona/crates/node/engine/src/sync/tests.rs @@ -0,0 +1,551 @@ +//! Regression vectors from op-node/rollup/sync/start_test.go, plus malformed traversal edges. + +use super::*; +use crate::test_utils::{RpcMock, test_engine_client, test_l1_hash, test_l2_block}; +use alloy_rpc_types_eth::Block; +use serde_json::json; +use std::sync::Arc; + +type L1Block = Block; +type L2Block = Block; + +fn l1_block(number: u64) -> L1Block { + let mut block = L1Block::default(); + block.header.inner.number = number; + block.header.hash = test_l1_hash(number); + block.header.inner.parent_hash = test_l1_hash(number.saturating_sub(1)); + block +} + +fn info(block: &L2Block, cfg: &RollupConfig) -> L2BlockInfo { + L2BlockInfo::from_block_and_genesis(&block.clone().into_consensus(), &cfg.genesis).unwrap() +} + +fn expect_heads(mock: &RpcMock, finalized: &L2Block, safe: &L2Block, head: &L2Block) { + for (tag, block) in [("finalized", finalized), ("safe", safe), ("latest", head)] { + mock.expect_params("eth_getBlockByNumber", json!([tag, true]), block); + } +} + +fn expect_parent(mock: &RpcMock, parent: &L2Block) { + mock.expect_params("eth_getBlockByHash", json!([parent.header.hash, true]), parent); +} + +/// Two-second blocks, so op-node's default 12h offset is 21,600 blocks. +const BLOCK_TIME: u64 = 2; +const DEFAULT_OFFSET_BLOCKS: u64 = 21_600; + +// Old chains with missing labels must recover in constant time, with safe and finalized placed +// the EL-sync offset behind the tip; recent chains still derive from genesis. Non-zero genesis +// heights exercise the threshold relative to chain genesis. +#[rstest::rstest] +#[case::genesis(0, false)] +#[case::recent_chain(1, false)] +#[case::at_recovery_threshold(28, false)] +#[case::past_recovery_threshold(29, false)] +#[case::distant_unsafe_head(100_000_000, false)] +#[case::missing_labels(100_000_000, true)] +#[tokio::test] +async fn recovers_el_sync_with_genesis_labels(#[case] distance: u64, #[case] missing_labels: bool) { + let genesis = test_l2_block(100, 50, 0, B256::ZERO); + let mut cfg = RollupConfig { seq_window_size: 2, block_time: BLOCK_TIME, ..Default::default() }; + cfg.genesis.l1 = alloy_eips::BlockNumHash::new(50, l1_block(50).header.hash); + cfg.genesis.l2 = alloy_eips::BlockNumHash::new(100, genesis.header.hash); + let mut blocks = vec![genesis.clone()]; + if distance <= 28 { + for i in 1..=distance { + blocks.push(test_l2_block(100 + i, 50 + i, 0, blocks.last().unwrap().header.hash)); + } + } else { + blocks.push(test_l2_block(100 + distance, 50 + distance, 0, B256::ZERO)); + } + let head = blocks.last().unwrap(); + let (client, l1, l2) = test_engine_client(Arc::new(cfg.clone())); + if missing_labels { + l2.expect_params( + "eth_getBlockByNumber", + json!(["finalized", true]), + Option::::None, + ); + l2.expect_params("eth_getBlockByNumber", json!(["0x64", true]), &genesis); + l2.expect_params("eth_getBlockByNumber", json!(["safe", true]), Option::::None); + l2.expect_params("eth_getBlockByNumber", json!(["latest", true]), head); + } else { + expect_heads(&l2, &genesis, &genesis, head); + } + let derived = if distance <= 28 { + l1.expect_params( + "eth_getBlockByNumber", + json!([format!("{:#x}", 50 + distance), false]), + l1_block(50 + distance), + ); + for i in (0..distance).rev() { + expect_parent(&l2, &blocks[i as usize]); + l1.expect_params( + "eth_getBlockByHash", + json!([l1_block(50 + i).header.hash, false]), + l1_block(50 + i), + ); + } + genesis.clone() + } else { + // A chain younger than the offset clamps at genesis. + let number = (100 + distance).saturating_sub(DEFAULT_OFFSET_BLOCKS).max(100); + let offset_head = if number == 100 { + genesis.clone() + } else { + test_l2_block(number, number - 50, 0, B256::ZERO) + }; + l2.expect_params( + "eth_getBlockByNumber", + json!([format!("{number:#x}"), true]), + &offset_head, + ); + offset_head + }; + let result = + find_starting_forkchoice(&cfg, &client, DEFAULT_EL_SYNC_SAFE_OFFSET).await.unwrap(); + assert_eq!(result.un_safe, info(head, &cfg)); + assert_eq!(result.safe, info(&derived, &cfg)); + assert_eq!(result.finalized, info(&derived, &cfg)); + l1.assert_finished(); + l2.assert_finished(); +} + +// Like op-node, an execution layer rewound below its safe or finalized label keeps its unsafe +// tip, and safe and finalized are placed the EL-sync offset behind that tip. That can move +// finalized up when it was below the tip, as in `behind_safe`. +#[rstest::rstest] +#[case::behind_safe(0, 20, 10, Some(13))] +#[case::behind_finalized(20, 17, 10, Some(13))] +#[case::behind_finalized_and_safe(20, 21, 10, Some(13))] +#[case::offset_clamped_at_genesis(20, 21, 3_600, Some(0))] +#[case::zero_offset(20, 21, 0, None)] +#[tokio::test] +async fn recovers_rewound_el_labels( + #[case] finalized_number: u64, + #[case] safe_number: u64, + #[case] offset_seconds: u64, + #[case] expected_number: Option, +) { + let genesis = test_l2_block(0, 0, 0, B256::ZERO); + let mut cfg = RollupConfig { block_time: BLOCK_TIME, ..Default::default() }; + cfg.genesis.l2.hash = genesis.header.hash; + let finalized = if finalized_number == 0 { + genesis.clone() + } else { + test_l2_block(finalized_number, 100, 0, B256::ZERO) + }; + let safe = test_l2_block(safe_number, 101, 0, B256::ZERO); + let head = test_l2_block(18, 99, 0, B256::ZERO); + let (client, l1, l2) = test_engine_client(Arc::new(cfg.clone())); + expect_heads(&l2, &finalized, &safe, &head); + let expected = match expected_number { + Some(0) => genesis.clone(), + Some(number) => test_l2_block(number, 98, 0, B256::ZERO), + None => head.clone(), + }; + if let Some(number) = expected_number { + l2.expect_params("eth_getBlockByNumber", json!([format!("{number:#x}"), true]), &expected); + } + let result = + find_starting_forkchoice(&cfg, &client, Duration::from_secs(offset_seconds)).await.unwrap(); + let expected = info(&expected, &cfg); + assert_eq!( + result, + L2ForkchoiceState { un_safe: info(&head, &cfg), safe: expected, finalized: expected } + ); + l1.assert_finished(); + l2.assert_finished(); +} + +// Mirrors op-node's OffsetBlockNum: ceiling division by the block time, clamped at genesis. +#[rstest::rstest] +#[case::zero_offset(0, 2, 100, 0, 100)] +#[case::rounds_up(3_000, 2, 100, 0, 98)] +#[case::whole_blocks(4_000, 2, 100, 0, 98)] +#[case::truncates_sub_second(1_500, 2, 100, 0, 99)] +#[case::clamped_at_genesis(1_000_000, 2, 105, 100, 100)] +#[case::head_at_genesis(10_000, 2, 100, 100, 100)] +#[case::zero_block_time(10_000, 0, 100, 0, 100)] +fn offset_block_number_matches_op_node( + #[case] offset_millis: u64, + #[case] block_time: u64, + #[case] head: u64, + #[case] genesis: u64, + #[case] expected: u64, +) { + let offset = Duration::from_millis(offset_millis); + assert_eq!(offset_block_number(offset, block_time, head, genesis), expected); +} + +#[rstest::rstest] +#[case::only_safe_established(0)] +#[case::finality_established(1)] +#[tokio::test] +async fn established_labels_keep_the_normal_recovery_walk(#[case] finalized: usize) { + let mut blocks = vec![]; + for number in 0..30 { + let parent = blocks.last().map_or(B256::ZERO, |block: &L2Block| block.header.hash); + blocks.push(test_l2_block(number, number, 0, parent)); + } + let mut cfg = RollupConfig { seq_window_size: 2, ..Default::default() }; + cfg.genesis.l2.hash = blocks[0].header.hash; + let (client, l1, l2) = test_engine_client(Arc::new(cfg.clone())); + expect_heads(&l2, &blocks[finalized], &blocks[1], &blocks[29]); + l1.expect_params("eth_getBlockByNumber", json!(["0x1d", false]), l1_block(29)); + for number in (1..29).rev() { + l1.expect_params( + "eth_getBlockByHash", + json!([l1_block(number).header.hash, false]), + l1_block(number), + ); + } + for parent in (finalized..29).rev() { + expect_parent(&l2, &blocks[parent]); + } + let result = + find_starting_forkchoice(&cfg, &client, DEFAULT_EL_SYNC_SAFE_OFFSET).await.unwrap(); + assert_eq!(result.un_safe, info(&blocks[29], &cfg)); + assert_eq!(result.safe, info(&blocks[finalized], &cfg)); + assert_eq!(result.finalized, info(&blocks[finalized], &cfg)); + l1.assert_finished(); + l2.assert_finished(); +} + +// The walk stops at the finalized head and never passes it, even when every newer origin was +// replaced, and a conflicting finalized block is fatal rather than retried. +#[rstest::rstest] +#[case::walk_stops_at_finality(22, false)] +#[case::already_finalized(20, false)] +#[case::conflicting_finalized(20, true)] +#[tokio::test] +async fn unsafe_walk_never_passes_finality(#[case] number: u64, #[case] conflict: bool) { + let config = Arc::new(RollupConfig { seq_window_size: 3600, ..Default::default() }); + let (client, l1, l2) = test_engine_client(config.clone()); + let finalized = test_l2_block(20, 99, 0, B256::ZERO); + let parent = test_l2_block(21, 100, 0, finalized.header.hash); + let mut head = if number == 20 { + finalized.clone() + } else { + test_l2_block(number, 100, 1, parent.header.hash) + }; + if number == 22 { + head.header.inner.parent_hash = parent.header.hash; + } + if conflict { + head.header.inner.timestamp += 1; + } + head.header.hash = head.header.inner.hash_slow(); + for (tag, block) in [("finalized", &finalized), ("safe", &finalized), ("latest", &head)] { + l2.expect_params("eth_getBlockByNumber", serde_json::json!([tag, true]), block); + } + if number == 22 { + let mut replaced = + alloy_rpc_types_eth::Block::::default(); + replaced.header.hash = B256::ZERO; + for _ in 0..2 { + l1.expect_params("eth_getBlockByNumber", serde_json::json!(["0x64", false]), &replaced); + } + for block in [&parent, &finalized] { + l2.expect_params( + "eth_getBlockByHash", + serde_json::json!([block.header.hash, true]), + block, + ); + } + } + if number >= 20 && !conflict { + let mut origin = alloy_rpc_types_eth::Block::::default(); + origin.header.hash = test_l1_hash(99); + origin.header.inner.number = 99; + l1.expect_params("eth_getBlockByNumber", serde_json::json!(["0x63", false]), origin); + } + let result = find_starting_forkchoice(&config, &client, DEFAULT_EL_SYNC_SAFE_OFFSET).await; + if conflict { + let err = result.unwrap_err(); + assert!(matches!(err, SyncStartError::MismatchedFinalizedBlock(..))); + assert!(!crate::EngineResetError::SyncStart(err).is_temporary()); + } else { + let heads = result.unwrap(); + assert_eq!(heads.un_safe, heads.finalized); + assert_eq!(heads.safe, heads.finalized); + } + l1.assert_finished(); + l2.assert_finished(); +} + +// An L1 reorg below the finalized block's origin cannot be recovered from: it is reported as +// such, not as a generic origin mismatch, and is not retried. +#[tokio::test] +async fn replaced_finalized_origin_is_fatal() { + let cfg = RollupConfig { seq_window_size: 3600, ..Default::default() }; + let finalized = test_l2_block(1, 99, 0, B256::ZERO); + let head = test_l2_block(2, 100, 0, finalized.header.hash); + let (client, l1, l2) = test_engine_client(Arc::new(cfg.clone())); + expect_heads(&l2, &finalized, &finalized, &head); + // Replaced blocks, so the walk reads each origin by number. + for number in [100u64, 99] { + let mut replaced = l1_block(number); + replaced.header.hash = B256::repeat_byte(0xff); + replaced.header.inner.parent_hash = B256::repeat_byte(0xee); + l1.expect_params("eth_getBlockByNumber", json!([format!("{number:#x}"), false]), replaced); + } + expect_parent(&l2, &finalized); + let err = + find_starting_forkchoice(&cfg, &client, DEFAULT_EL_SYNC_SAFE_OFFSET).await.unwrap_err(); + assert!(matches!(err, SyncStartError::FinalizedL1OriginReplaced), "{err:?}"); + assert!(!crate::EngineResetError::SyncStart(err).is_temporary()); + l1.assert_finished(); + l2.assert_finished(); +} + +#[tokio::test] +async fn missing_l2_parent_can_be_retried() { + let cfg = RollupConfig::default(); + let finalized = test_l2_block(1, 99, 0, B256::ZERO); + let head = test_l2_block(2, 100, 0, finalized.header.hash); + let (client, l1, l2) = test_engine_client(Arc::new(cfg.clone())); + expect_heads(&l2, &finalized, &finalized, &head); + l1.expect_params("eth_getBlockByNumber", json!(["0x64", false]), l1_block(100)); + l2.expect_params( + "eth_getBlockByHash", + json!([finalized.header.hash, true]), + Option::::None, + ); + let err = + find_starting_forkchoice(&cfg, &client, DEFAULT_EL_SYNC_SAFE_OFFSET).await.unwrap_err(); + assert!(matches!(err, SyncStartError::BlockNotFound(_))); + assert!(crate::EngineResetError::SyncStart(err).is_temporary()); + + expect_heads(&l2, &finalized, &finalized, &head); + l1.expect_params("eth_getBlockByNumber", json!(["0x64", false]), l1_block(100)); + expect_parent(&l2, &finalized); + l1.expect_params("eth_getBlockByHash", json!([l1_block(99).header.hash, false]), l1_block(99)); + let result = + find_starting_forkchoice(&cfg, &client, DEFAULT_EL_SYNC_SAFE_OFFSET).await.unwrap(); + assert_eq!(result.un_safe, info(&head, &cfg)); + assert_eq!(result.safe, info(&finalized, &cfg)); + assert_eq!(result.finalized, info(&finalized, &cfg)); + l1.assert_finished(); + l2.assert_finished(); +} + +// Same histories and expected labels as op-node's small-reorg, L1-ahead and L2-ahead vectors. +// The final case keeps the previous safe head below the lookback boundary. +#[rstest::rstest] +#[case::small_reorg(8, 8, Some(7), 1, 7, (6, 2), &[(7,false),(6,true),(5,true),(4,true),(3,true)])] +#[case::l1_ahead(5, 6, None, 0, 3, (4, 0), &[(4,false),(3,true),(2,true),(1,true)])] +#[case::l2_ahead_after_reorg(6, 4, None, 1, 3, (5, 1), &[(5,false),(4,false),(3,false),(2,true),(1,true)])] +#[case::prior_safe_limits_recovery(12, 12, None, 1, 3, (11, 2), &[(11,false),(10,true),(9,true),(8,true),(7,true),(6,true),(5,true),(4,true),(3,true)])] +#[tokio::test] +async fn matches_op_node_sync_start_vectors( + #[case] l2_len: usize, + #[case] l1_len: usize, + #[case] replaced: Option, + #[case] finalized: usize, + #[case] safe: usize, + #[case] expected: (usize, usize), + #[case] reads: &[(usize, bool)], +) { + let (expected_unsafe, expected_safe) = expected; + let mut blocks = vec![]; + for i in 0..l2_len { + let parent = blocks.last().map_or(B256::ZERO, |b: &L2Block| b.header.hash); + blocks.push(test_l2_block(i as u64, i as u64, 0, parent)); + } + let mut cfg = RollupConfig { seq_window_size: 2, ..Default::default() }; + cfg.genesis.l2.hash = blocks[0].header.hash; + let (client, l1, l2) = test_engine_client(Arc::new(cfg.clone())); + expect_heads(&l2, &blocks[finalized], &blocks[safe], blocks.last().unwrap()); + for &(number, by_hash) in reads { + let mut block = (number < l1_len).then(|| l1_block(number as u64)); + if replaced == Some(number) { + block.as_mut().unwrap().header.hash = B256::repeat_byte(0xff); + } + let (method, id) = if by_hash { + ("eth_getBlockByHash", json!(test_l1_hash(number as u64))) + } else { + ("eth_getBlockByNumber", json!(format!("{number:#x}"))) + }; + l1.expect_params(method, json!([id, false]), block); + } + for parent in (expected_safe..l2_len - 1).rev() { + expect_parent(&l2, &blocks[parent]); + } + let result = + find_starting_forkchoice(&cfg, &client, DEFAULT_EL_SYNC_SAFE_OFFSET).await.unwrap(); + assert_eq!(result.un_safe, info(&blocks[expected_unsafe], &cfg)); + assert_eq!(result.safe, info(&blocks[expected_safe], &cfg)); + assert_eq!(result.finalized, info(&blocks[finalized], &cfg)); + l1.assert_finished(); + l2.assert_finished(); +} + +#[tokio::test] +async fn safe_head_is_last_block_before_epoch_boundary() { + let mut blocks = vec![]; + for i in 0..8 { + let parent = blocks.last().map_or(B256::ZERO, |b: &L2Block| b.header.hash); + blocks.push(test_l2_block(i, i / 2, i % 2, parent)); + } + let mut cfg = RollupConfig { seq_window_size: 1, ..Default::default() }; + cfg.genesis.l2.hash = blocks[0].header.hash; + let (client, l1, l2) = test_engine_client(Arc::new(cfg.clone())); + expect_heads(&l2, &blocks[0], &blocks[7], &blocks[7]); + l1.expect_params("eth_getBlockByNumber", json!(["0x3", false]), l1_block(3)); + for number in (1..=2).rev() { + l1.expect_params( + "eth_getBlockByHash", + json!([test_l1_hash(number), false]), + l1_block(number), + ); + } + for parent in (1..7).rev() { + expect_parent(&l2, &blocks[parent]); + } + let result = + find_starting_forkchoice(&cfg, &client, DEFAULT_EL_SYNC_SAFE_OFFSET).await.unwrap(); + assert_eq!(result.un_safe, info(&blocks[7], &cfg)); + assert_eq!(result.safe, info(&blocks[1], &cfg)); + l1.assert_finished(); + l2.assert_finished(); +} + +#[derive(Debug, Clone, Copy)] +enum Path { + ReplacedUnsafe, + Conservative, +} +#[derive(Debug, Clone, Copy)] +enum Invalid { + EpochSkip, + NonzeroEpoch, + Sequence, + ParentHash, +} + +// Both traversal paths call the production RPC client and decode real L1-info deposits. +#[rstest::rstest] +#[case(Path::ReplacedUnsafe, Invalid::EpochSkip)] +#[case(Path::ReplacedUnsafe, Invalid::NonzeroEpoch)] +#[case(Path::ReplacedUnsafe, Invalid::Sequence)] +#[case(Path::Conservative, Invalid::EpochSkip)] +#[case(Path::Conservative, Invalid::NonzeroEpoch)] +#[case(Path::Conservative, Invalid::Sequence)] +#[case(Path::Conservative, Invalid::ParentHash)] +#[tokio::test] +async fn rejects_inconsistent_edges_on_every_path(#[case] path: Path, #[case] invalid: Invalid) { + let cfg = RollupConfig::default(); + let finalized = test_l2_block(1, 99, 0, B256::ZERO); + let parent = test_l2_block(2, 100, 0, finalized.header.hash); + let (origin, seq) = match invalid { + Invalid::EpochSkip => (102, 0), + Invalid::NonzeroEpoch => (101, 1), + Invalid::Sequence => (100, 2), + Invalid::ParentHash => (101, 0), + }; + let child = test_l2_block(3, origin, seq, parent.header.hash); + let mut l1_origin = l1_block(origin); + if matches!(invalid, Invalid::ParentHash) { + l1_origin.header.inner.parent_hash = B256::repeat_byte(0xff); + } + if matches!(path, Path::ReplacedUnsafe) { + l1_origin.header.hash = B256::repeat_byte(0xff); + } + let (client, l1, l2) = test_engine_client(Arc::new(cfg.clone())); + expect_heads(&l2, &finalized, &parent, &child); + l1.expect_params("eth_getBlockByNumber", json!([format!("{origin:#x}"), false]), &l1_origin); + expect_parent(&l2, &parent); + let err = + find_starting_forkchoice(&cfg, &client, DEFAULT_EL_SYNC_SAFE_OFFSET).await.unwrap_err(); + assert!( + match invalid { + Invalid::EpochSkip | Invalid::ParentHash => + matches!(err, SyncStartError::L1OriginMismatch), + Invalid::NonzeroEpoch => matches!(err, SyncStartError::NonZeroSequenceNumber), + Invalid::Sequence => matches!(err, SyncStartError::InconsistentSequenceNumber), + }, + "unexpected error: {err}" + ); + l1.assert_finished(); + l2.assert_finished(); +} + +#[test] +fn epoch_checks_compare_full_ids_and_reject_overflow() { + let mut parent = L2BlockInfo::default(); + let mut child = parent; + child.l1_origin.hash = B256::repeat_byte(1); // Same height, different hash is not a new epoch. + assert!(matches!( + check_epoch_and_sequence(child, parent, None), + Err(SyncStartError::L1OriginMismatch) + )); + parent.l1_origin.number = u64::MAX; + assert!(matches!( + check_epoch_and_sequence(child, parent, None), + Err(SyncStartError::L1OriginMismatch) + )); + child = parent; + parent.seq_num = u64::MAX; + assert!(matches!( + check_epoch_and_sequence(child, parent, None), + Err(SyncStartError::InconsistentSequenceNumber) + )); +} + +#[tokio::test] +async fn missing_l1_parent_retries_from_a_fresh_chain_view() { + let cfg = RollupConfig::default(); + let (client, l1, l2) = test_engine_client(Arc::new(cfg.clone())); + let mut walker = SyncWalker::new(&cfg, &client); + let mut head = L2BlockInfo::default(); + head.l1_origin.number = 101; + head.l1_origin.hash = l1_block(101).header.hash; + l1.expect_params("eth_getBlockByNumber", json!(["0x65", false]), l1_block(101)); + assert_eq!(walker.origin(head).await.unwrap(), OriginStatus::Canonical); + head.l1_origin.number = 100; + head.l1_origin.hash = l1_block(100).header.hash; + l1.expect_params( + "eth_getBlockByHash", + json!([head.l1_origin.hash, false]), + Option::::None, + ); + let err = walker.origin(head).await.unwrap_err(); + assert!(matches!(err, SyncStartError::ChainViewChanged)); + assert!(crate::EngineResetError::SyncStart(err).is_temporary()); + l1.assert_finished(); + l2.assert_finished(); +} + +#[tokio::test] +async fn rejects_reorgs_deeper_than_five_lookback_windows() { + let mut blocks = vec![]; + for number in 0..9 { + let parent = blocks.last().map_or(B256::ZERO, |b: &L2Block| b.header.hash); + blocks.push(test_l2_block(number, number, 0, parent)); + } + let mut cfg = RollupConfig { seq_window_size: 1, ..Default::default() }; + cfg.genesis.l2.hash = blocks[0].header.hash; + let (client, l1, l2) = test_engine_client(Arc::new(cfg.clone())); + expect_heads(&l2, &blocks[0], &blocks[0], &blocks[8]); + for number in (2..=8).rev() { + let mut replacement = l1_block(number); + replacement.header.hash = B256::with_last_byte(number as u8 + 100); + replacement.header.inner.parent_hash = B256::with_last_byte(number as u8 + 99); + l1.expect_params( + "eth_getBlockByNumber", + json!([format!("{number:#x}"), false]), + replacement, + ); + } + for parent in (2..8).rev() { + expect_parent(&l2, &blocks[parent]); + } + let err = + find_starting_forkchoice(&cfg, &client, DEFAULT_EL_SYNC_SAFE_OFFSET).await.unwrap_err(); + assert!(matches!(err, SyncStartError::ReorgTooDeep)); + assert!(!crate::EngineResetError::SyncStart(err).is_temporary()); + l1.assert_finished(); + l2.assert_finished(); +} diff --git a/rust/kona/crates/node/engine/src/task_queue/core.rs b/rust/kona/crates/node/engine/src/task_queue/core.rs index 816c8d0519e..f3f5369d6a9 100644 --- a/rust/kona/crates/node/engine/src/task_queue/core.rs +++ b/rust/kona/crates/node/engine/src/task_queue/core.rs @@ -2,14 +2,14 @@ use super::ExecutableEngineTask; use crate::{ - EngineClient, EngineState, EngineSyncStateUpdate, EngineTask, EngineTaskError, - EngineTaskErrorSeverity, Metrics, SyncStartError, SynchronizeTaskError, + DEFAULT_EL_SYNC_SAFE_OFFSET, EngineClient, EngineState, EngineSyncStateUpdate, EngineTask, + EngineTaskError, EngineTaskErrorSeverity, Metrics, SyncStartError, SynchronizeTaskError, find_starting_forkchoice, task_queue::{EngineTaskErrors, synchronize}, }; use kona_genesis::RollupConfig; use kona_protocol::L2BlockInfo; -use std::{cmp::Ordering, collections::BinaryHeap, sync::Arc}; +use std::{cmp::Ordering, collections::BinaryHeap, sync::Arc, time::Duration}; use thiserror::Error; use tokio::sync::watch::Sender; @@ -63,6 +63,9 @@ pub struct Engine { /// Preserve the task being retried ahead of newly enqueued work: it may have already /// performed part of an Engine API operation. active: Option, + /// How far behind an EL-synced tip a reset places safe and finalized when the execution + /// layer's labels are missing or stale. + el_sync_safe_offset: Duration, } impl Engine { @@ -74,9 +77,17 @@ impl Engine { tasks: BinaryHeap::default(), active: None, next_sequence: 0, + el_sync_safe_offset: DEFAULT_EL_SYNC_SAFE_OFFSET, } } + /// Sets how far behind an EL-synced tip a reset places safe and finalized when the execution + /// layer's labels are missing or stale. Defaults to [`DEFAULT_EL_SYNC_SAFE_OFFSET`]. + pub const fn with_el_sync_safe_offset(mut self, el_sync_safe_offset: Duration) -> Self { + self.el_sync_safe_offset = el_sync_safe_offset; + self + } + /// Returns a reference to the inner [`EngineState`]. pub const fn state(&self) -> &EngineState { &self.state @@ -113,7 +124,8 @@ impl Engine { client: Arc, config: Arc, ) -> Result { - let start = find_starting_forkchoice(&config, client.as_ref()).await?; + let start = + find_starting_forkchoice(&config, client.as_ref(), self.el_sync_safe_offset).await?; // One attempt. The actor retains the reset request and retries transient failures. synchronize( @@ -184,7 +196,11 @@ impl EngineResetError { /// Whether retrying the reset can recover without changing the node configuration. pub fn is_temporary(&self) -> bool { match self { - Self::SyncStart(SyncStartError::RpcError(_)) => true, + Self::SyncStart( + SyncStartError::RpcError(_) | + SyncStartError::ChainViewChanged | + SyncStartError::BlockNotFound(_), + ) => true, Self::SyncStart(_) => false, Self::Forkchoice(err) => err.severity() != EngineTaskErrorSeverity::Critical, } diff --git a/rust/kona/crates/node/engine/src/test_utils/misc.rs b/rust/kona/crates/node/engine/src/test_utils/misc.rs index fd77bf7ccf4..c506503aca5 100644 --- a/rust/kona/crates/node/engine/src/test_utils/misc.rs +++ b/rust/kona/crates/node/engine/src/test_utils/misc.rs @@ -1,6 +1,6 @@ use alloy_eips::BlockNumHash; -use alloy_primitives::B256; -use kona_protocol::{BlockInfo, L2BlockInfo}; +use alloy_primitives::{B256, Sealed, U256}; +use kona_protocol::{BlockInfo, L1BlockInfoBedrock, L2BlockInfo}; /// Helper to create a test `L2BlockInfo` at a specific block number pub fn test_block_info(number: u64) -> L2BlockInfo { @@ -15,3 +15,45 @@ pub fn test_block_info(number: u64) -> L2BlockInfo { seq_num: 0, } } + +/// The hash that reset traversal tests give the L1 block at `number`. Unlike a single byte, it is +/// distinct for every height. +pub fn test_l1_hash(number: u64) -> B256 { + B256::from(U256::from(number)) +} + +/// Builds a full L2 block with a real L1-info deposit for reset traversal tests. +pub fn test_l2_block( + number: u64, + origin: u64, + sequence: u64, + parent: B256, +) -> alloy_rpc_types_eth::Block { + let data = L1BlockInfoBedrock::new( + origin, + 0, + 0, + test_l1_hash(origin), + sequence, + Default::default(), + Default::default(), + Default::default(), + ) + .encode_calldata(); + let transaction = + op_alloy_consensus::OpTxEnvelope::Deposit(Sealed::new(op_alloy_consensus::TxDeposit { + input: data, + ..Default::default() + })); + let transaction = op_alloy_rpc_types::Transaction::from_transaction( + alloy_consensus::transaction::Recovered::new_unchecked(transaction, Default::default()), + Default::default(), + ); + let mut block: alloy_rpc_types_eth::Block = Default::default(); + block.header.inner.number = number; + block.header.inner.parent_hash = parent; + block.header.inner.timestamp = number * 2; + block.header.hash = block.header.inner.hash_slow(); + block.transactions = alloy_rpc_types_eth::BlockTransactions::Full(vec![transaction]); + block +} diff --git a/rust/kona/crates/node/engine/src/test_utils/mod.rs b/rust/kona/crates/node/engine/src/test_utils/mod.rs index a5575963a67..04e3018e7e7 100644 --- a/rust/kona/crates/node/engine/src/test_utils/mod.rs +++ b/rust/kona/crates/node/engine/src/test_utils/mod.rs @@ -8,4 +8,4 @@ mod engine_state; pub use engine_state::TestEngineStateBuilder; mod misc; -pub use misc::test_block_info; +pub use misc::{test_block_info, test_l1_hash, test_l2_block}; diff --git a/rust/kona/crates/node/engine/src/test_utils/rpc.rs b/rust/kona/crates/node/engine/src/test_utils/rpc.rs index be93a9816fe..7d14f1f0927 100644 --- a/rust/kona/crates/node/engine/src/test_utils/rpc.rs +++ b/rust/kona/crates/node/engine/src/test_utils/rpc.rs @@ -21,7 +21,9 @@ use std::{ use tower::Service; /// Queues replies and checks the method and optional parameters of every outgoing RPC request. -/// Unexpected requests panic rather than silently satisfying a different operation's reply. +/// A request takes the first queued reply whose method and parameters match, so tests pin which +/// requests are made without pinning the order of unrelated ones. Unexpected requests panic +/// rather than silently satisfying a different operation's reply. #[derive(Clone, Debug, Default)] pub struct RpcMock { requests: Arc>>, @@ -141,18 +143,22 @@ impl Service for RpcMock { let RequestPacket::Single(ref rpc) = request else { panic!("engine tests expect individual RPC requests"); }; - let expected = self - .requests - .lock() - .unwrap() - .pop_front() - .unwrap_or_else(|| panic!("unexpected RPC request: {}", rpc.method())); - assert_eq!(rpc.method(), expected.method, "wrong RPC method"); - if let Some(params) = expected.params { - let actual: Value = - serde_json::from_str(rpc.params().expect("request has parameters").get()).unwrap(); - assert_eq!(actual, params, "wrong RPC parameters for {}", expected.method); - } + let actual: Option = + rpc.params().map(|params| serde_json::from_str(params.get()).unwrap()); + let expected = { + let mut requests = self.requests.lock().unwrap(); + let index = requests + .iter() + .position(|expected| { + expected.method == rpc.method() && + expected + .params + .as_ref() + .is_none_or(|params| actual.as_ref() == Some(params)) + }) + .unwrap_or_else(|| panic!("unexpected RPC request: {} {actual:?}", rpc.method())); + requests.remove(index).expect("index is in bounds") + }; let replies = Asserter::new(); let Some(reply) = expected.response else { return Box::pin(async { diff --git a/rust/kona/crates/node/service/src/actors/engine/tests.rs b/rust/kona/crates/node/service/src/actors/engine/tests.rs index b61712d95b5..a013ec06928 100644 --- a/rust/kona/crates/node/service/src/actors/engine/tests.rs +++ b/rust/kona/crates/node/service/src/actors/engine/tests.rs @@ -4,7 +4,7 @@ use alloy_primitives::B256; use alloy_rpc_types_engine::{ForkchoiceUpdated, PayloadStatus, PayloadStatusEnum}; use kona_engine::{ Engine, EngineActorRequest, EngineState, EngineSyncStateUpdate, FinalizeBlockId, NoopBlockSink, - test_utils::{TestAttributesBuilder, test_engine_client}, + test_utils::{TestAttributesBuilder, test_engine_client, test_l2_block}, }; use kona_genesis::RollupConfig; use kona_protocol::{BlockInfo, L2BlockInfo}; @@ -86,7 +86,7 @@ async fn invalid_forkchoice_resets_and_resumes(#[case] rebuild: bool, #[case] ho l2.expect_params("eth_getBlockByNumber", serde_json::json!([tag, true]), &block); } l1.expect( - "eth_getBlockByHash", + "eth_getBlockByNumber", alloy_rpc_types_eth::Block::::default(), ); l2.expect( @@ -141,29 +141,55 @@ async fn invalid_forkchoice_resets_and_resumes(#[case] rebuild: bool, #[case] ho tasks.abort_all(); } +#[derive(Debug, Clone, Copy)] +enum ResetFailure { + L1, + L2, + L2Parent, +} + /// Both L1 and L2 outages during a reset must retain the original request and finish it /// after recovery, without a new reset request or L1 head to wake the actor. #[rstest::rstest] -#[case(true, Duration::ZERO)] -#[case(false, Duration::ZERO)] -#[case(false, Duration::from_secs(4))] +#[case::l1(ResetFailure::L1, Duration::ZERO)] +#[case::l2(ResetFailure::L2, Duration::ZERO)] +#[case::slow_l2(ResetFailure::L2, Duration::from_secs(4))] +#[case::missing_l2_parent(ResetFailure::L2Parent, Duration::ZERO)] #[tokio::test(start_paused = true)] async fn admin_reset_recovers_and_completes_original_request( - #[case] l1_failure: bool, + #[case] failure: ResetFailure, #[case] read_delay: Duration, ) { - let block: ::BlockResponse = - Default::default(); + let block = test_l2_block(0, 0, 0, B256::ZERO); let mut config = RollupConfig::default(); - config.genesis.l2.hash = block.header.inner.hash_slow(); + config.genesis.l2.hash = block.header.hash; + // Keep an unsafe block above genesis so both the L1 origin check and the safe walk run. + let unsafe_block = test_l2_block(1, 0, 1, config.genesis.l2.hash); + let config = Arc::new(config); let (client, l1, l2) = test_engine_client(config.clone()); let client = Arc::new(client); - if l1_failure { + if matches!(failure, ResetFailure::L1 | ResetFailure::L2Parent) { for tag in ["finalized", "safe", "latest"] { - l2.expect_params("eth_getBlockByNumber", serde_json::json!([tag, true]), &block); + l2.expect_params( + "eth_getBlockByNumber", + serde_json::json!([tag, true]), + if tag == "latest" { &unsafe_block } else { &block }, + ); + } + if matches!(failure, ResetFailure::L1) { + l1.expect_error("eth_getBlockByNumber"); + } else { + l1.expect( + "eth_getBlockByNumber", + alloy_rpc_types_eth::Block::::default(), + ); + l2.expect_params( + "eth_getBlockByHash", + serde_json::json!([block.header.hash, true]), + serde_json::Value::Null, + ); } - l1.expect_error("eth_getBlockByHash"); } else { l2.expect_error("eth_getBlockByNumber"); } @@ -180,7 +206,7 @@ async fn admin_reset_recovers_and_completes_original_request( tokio::sync::watch::channel(kona_protocol::L2BlockInfo::default()); let mut actor = EngineActor::new( client, - config, + config.clone(), derivation, Engine::new(EngineState::default(), state_tx), Some(unsafe_head_tx), @@ -193,11 +219,16 @@ async fn admin_reset_recovers_and_completes_original_request( actor.step().await.unwrap(); // retain the reset actor.step().await.unwrap(); // failed attempt, then wait for backoff assert!(!reset.is_finished()); - for _ in 0..3 { - l2.expect_with_delay("eth_getBlockByNumber", &block, read_delay); + for response in [&block, &block, &unsafe_block] { + l2.expect_with_delay("eth_getBlockByNumber", response, read_delay); } - l1.expect( + l2.expect_params( "eth_getBlockByHash", + serde_json::json!([config.genesis.l2.hash, true]), + &block, + ); + l1.expect( + "eth_getBlockByNumber", alloy_rpc_types_eth::Block::::default(), ); l2.expect( @@ -218,14 +249,14 @@ async fn admin_reset_recovers_and_completes_original_request( // when stopped sequencing leaves the engine task queue empty. let published = *state_rx.borrow(); for head in [ - published.sync_state.unsafe_head(), published.sync_state.local_safe_head(), published.sync_state.safe_head(), published.sync_state.finalized_head(), ] { assert_eq!(head.hash(), block.header.inner.hash_slow()); } - assert_eq!(unsafe_head_rx.borrow().hash(), block.header.inner.hash_slow()); + assert_eq!(published.sync_state.unsafe_head().hash(), unsafe_block.header.hash); + assert_eq!(unsafe_head_rx.borrow().hash(), unsafe_block.header.hash); assert!(matches!( derivation_rx.recv().await.unwrap(), DerivationActorRequest::ProcessL1HeadUpdateRequest(_) @@ -250,6 +281,95 @@ async fn admin_reset_recovers_and_completes_original_request( l2.assert_finished(); } +/// EL-sync completion must bootstrap derivation without reading the synced chain's ancestry. Like +/// op-node, safe and finalized are placed the EL-sync offset behind the tip, which arrived over +/// P2P and may not be on L1 yet. +#[tokio::test(start_paused = true)] +async fn el_sync_completion_recovers_genesis_labels_without_a_historical_walk() { + let genesis = test_l2_block(0, 0, 0, B256::ZERO); + let mut config = RollupConfig { block_time: 2, ..Default::default() }; + config.genesis.l2.hash = genesis.header.hash; + let tip = test_l2_block(100_000_000, 14 * config.seq_window_size + 1, 0, B256::ZERO); + let tip_info = + L2BlockInfo::from_block_and_genesis(&tip.clone().into_consensus(), &config.genesis) + .unwrap(); + // The default 12h offset is 21,600 two-second blocks. + let offset_number = 100_000_000 - 21_600; + let offset_head = test_l2_block(offset_number, 14 * config.seq_window_size, 0, B256::ZERO); + let offset_info = + L2BlockInfo::from_block_and_genesis(&offset_head.clone().into_consensus(), &config.genesis) + .unwrap(); + let config = Arc::new(config); + let (client, l1, l2) = test_engine_client(config.clone()); + let state = EngineState { + sync_state: EngineState::default().sync_state.apply_update(EngineSyncStateUpdate { + unsafe_head: Some(tip_info), + ..Default::default() + }), + el_sync_finished: true, + ..Default::default() + }; + let (state_tx, state_rx) = tokio::sync::watch::channel(state); + let (requests, request_rx) = mpsc::channel(1); + let (derivation_tx, mut derivation_rx) = mpsc::channel(2); + let mut actor = EngineActor::new( + Arc::new(client), + config, + QueuedEngineDerivationClient { derivation_actor_request_tx: derivation_tx }, + Engine::new(state, state_tx), + None, + request_rx, + Arc::new(NoopBlockSink), + ); + // The completed EL sync with an unset finalized head schedules its own reset. + actor.step().await.unwrap(); + for (tag, block) in [("finalized", &genesis), ("safe", &genesis), ("latest", &tip)] { + l2.expect_params("eth_getBlockByNumber", serde_json::json!([tag, true]), block); + } + l2.expect_params( + "eth_getBlockByNumber", + serde_json::json!([format!("{offset_number:#x}"), true]), + &offset_head, + ); + l2.expect_params( + "engine_forkchoiceUpdatedV3", + serde_json::json!([ + alloy_rpc_types_engine::ForkchoiceState { + head_block_hash: tip_info.hash(), + safe_block_hash: offset_info.hash(), + finalized_block_hash: offset_info.hash(), + }, + null + ]), + ForkchoiceUpdated::new(PayloadStatus::from_status(PayloadStatusEnum::Valid)), + ); + // Closing input lets the step return after recovery and its notifications have completed. + drop(requests); + assert!(matches!(actor.step().await, Err(crate::EngineError::ChannelClosed))); + let published = *state_rx.borrow(); + assert_eq!(published.sync_state.unsafe_head(), tip_info); + assert_eq!(published.sync_state.local_safe_head(), offset_info); + assert_eq!(published.sync_state.safe_head(), offset_info); + assert_eq!(published.sync_state.finalized_head(), offset_info); + let DerivationActorRequest::ProcessEngineSignalRequest(signal) = + derivation_rx.recv().await.unwrap() + else { + panic!("expected the reset head"); + }; + assert!( + matches!(*signal, kona_derive::Signal::Reset(reset) if reset.l2_safe_head == offset_info) + ); + let DerivationActorRequest::ProcessEngineSyncCompletionRequest(head) = + derivation_rx.recv().await.unwrap() + else { + panic!("expected EL-sync completion after reset"); + }; + assert_eq!(*head, offset_info); + assert!(derivation_rx.try_recv().is_err()); + l1.assert_finished(); + l2.assert_finished(); +} + fn l2_block(number: u64) -> L2BlockInfo { L2BlockInfo { block_info: BlockInfo { diff --git a/rust/kona/crates/node/service/src/service/builder.rs b/rust/kona/crates/node/service/src/service/builder.rs index 8400aec1f6f..3bd4a4fb514 100644 --- a/rust/kona/crates/node/service/src/service/builder.rs +++ b/rust/kona/crates/node/service/src/service/builder.rs @@ -13,10 +13,11 @@ use alloy_transport_http::{ }; use http_body_util::Full; use op_alloy_network::Optimism; -use std::sync::Arc; +use std::{sync::Arc, time::Duration}; use tower::ServiceBuilder; use url::Url; +use kona_engine::DEFAULT_EL_SYNC_SAFE_OFFSET; use kona_genesis::{L1ChainConfig, RollupConfig}; use kona_interop::DependencySet; use kona_providers_alloy::OnlineBeaconClient; @@ -82,6 +83,9 @@ pub struct RollupNodeBuilder { pub safe_db: Option>, /// Number of L1 confirmations required before deriving L2 blocks. pub verifier_l1_confs: u64, + /// How far behind an EL-synced tip a reset places safe and finalized when the execution + /// layer's labels are missing or stale. + pub el_sync_safe_offset: Duration, } impl RollupNodeBuilder { @@ -108,6 +112,7 @@ impl RollupNodeBuilder { dependency_set: None, safe_db: None, verifier_l1_confs: 0, + el_sync_safe_offset: DEFAULT_EL_SYNC_SAFE_OFFSET, } } @@ -130,6 +135,12 @@ impl RollupNodeBuilder { Self { verifier_l1_confs, ..self } } + /// Sets how far behind an EL-synced tip a reset places safe and finalized when the execution + /// layer's labels are missing or stale. + pub fn with_el_sync_safe_offset(self, el_sync_safe_offset: Duration) -> Self { + Self { el_sync_safe_offset, ..self } + } + /// Sets the [`EngineConfig`] on the [`RollupNodeBuilder`]. pub fn with_engine_config(self, engine_config: EngineConfig) -> Self { Self { engine_config, ..self } @@ -222,6 +233,7 @@ impl RollupNodeBuilder { dependency_set: self.dependency_set, safe_db: self.safe_db, verifier_l1_confs: self.verifier_l1_confs, + el_sync_safe_offset: self.el_sync_safe_offset, } } } diff --git a/rust/kona/crates/node/service/src/service/node/rollup.rs b/rust/kona/crates/node/service/src/service/node/rollup.rs index d1b5fb0a12e..786852a991b 100644 --- a/rust/kona/crates/node/service/src/service/node/rollup.rs +++ b/rust/kona/crates/node/service/src/service/node/rollup.rs @@ -119,6 +119,8 @@ pub struct RollupNode { pub(crate) safe_db: Option>, /// L1 confirmation depth for online derivation, independent of sequencing. pub(crate) verifier_l1_confs: u64, + /// How far behind an EL-synced tip a reset places safe and finalized. + pub(crate) el_sync_safe_offset: Duration, } /// A RollupNode-level derivation actor wrapper. @@ -262,7 +264,8 @@ impl RollupNode { // Share engine state with RPC without routing reads through the actor. let engine_state = EngineState::default(); let (engine_state_tx, engine_state_rx) = watch::channel(engine_state); - let engine = Engine::new(engine_state, engine_state_tx); + let engine = Engine::new(engine_state, engine_state_tx) + .with_el_sync_safe_offset(self.el_sync_safe_offset); let engine_client = Arc::new(self.engine_config.clone().build_engine_client()); From ae5aef3490a95172eda057dae82232e5ce8abf22 Mon Sep 17 00:00:00 2001 From: Josh Klopfenstein Date: Mon, 2 Nov 2026 20:24:19 -0800 Subject: [PATCH 2/2] op-acceptance-tests: run reorg recovery tests on kona TestUnsafeGapFillAfterUnsafeReorg_RestartL2CL and _RestartCLP2P failed on kona-node because a restarted verifier kept unsafe blocks built on a replaced L1 origin. The op-node FindL2Heads port in this branch drops them, and both tests now pass on kona-node. TestUnsafeGapFillAfterSafeReorg still fails on kona-node and keeps its skip. Co-Authored-By: Claude Opus 5.5 --- .../tests/sync/elsync/reorg/sync_test.go | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/op-acceptance-tests/tests/sync/elsync/reorg/sync_test.go b/op-acceptance-tests/tests/sync/elsync/reorg/sync_test.go index 3dad7ddbfcc..09644136d2a 100644 --- a/op-acceptance-tests/tests/sync/elsync/reorg/sync_test.go +++ b/op-acceptance-tests/tests/sync/elsync/reorg/sync_test.go @@ -115,15 +115,6 @@ func TestUnsafeGapFillAfterSafeReorg(gt *testing.T) { // 4. Verifier then backfills and closes the unsafe gap once reconnected via CLP2P. func TestUnsafeGapFillAfterUnsafeReorg_RestartL2CL(gt *testing.T) { t := devtest.ParallelT(gt) - // Example error with kona-node: - // - // assertions.go:387: ERROR[03-30|22:17:07.231] - // assertions.go:387: Error Trace: /optimism/op-devstack/dsl/l2_el.go:204 - // assertions.go:387: /optimism/op-acceptance-tests/tests/sync/elsync/reorg/sync_test.go:211 - // assertions.go:387: Error: Received unexpected error: - // assertions.go:387: operation failed permanently after 30 attempts: expected head to reorg 0x893d77533b0ff9b37a92090679bf256d987b4535f06186ec71f29e68ddccd9a5:14, but got 0x893d77533b0ff9b37a92090679bf256d987b4535f06186ec71f29e68ddccd9a5:14 - // assertions.go:387: Test: TestUnsafeGapFillAfterUnsafeReorg_RestartL2CL - sysgo.SkipOnKonaNode(t, "not supported (timeout)") sys := newReorgSystem(t) require := t.Require() logger := t.Logger() @@ -259,15 +250,6 @@ func TestUnsafeGapFillAfterUnsafeReorg_RestartL2CL(gt *testing.T) { // 4. CLP2P is restored Verifier, the verifier backfills and the unsafe gap is closed. func TestUnsafeGapFillAfterUnsafeReorg_RestartCLP2P(gt *testing.T) { t := devtest.ParallelT(gt) - // Example error with kona-node: - // - // assertions.go:387: ERROR[03-31|11:15:39.398] - // assertions.go:387: Error Trace: /optimism/op-devstack/dsl/l2_el.go:204 - // assertions.go:387: /optimism/op-acceptance-tests/tests/sync/elsync/reorg/sync_test.go:356 - // assertions.go:387: Error: Received unexpected error: - // assertions.go:387: operation failed permanently after 30 attempts: expected head to reorg 0x166970054ad16ad090210e5d1045538eeccd2afd88ea991b010de026d0106870:18, but got 0x166970054ad16ad090210e5d1045538eeccd2afd88ea991b010de026d0106870:18 - // assertions.go:387: Test: TestUnsafeGapFillAfterUnsafeReorg_RestartCLP2P - sysgo.SkipOnKonaNode(t, "not supported (timeout)") sys := newReorgSystem(t) require := t.Require() logger := t.Logger()