From 383feb948396be186db3fdf2bef86d46d45804f5 Mon Sep 17 00:00:00 2001 From: Teddy Knox Date: Sat, 24 Oct 2026 16:14:59 +0200 Subject: [PATCH 1/8] kona-node: keep the safe head across L1 reorgs that leave its L1 data canonical Every L1 reorg that derivation detected triggered a full engine reset, which re-runs find_starting_forkchoice and moves the safe head back a full sequencing window, stopping early only at the finalized head. A one-block replacement of the L1 tip cost as much as a cold start, and while L1 finality was stalled it rewound the safe head ~72k L2 blocks (ethereum-optimism/protocol-team#328). Derivation now records its recent safe heads, each with an L1 block that all of the L1 data the safe head depends on is at or below, and sends them with the reset request. The engine keeps the newest one whose L1 block is still canonical, and falls back to the full reset when none qualifies, when it is off the engine's chain, or when the unsafe head's L1 origin was reorged out. A tip reorg usually keeps the current safe head with no forkchoice update. Co-Authored-By: Claude Opus 5.5 --- rust/kona/crates/node/engine/src/lib.rs | 9 +- .../crates/node/engine/src/metrics/mod.rs | 10 + rust/kona/crates/node/engine/src/request.rs | 46 ++- rust/kona/crates/node/engine/src/sync/mod.rs | 3 + .../kona/crates/node/engine/src/sync/reorg.rs | 100 ++++++ .../crates/node/engine/src/task_queue/core.rs | 297 +++++++++++++++++- rust/kona/crates/node/rpc/src/admin.rs | 7 +- .../service/src/actors/derivation/actor.rs | 69 +++- .../src/actors/derivation/actor/tests.rs | 166 +++++++++- .../src/actors/derivation/engine_client.rs | 33 +- .../node/service/src/actors/derivation/mod.rs | 3 + .../actors/derivation/safe_head_history.rs | 195 ++++++++++++ .../src/actors/derivation/state_machine.rs | 2 +- .../node/service/src/actors/engine/actor.rs | 57 +++- .../node/service/src/actors/engine/mod.rs | 2 +- .../node/service/src/actors/engine/tests.rs | 93 +++++- .../crates/node/service/src/actors/mod.rs | 4 +- .../src/actors/sequencer/engine_client.rs | 7 +- rust/kona/crates/node/service/src/lib.rs | 6 +- 19 files changed, 1041 insertions(+), 68 deletions(-) create mode 100644 rust/kona/crates/node/engine/src/sync/reorg.rs create mode 100644 rust/kona/crates/node/service/src/actors/derivation/safe_head_history.rs diff --git a/rust/kona/crates/node/engine/src/lib.rs b/rust/kona/crates/node/engine/src/lib.rs index eff4de433bd..45eae8c70d7 100644 --- a/rust/kona/crates/node/engine/src/lib.rs +++ b/rust/kona/crates/node/engine/src/lib.rs @@ -57,8 +57,8 @@ pub use client::{EngineClient, EngineClientBuilder, EngineClientError, EngineQue mod request; pub use request::{ - BuildRequest, EngineActorRequest, EngineRequestError, EngineRequestResult, ResetRequest, - SealRequest, + BuildRequest, EngineActorRequest, EngineRequestError, EngineRequestResult, ResetKind, + ResetRequest, SealRequest, }; mod versions; @@ -74,7 +74,10 @@ mod metrics; pub use metrics::Metrics; mod sync; -pub use sync::{L2ForkchoiceState, SyncStartError, find_starting_forkchoice}; +pub use sync::{ + DerivedSafeHead, L2ForkchoiceState, SyncStartError, find_reorg_safe_head, + find_starting_forkchoice, +}; #[cfg(any(test, feature = "test-utils"))] /// Utilities that are useful when creating unit tests using structs within this library. diff --git a/rust/kona/crates/node/engine/src/metrics/mod.rs b/rust/kona/crates/node/engine/src/metrics/mod.rs index f7596bb5675..5d398fbd711 100644 --- a/rust/kona/crates/node/engine/src/metrics/mod.rs +++ b/rust/kona/crates/node/engine/src/metrics/mod.rs @@ -67,6 +67,10 @@ impl Metrics { /// Identifier for the counter that tracks the number of times the engine has been reset. pub const ENGINE_RESET_COUNT: &str = "kona_node_engine_reset_count"; + /// Identifier for the counter that tracks the number of L1 reorgs the engine answered by + /// keeping a safe head derived from canonical L1 data, instead of a full reset. + pub const ENGINE_REORG_RESET_COUNT: &str = "kona_node_engine_reorg_reset_count"; + /// Initializes metrics for the engine. /// /// This does two things: @@ -99,6 +103,11 @@ impl Metrics { metrics::Unit::Count, "Engine reset count" ); + metrics::describe_counter!( + Self::ENGINE_REORG_RESET_COUNT, + metrics::Unit::Count, + "L1 reorgs answered without a full engine reset" + ); } /// Initializes metrics to `0` so they can be queried immediately by consumers of prometheus @@ -121,5 +130,6 @@ impl Metrics { // Engine reset count metrics::counter!(Self::ENGINE_RESET_COUNT).absolute(0); + metrics::counter!(Self::ENGINE_REORG_RESET_COUNT).absolute(0); } } diff --git a/rust/kona/crates/node/engine/src/request.rs b/rust/kona/crates/node/engine/src/request.rs index 045ac0233b2..13372a59418 100644 --- a/rust/kona/crates/node/engine/src/request.rs +++ b/rust/kona/crates/node/engine/src/request.rs @@ -1,6 +1,6 @@ //! Requests sent to the node engine actor and their replies. -use crate::{BuildTaskError, ConsolidateInput, FinalizeBlockId, SealTaskError}; +use crate::{BuildTaskError, ConsolidateInput, DerivedSafeHead, FinalizeBlockId, SealTaskError}; use alloy_rpc_types_engine::PayloadId; use kona_protocol::OpAttributesWithParent; use op_alloy_rpc_types_engine::OpExecutionPayloadEnvelope; @@ -68,6 +68,31 @@ pub struct BuildRequest { pub struct ResetRequest { /// The channel on which the reset result will be sent. pub result_tx: mpsc::Sender>, + /// The reset to perform. + pub kind: ResetKind, +} + +/// The reset a [`ResetRequest`] asks the engine for. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ResetKind { + /// Find a sync starting point with [`crate::find_starting_forkchoice`]. + Full, + /// Answer an L1 reorg detected by derivation. Holds the safe heads derivation recently + /// produced, newest first, each with an L1 block that all of the L1 data it depends on is at + /// or below. The engine keeps the newest one the reorg left canonical; see + /// [`crate::Engine::reset_after_l1_reorg`]. + AfterL1Reorg(Vec), +} + +impl ResetKind { + /// The reset to perform when `next` is requested while `self` is still retained: a full reset + /// wins, and otherwise the newer request does. + pub fn merge(self, next: Self) -> Self { + match self { + Self::Full => Self::Full, + Self::AfterL1Reorg(_) => next, + } + } } /// A request to seal and canonicalize a payload. @@ -81,3 +106,22 @@ pub struct SealRequest { /// The channel on which the result, successful or not, will be sent. pub result_tx: mpsc::Sender>, } + +#[cfg(test)] +mod tests { + use super::ResetKind; + use crate::DerivedSafeHead; + + #[test] + fn a_full_reset_wins_and_otherwise_the_newer_request_does() { + let older = ResetKind::AfterL1Reorg(vec![]); + let newer = ResetKind::AfterL1Reorg(vec![DerivedSafeHead { + safe_head: Default::default(), + derived_from: Default::default(), + }]); + + assert_eq!(older.clone().merge(ResetKind::Full), ResetKind::Full); + assert_eq!(ResetKind::Full.merge(older.clone()), ResetKind::Full); + assert_eq!(older.merge(newer.clone()), newer); + } +} diff --git a/rust/kona/crates/node/engine/src/sync/mod.rs b/rust/kona/crates/node/engine/src/sync/mod.rs index cdf468ba554..fb570a6a162 100644 --- a/rust/kona/crates/node/engine/src/sync/mod.rs +++ b/rust/kona/crates/node/engine/src/sync/mod.rs @@ -9,6 +9,9 @@ pub use forkchoice::L2ForkchoiceState; mod error; pub use error::SyncStartError; +mod reorg; +pub use reorg::{DerivedSafeHead, find_reorg_safe_head}; + use tracing::info; use crate::EngineClient; diff --git a/rust/kona/crates/node/engine/src/sync/reorg.rs b/rust/kona/crates/node/engine/src/sync/reorg.rs new file mode 100644 index 00000000000..d1eb07c7f7f --- /dev/null +++ b/rust/kona/crates/node/engine/src/sync/reorg.rs @@ -0,0 +1,100 @@ +//! Choosing a safe head to keep across an L1 reorg. +//! +//! [`find_starting_forkchoice`] cannot know which L1 block made the engine's safe head safe, so it +//! walks the safe head back a full sequencing window, stopping early only at the finalized head. +//! That is the right answer on a cold start, but after an L1 reorg detected mid-derivation it +//! makes the cost of a reset independent of the reorg's depth: a one-block replacement of the L1 +//! tip rewinds the safe head as far as a cold start does, and while L1 finality is stalled that is +//! the whole sequencing window. +//! +//! Derivation does know where each safe head came from. A [`DerivedSafeHead`] records that all of +//! the L1 data `safe_head` depends on is at or below `derived_from`. Derivation is a function of +//! the L1 chain it has read, so while `derived_from` is canonical the reorg changed none of that +//! input and `safe_head` is still the safe head that L1 chain derives. Keeping it is then exactly +//! as sound as any reset: the pipeline restarts from a known-correct safe head, a channel timeout +//! behind its L1 origin, as it does after [`find_starting_forkchoice`]. +//! +//! [`find_starting_forkchoice`]: crate::find_starting_forkchoice + +use crate::{EngineClient, EngineState, SyncStartError}; +use alloy_eips::{BlockId, BlockNumHash}; +use kona_protocol::L2BlockInfo; +use tracing::info; + +/// A safe head, and an L1 block that all of the L1 data the safe head depends on is at or below. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DerivedSafeHead { + /// The safe head. + pub safe_head: L2BlockInfo, + /// An L1 block at or above every L1 block `safe_head` depends on, including through its + /// ancestors. + pub derived_from: BlockNumHash, +} + +/// Picks the safe head to keep after an L1 reorg, or `None` when only a full reset is sound. +/// +/// `candidates` must be ordered newest first. The first candidate whose `derived_from` is still +/// canonical is chosen. It is kept only if it is still on the engine's chain, at or below the +/// local safe head and at or above the finalized head. No candidate is kept if the unsafe head's +/// L1 origin was reorged out: [`find_starting_forkchoice`] must rewind that unsafe chain. +/// +/// [`find_starting_forkchoice`]: crate::find_starting_forkchoice +pub async fn find_reorg_safe_head( + engine_client: &EngineClient, + state: &EngineState, + candidates: &[DerivedSafeHead], +) -> Result, SyncStartError> { + let sync_state = state.sync_state; + let unsafe_head = sync_state.unsafe_head(); + if !is_canonical_l1(engine_client, unsafe_head.l1_origin).await? { + info!( + target: "sync_start", + l1_origin = ?unsafe_head.l1_origin, + "Unsafe head's L1 origin is no longer canonical; a full reset is needed" + ); + return Ok(None); + } + + for candidate in candidates { + if !is_canonical_l1(engine_client, candidate.derived_from).await? { + continue; + } + + let safe_head = candidate.safe_head; + let number = safe_head.block_info.number; + if number > sync_state.local_safe_head().block_info.number || + number < sync_state.finalized_head().block_info.number + { + return Ok(None); + } + let on_chain = engine_client + .get_l2_block(BlockId::number(number)) + .await? + .is_some_and(|block| block.header.hash == safe_head.block_info.hash); + if !on_chain { + return Ok(None); + } + + info!( + target: "sync_start", + safe_head = %number, + derived_from = ?candidate.derived_from, + "Keeping a safe head derived from canonical L1 data across the L1 reorg" + ); + return Ok(Some(safe_head)); + } + + Ok(None) +} + +/// Whether `block` is the canonical L1 block at its height. A block the L1 node does not have at +/// that height yet is not known to be canonical. +async fn is_canonical_l1( + engine_client: &EngineClient, + block: BlockNumHash, +) -> Result { + Ok(engine_client + .get_l1_block(BlockId::number(block.number)) + .await? + .is_some_and(|canonical| canonical.header.hash == block.hash)) +} diff --git a/rust/kona/crates/node/engine/src/task_queue/core.rs b/rust/kona/crates/node/engine/src/task_queue/core.rs index 1ce6deea550..a5315552c9a 100644 --- a/rust/kona/crates/node/engine/src/task_queue/core.rs +++ b/rust/kona/crates/node/engine/src/task_queue/core.rs @@ -2,9 +2,9 @@ use super::EngineTaskExt; use crate::{ - EngineClient, EngineState, EngineSyncStateUpdate, EngineTask, EngineTaskError, + DerivedSafeHead, EngineClient, EngineState, EngineSyncStateUpdate, EngineTask, EngineTaskError, EngineTaskErrorSeverity, Metrics, SyncStartError, SynchronizeTask, SynchronizeTaskError, - find_starting_forkchoice, task_queue::EngineTaskErrors, + find_reorg_safe_head, find_starting_forkchoice, task_queue::EngineTaskErrors, }; use kona_genesis::RollupConfig; use kona_protocol::L2BlockInfo; @@ -132,6 +132,43 @@ impl Engine { Ok(start.safe) } + /// Resets the engine after an L1 reorg, keeping the newest of `candidates` (ordered newest + /// first) that [`find_reorg_safe_head`] shows was derived from L1 data the reorg left + /// canonical. The unsafe head is kept, and the safe heads are rewound to the kept one if they + /// are above it. Falls back to [`Engine::reset`] when no candidate can be kept. + pub async fn reset_after_l1_reorg( + &mut self, + client: Arc, + config: Arc, + candidates: &[DerivedSafeHead], + ) -> Result { + let Some(safe_head) = + find_reorg_safe_head(client.as_ref(), &self.state, candidates).await? + else { + return self.reset(client, config).await; + }; + + let sync_state = self.state.sync_state; + if safe_head != sync_state.local_safe_head() { + let above = |head: L2BlockInfo| head.block_info.number > safe_head.block_info.number; + let synchronize = SynchronizeTask::new( + client, + config, + EngineSyncStateUpdate { + unsafe_head: None, + local_safe_head: Some(safe_head), + safe_head: above(sync_state.safe_head()).then_some(safe_head), + finalized_head: None, + }, + ); + synchronize.execute(&mut self.state).await?; + } + + metrics::counter!(Metrics::ENGINE_REORG_RESET_COUNT).increment(1); + + Ok(safe_head) + } + /// Clears the task queue. pub fn clear(&mut self) { self.tasks.clear(); @@ -344,3 +381,259 @@ mod recovery_tests { ); } } + +#[cfg(test)] +mod l1_reorg_reset_tests { + use super::*; + use crate::test_utils::{RpcMock, test_engine_client}; + use alloy_eips::BlockNumHash; + use alloy_primitives::B256; + use alloy_rpc_types_engine::{ForkchoiceUpdated, PayloadStatus, PayloadStatusEnum}; + use kona_protocol::BlockInfo; + use rstest::rstest; + use serde_json::json; + + /// L1 block `number` on `fork`: fork 0 is the chain derivation read, fork 1 replaced it from + /// the reorg height on. + fn l1_id(number: u64, fork: u8) -> BlockNumHash { + BlockNumHash { number, hash: B256::left_padding_from(&[fork, number as u8]) } + } + + fn l2_block(number: u64, l1_origin: u64) -> L2BlockInfo { + let hash = |n: u64| B256::left_padding_from(&[0xee, n as u8]); + L2BlockInfo { + block_info: BlockInfo { + number, + hash: hash(number), + parent_hash: hash(number - 1), + timestamp: number, + }, + l1_origin: l1_id(l1_origin, 0), + seq_num: 0, + } + } + + /// Scripts the L1 lookup of block `number`, answered from the chain reorged at + /// `reorg_height`. + fn expect_l1(l1: &RpcMock, number: u64, reorg_height: u64) { + let mut block = alloy_rpc_types_eth::Block::::default(); + block.header.hash = l1_id(number, u8::from(number >= reorg_height)).hash; + block.header.inner.number = number; + l1.expect_params("eth_getBlockByNumber", json!([format!("{number:#x}"), false]), block); + } + + /// Scripts the engine's lookup of L2 block `number`, answered with `hash`. + fn expect_l2(l2: &RpcMock, number: u64, hash: B256) { + let mut block: ::BlockResponse = + Default::default(); + block.header.hash = hash; + block.header.inner.number = number; + l2.expect_params("eth_getBlockByNumber", json!([format!("{number:#x}"), false]), block); + } + + const FINALIZED: u64 = 5; + const UNSAFE: u64 = 40; + const UNSAFE_ORIGIN: u64 = 102; + + /// Derivation's recent safe heads, newest first: L2 block 20 depends on L1 data up to block + /// 106, and L2 block 18 on L1 data up to block 103. + fn recent_safe_heads() -> [DerivedSafeHead; 2] { + [ + DerivedSafeHead { safe_head: l2_block(20, 100), derived_from: l1_id(106, 0) }, + DerivedSafeHead { safe_head: l2_block(18, 99), derived_from: l1_id(103, 0) }, + ] + } + + /// An engine at the local safe head L2 block 20 and the unsafe head L2 block 40, whose L1 + /// origin is `unsafe_origin`. + fn setup( + unsafe_origin: u64, + ) -> (Engine, Arc, RpcMock, RpcMock, Arc) { + let mut config = RollupConfig::default(); + config.hardforks.ecotone_time = Some(0); + let config = Arc::new(config); + let (client, l1, l2) = test_engine_client(config.clone()); + + let mut state = EngineState::default(); + state.sync_state = state.sync_state.apply_update(EngineSyncStateUpdate { + unsafe_head: Some(l2_block(UNSAFE, unsafe_origin)), + local_safe_head: Some(l2_block(20, 100)), + safe_head: Some(l2_block(20, 100)), + finalized_head: Some(l2_block(FINALIZED, 90)), + }); + let (state_tx, _) = tokio::sync::watch::channel(state); + (Engine::new(state, state_tx), Arc::new(client), l1, l2, config) + } + + /// Each case scripts exactly the lookups expected, in order: the unsafe head's L1 origin, + /// then each recent safe head's L1 block until one is canonical, then that safe head on the + /// engine's chain. + #[rstest] + #[case::tip_reorg_keeps_the_safe_head(108, UNSAFE_ORIGIN, &[UNSAFE_ORIGIN, 106], Some(20))] + #[case::reorg_of_the_safe_heads_l1_block_keeps_the_previous_one( + 105, UNSAFE_ORIGIN, &[UNSAFE_ORIGIN, 106, 103], Some(18) + )] + #[case::reorg_of_every_recent_safe_heads_l1_block( + 103, UNSAFE_ORIGIN, &[UNSAFE_ORIGIN, 106, 103], None + )] + // L2 block 20's L1 block 106 is still canonical; only the unsafe head's origin was replaced. + #[case::reorg_of_the_unsafe_heads_l1_origin(107, 107, &[107], None)] + #[tokio::test] + async fn finds_the_newest_safe_head_derived_from_canonical_l1( + #[case] reorg_height: u64, + #[case] unsafe_origin: u64, + #[case] l1_lookups: &[u64], + #[case] kept: Option, + ) { + let (engine, client, l1, l2, _) = setup(unsafe_origin); + for &number in l1_lookups { + expect_l1(&l1, number, reorg_height); + } + if let Some(kept) = kept { + expect_l2(&l2, kept, l2_block(kept, 0).block_info.hash); + } + + let safe_head = + find_reorg_safe_head(&client, engine.state(), &recent_safe_heads()).await.unwrap(); + + assert_eq!(safe_head.map(|head| head.block_info.number), kept); + l1.assert_finished(); + l2.assert_finished(); + } + + /// A safe head outside the engine's local-safe and finalized heads is not kept, even if its L1 + /// block is canonical. + #[rstest] + #[case::above_the_local_safe_head(25)] + #[case::below_the_finalized_head(FINALIZED - 1)] + #[tokio::test] + async fn does_not_keep_a_safe_head_outside_the_engines_safe_range(#[case] number: u64) { + let (engine, client, l1, l2, _) = setup(UNSAFE_ORIGIN); + expect_l1(&l1, UNSAFE_ORIGIN, 108); + expect_l1(&l1, 106, 108); + let candidate = + DerivedSafeHead { safe_head: l2_block(number, 99), derived_from: l1_id(106, 0) }; + + let safe_head = find_reorg_safe_head(&client, engine.state(), &[candidate]).await.unwrap(); + + assert_eq!(safe_head, None); + l1.assert_finished(); + l2.assert_finished(); + } + + /// Scripts the forkchoice update that moves the safe labels, checking the heads it sends. + fn expect_forkchoice_update(l2: &RpcMock, head: L2BlockInfo, safe: L2BlockInfo) { + let state = alloy_rpc_types_engine::ForkchoiceState { + head_block_hash: head.block_info.hash, + safe_block_hash: safe.block_info.hash, + finalized_block_hash: l2_block(FINALIZED, 90).block_info.hash, + }; + l2.expect_params( + "engine_forkchoiceUpdatedV3", + json!([state, null]), + ForkchoiceUpdated::new(PayloadStatus::from_status(PayloadStatusEnum::Valid)), + ); + } + + /// A safe head no longer on the engine's chain is not kept, even if its L1 block is canonical. + #[tokio::test] + async fn does_not_keep_a_safe_head_off_the_engines_chain() { + let (engine, client, l1, l2, _) = setup(UNSAFE_ORIGIN); + expect_l1(&l1, UNSAFE_ORIGIN, 108); + expect_l1(&l1, 106, 108); + expect_l2(&l2, 20, B256::repeat_byte(0xaa)); + + let safe_head = + find_reorg_safe_head(&client, engine.state(), &recent_safe_heads()).await.unwrap(); + + assert_eq!(safe_head, None); + l1.assert_finished(); + l2.assert_finished(); + } + + #[tokio::test] + async fn tip_reorg_keeps_every_head_without_a_forkchoice_update() { + let (mut engine, client, l1, l2, config) = setup(UNSAFE_ORIGIN); + let before = engine.state().sync_state; + expect_l1(&l1, UNSAFE_ORIGIN, 108); + expect_l1(&l1, 106, 108); + expect_l2(&l2, 20, l2_block(20, 100).block_info.hash); + + let safe_head = + engine.reset_after_l1_reorg(client, config, &recent_safe_heads()).await.unwrap(); + + assert_eq!(safe_head, l2_block(20, 100)); + assert_eq!(engine.state().sync_state, before); + l1.assert_finished(); + l2.assert_finished(); + } + + #[tokio::test] + async fn reorg_of_the_safe_heads_l1_block_rewinds_only_the_safe_heads() { + let (mut engine, client, l1, l2, config) = setup(UNSAFE_ORIGIN); + for number in [UNSAFE_ORIGIN, 106, 103] { + expect_l1(&l1, number, 105); + } + expect_l2(&l2, 18, l2_block(18, 99).block_info.hash); + expect_forkchoice_update(&l2, l2_block(UNSAFE, UNSAFE_ORIGIN), l2_block(18, 99)); + + let safe_head = + engine.reset_after_l1_reorg(client, config, &recent_safe_heads()).await.unwrap(); + + assert_eq!(safe_head, l2_block(18, 99)); + let sync_state = engine.state().sync_state; + assert_eq!(sync_state.local_safe_head(), safe_head); + assert_eq!(sync_state.safe_head(), safe_head); + assert_eq!(sync_state.unsafe_head(), l2_block(UNSAFE, UNSAFE_ORIGIN)); + assert_eq!(sync_state.finalized_head(), l2_block(FINALIZED, 90)); + l1.assert_finished(); + l2.assert_finished(); + } + + /// A cross-safe head already below the kept safe head stays where it is: only local-safe moves. + #[tokio::test] + async fn a_cross_safe_head_below_the_kept_one_is_left_alone() { + let (mut engine, client, l1, l2, config) = setup(UNSAFE_ORIGIN); + let cross_safe = l2_block(15, 98); + engine.state.sync_state = engine.state.sync_state.apply_update(EngineSyncStateUpdate { + unsafe_head: None, + local_safe_head: None, + safe_head: Some(cross_safe), + finalized_head: None, + }); + for number in [UNSAFE_ORIGIN, 106, 103] { + expect_l1(&l1, number, 105); + } + expect_l2(&l2, 18, l2_block(18, 99).block_info.hash); + expect_forkchoice_update(&l2, l2_block(UNSAFE, UNSAFE_ORIGIN), cross_safe); + + let safe_head = + engine.reset_after_l1_reorg(client, config, &recent_safe_heads()).await.unwrap(); + + assert_eq!(engine.state().sync_state.local_safe_head(), safe_head); + assert_eq!(engine.state().sync_state.safe_head(), cross_safe); + l1.assert_finished(); + l2.assert_finished(); + } + + /// With no safe head to keep, the reset falls back to the full walk, which starts by reading + /// the engine's finalized head. + #[tokio::test] + async fn falls_back_to_the_full_walk() { + let (mut engine, client, l1, l2, config) = setup(UNSAFE_ORIGIN); + for number in [UNSAFE_ORIGIN, 106, 103] { + expect_l1(&l1, number, 103); + } + l2.expect_error("eth_getBlockByNumber"); + + let err = + engine.reset_after_l1_reorg(client, config, &recent_safe_heads()).await.unwrap_err(); + + assert!( + matches!(err, EngineResetError::SyncStart(SyncStartError::RpcError(_))), + "got {err:?}" + ); + l1.assert_finished(); + l2.assert_finished(); + } +} diff --git a/rust/kona/crates/node/rpc/src/admin.rs b/rust/kona/crates/node/rpc/src/admin.rs index a6319a6f788..8c9c933a54c 100644 --- a/rust/kona/crates/node/rpc/src/admin.rs +++ b/rust/kona/crates/node/rpc/src/admin.rs @@ -8,7 +8,7 @@ use jsonrpsee::{ core::RpcResult, types::{ErrorCode, ErrorObject}, }; -use kona_engine::{EngineActorRequest, ResetRequest}; +use kona_engine::{EngineActorRequest, ResetKind, ResetRequest}; use op_alloy_rpc_types_engine::{OpExecutionPayloadEnvelope, OpPayloadError}; use thiserror::Error; use tokio::sync::{mpsc, oneshot, watch}; @@ -174,7 +174,10 @@ impl AdminApiServer for AdminRpc { async fn admin_reset_derivation_pipeline(&self) -> RpcResult<()> { let (result_tx, mut result_rx) = mpsc::channel(1); self.engine - .send(EngineActorRequest::Reset(Box::new(ResetRequest { result_tx }))) + .send(EngineActorRequest::Reset(Box::new(ResetRequest { + result_tx, + kind: ResetKind::Full, + }))) .await .map_err(|_| ErrorObject::from(ErrorCode::InternalError))?; result_rx diff --git a/rust/kona/crates/node/service/src/actors/derivation/actor.rs b/rust/kona/crates/node/service/src/actors/derivation/actor.rs index 73297ab8c4f..90b662433e9 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/actor.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/actor.rs @@ -3,7 +3,7 @@ use crate::{ DerivationActorRequest, DerivationEngineClient, DerivationState, DerivationStateMachine, DerivationStateTransitionError, DerivationStateUpdate, Metrics, NodeActor, - actors::derivation::L2Finalizer, + actors::derivation::{L2Finalizer, SafeHeadHistory}, }; use async_trait::async_trait; use kona_derive::{ @@ -45,6 +45,8 @@ where safe_db: Option>, /// The attributes sent to the engine and not yet confirmed as a safe head. attributes_awaiting_confirmation: Option, + /// The recent safe heads an L1 reorg may keep, with the L1 data they depend on. + safe_head_history: SafeHeadHistory, } impl @@ -70,6 +72,7 @@ where finalizer: L2Finalizer::default(), safe_db, attributes_awaiting_confirmation: None, + safe_head_history: SafeHeadHistory::default(), } } @@ -80,6 +83,7 @@ where self.derivation_origin_tx.send_replace(None); self.finalizer.clear(); self.attributes_awaiting_confirmation = None; + self.reset_safe_head_history(reset.l2_safe_head); self.truncate_safe_db(reset.l2_safe_head, "engine reset")?; let result = self.pipeline.signal(Signal::Reset(reset)).await; if result.is_ok() { @@ -100,6 +104,40 @@ where Ok(()) } + /// Restarts the [`SafeHeadHistory`] from the safe head derivation restarts from. + fn reset_safe_head_history(&mut self, safe_head: L2BlockInfo) { + let seq_window_size = self.pipeline.rollup_config().seq_window_size; + self.safe_head_history.reset_to(safe_head, seq_window_size); + } + + /// Asks the engine to reset for a pipeline [`ResetError`]. An L1 reorg lets the engine keep a + /// safe head the reorg left derivable, rather than rewind a full sequencing window. + async fn reset_engine(&self, e: &ResetError) -> Result<(), DerivationError> { + let reset = if let ResetError::ReorgDetected(expected, new) = e { + warn!(target: "derivation", "L1 reorg detected! Expected: {expected} | New: {new}"); + metrics::counter!(Metrics::L1_REORG_COUNT).increment(1); + + // The pipeline's origin has no canonical child, so it was reorged out, and so was every + // L1 block at or above it. Drop the safe heads that depend on one without asking the + // engine, whose L1 node may not have seen the reorg yet. + let reorged_from = self.pipeline.origin().map_or(u64::MAX, |origin| origin.number); + let recent_safe_heads = self + .safe_head_history + .newest_first() + .into_iter() + .filter(|derived| derived.derived_from.number < reorged_from) + .collect(); + self.engine_client.reset_engine_after_l1_reorg(recent_safe_heads).await + } else { + self.engine_client.reset_engine_forkchoice().await + }; + + reset.map_err(|e| { + error!(target: "derivation", ?e, "Failed to send reset request"); + DerivationError::Sender(Box::new(e)) + }) + } + /// Activates a hardfork while preserving the pipeline's derivation state. async fn signal_activation(&mut self, activation: ActivationSignal) -> PipelineResult<()> { self.pipeline.signal(Signal::Activation(activation)).await @@ -170,20 +208,9 @@ where })) .await?; } else { - if let ResetError::ReorgDetected(expected, new) = e { - warn!( - target: "derivation", - "L1 reorg detected! Expected: {expected} | New: {new}" - ); - - metrics::counter!(Metrics::L1_REORG_COUNT).increment(1); - } // Like op-node, report no origin until the reset sets a new one. self.derivation_origin_tx.send_replace(None); - self.engine_client.reset_engine_forkchoice().await.map_err(|e| { - error!(target: "derivation", ?e, "Failed to send reset request"); - DerivationError::Sender(Box::new(e)) - })?; + self.reset_engine(&e).await?; self.derivation_state_machine .update(&DerivationStateUpdate::SignalNeeded)?; return Err(DerivationError::Yield); @@ -239,6 +266,18 @@ where } DerivationActorRequest::ProcessEngineSafeHeadUpdateRequest(safe_head) => { info!(target: "derivation", safe_head = ?*safe_head, "Received safe head from engine."); + // The engine reports its safe head after every reset, even when unchanged, and an + // L1 head can arrive between the reset signal and that report. Unless derivation + // is still waiting for it, a head it has already confirmed carries nothing new: it + // must not consume attributes awaiting confirmation or move the state machine. + if safe_head.block_info.hash == + self.derivation_state_machine.last_confirmed_safe_head().block_info.hash && + self.derivation_state_machine.current_state() != + DerivationState::AwaitingUpdateAfterSignal + { + debug!(target: "derivation", safe_head = ?*safe_head, "Ignoring an already confirmed safe head."); + return Ok(()); + } self.derivation_state_machine .update(&DerivationStateUpdate::NewAttributesConfirmed(safe_head.clone()))?; self.settle_derived_attributes(*safe_head)?; @@ -249,6 +288,7 @@ where info!(target: "derivation", "Engine finished syncing, starting derivation."); self.derivation_state_machine .update(&DerivationStateUpdate::ELSyncCompleted(safe_head.clone()))?; + self.reset_safe_head_history(*safe_head); self.truncate_safe_db(*safe_head, "EL sync completion")?; self.attempt_derivation().await?; @@ -281,6 +321,9 @@ where ); return Ok(()); } + if let Some(derived_from) = attributes.derived_from { + self.safe_head_history.record(safe_head, derived_from.id()); + } if let (Some(database), Some(derived_from)) = (&self.safe_db, attributes.derived_from) { debug!(target: "derivation", l1_block = derived_from.number, safe_head = safe_head.block_info.number, "Recording safe head"); database.safe_head_updated(safe_head, derived_from.id()).inspect_err(|error| { diff --git a/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs b/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs index 67d7d153581..8c95355bd40 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs @@ -2,6 +2,7 @@ use super::*; use crate::actors::derivation::engine_client::MockDerivationEngineClient; use alloy_eips::BlockNumHash; use alloy_primitives::B256; +use kona_engine::DerivedSafeHead; use kona_genesis::{HardForkConfig, RollupConfig, SystemConfig}; use kona_protocol::BlockInfo; use kona_safedb::{SafeDatabase, SafeHeadRecord}; @@ -82,19 +83,31 @@ impl Pipeline for StubPipeline { /// A pipeline-driven reset must always reach the engine actor. The engine's reset is what /// sends the pipeline its [`Signal`] back; without it the actor parks in -/// [`DerivationState::AwaitingSignal`] forever. +/// [`DerivationState::AwaitingSignal`] forever. An L1 reorg asks for the reset after a reorg; +/// any other reset asks for the full one. #[rstest] -#[case::interop_inactive(None)] -#[case::interop_active(Some(0))] +#[case::reorg_interop_inactive(None, true)] +#[case::reorg_interop_active(Some(0), true)] +#[case::other_reset_interop_inactive(None, false)] +#[case::other_reset_interop_active(Some(0), false)] #[tokio::test] -async fn test_pipeline_reset_always_resets_engine(#[case] lagoon_time: Option) { +async fn test_pipeline_reset_always_resets_engine( + #[case] lagoon_time: Option, + #[case] reorg: bool, +) { let rollup_config = Arc::new(RollupConfig { hardforks: HardForkConfig { lagoon_time, ..Default::default() }, ..Default::default() }); let mut engine_client = MockDerivationEngineClient::new(); - engine_client.expect_reset_engine_forkchoice().times(1).returning(|| Ok(())); + let reset = if reorg { + engine_client.expect_reset_engine_after_l1_reorg().times(1).returning(|_| Ok(())); + ResetError::ReorgDetected(B256::ZERO, B256::repeat_byte(1)) + } else { + engine_client.expect_reset_engine_forkchoice().times(1).returning(|| Ok(())); + ResetError::BadParentHash(B256::ZERO, B256::repeat_byte(1)) + }; let (derivation_origin_tx, derivation_origin_rx) = watch::channel(None); let (request_tx, request_rx) = mpsc::channel(1); @@ -105,9 +118,7 @@ async fn test_pipeline_reset_always_resets_engine(#[case] lagoon_time: Option, derived_from: BlockInfo, rollup_config: Arc, + reorg_when_exhausted: bool, } impl Iterator for QueuedPipeline { @@ -306,10 +319,14 @@ impl Pipeline for QueuedPipeline { } async fn step(&mut self, _: L2BlockInfo) -> StepResult { - if self.attributes.is_empty() { - StepResult::StepFailed(PipelineError::Eof.temp()) - } else { + if !self.attributes.is_empty() { StepResult::PreparedAttributes + } else if self.reorg_when_exhausted { + StepResult::StepFailed( + ResetError::ReorgDetected(B256::ZERO, B256::repeat_byte(1)).reset(), + ) + } else { + StepResult::StepFailed(PipelineError::Eof.temp()) } } @@ -388,6 +405,7 @@ impl RecordingActor { attributes: VecDeque::from([derived_attributes()]), derived_from: l1_block(DERIVED_FROM_L1), rollup_config: Arc::new(RollupConfig::default()), + reorg_when_exhausted: false, }; let (request_tx, request_rx) = mpsc::channel(1); let actor = DerivationActor::new( @@ -444,6 +462,128 @@ impl RecordingActor { } } +/// The recorded safe head [`DERIVED_HEAD`], derived from [`DERIVED_FROM_L1`]. +fn derived_head() -> DerivedSafeHead { + DerivedSafeHead { + safe_head: l2_block(DERIVED_HEAD), + derived_from: l1_block(DERIVED_FROM_L1).id(), + } +} + +/// An engine client that accepts attributes and expects one reset after an L1 reorg, handed +/// `recent_safe_heads`. +fn engine_client_expecting_reorg_reset( + recent_safe_heads: Vec, +) -> MockDerivationEngineClient { + let mut engine_client = engine_client_accepting_attributes(); + engine_client + .expect_reset_engine_after_l1_reorg() + .withf(move |heads| *heads == recent_safe_heads) + .times(1) + .returning(|_| Ok(())); + engine_client +} + +/// An L1 reorg hands the engine the safe heads derivation produced, each with the L1 block it +/// was derived from, so the engine can keep one the reorg left canonical. The pipeline's own +/// origin was reorged out, so a safe head derived at it is not offered. +#[rstest] +#[case::derived_below_the_reorged_origin(DERIVED_FROM_L1 + 1, vec![derived_head()])] +#[case::derived_at_the_reorged_origin(DERIVED_FROM_L1, vec![])] +#[tokio::test] +async fn an_l1_reorg_hands_the_engine_the_recent_safe_heads( + #[case] reorged_origin: u64, + #[case] expected: Vec, +) { + let mut node = RecordingActor::new(engine_client_expecting_reorg_reset(expected)); + node.actor.pipeline.derived_from = l1_block(reorged_origin); + node.actor.pipeline.reorg_when_exhausted = true; + + node.derive_and_confirm_one_block().await; + + assert_eq!( + node.actor.derivation_state_machine.current_state(), + DerivationState::AwaitingSignal + ); +} + +/// A reset to a safe head the history did not record clears the history, so a later reorg +/// offers nothing that may depend on data below the reset. +#[tokio::test] +async fn a_reset_to_an_unrecorded_safe_head_clears_the_history() { + let mut engine_client = engine_client_accepting_attributes(); + let mut sequence = mockall::Sequence::new(); + for expected in [vec![derived_head()], vec![]] { + engine_client + .expect_reset_engine_after_l1_reorg() + .withf(move |heads| *heads == expected) + .times(1) + .in_sequence(&mut sequence) + .returning(|_| Ok(())); + } + let mut node = RecordingActor::new(engine_client); + node.actor.pipeline.derived_from = l1_block(DERIVED_FROM_L1 + 1); + node.actor.pipeline.reorg_when_exhausted = true; + node.derive_and_confirm_one_block().await; + + node.signal(Signal::Reset(kona_derive::ResetSignal { l2_safe_head: l2_block(RESUME_HEAD) })) + .await; + // Reorg again before deriving anything. + node.actor.pipeline.attributes.clear(); + node.handle(DerivationActorRequest::ProcessL1HeadUpdateRequest(Box::default())).await; +} + +/// At EL sync completion, the safe head's L1 data is unknown beyond the end of its sequencing +/// window, so a block derived below that is not recorded. +#[tokio::test] +async fn el_sync_completion_floors_the_history_at_the_sequencing_window() { + let mut node = RecordingActor::new(engine_client_expecting_reorg_reset(vec![])); + node.actor.pipeline.rollup_config = + Arc::new(RollupConfig { seq_window_size: DERIVED_FROM_L1, ..Default::default() }); + node.actor.pipeline.derived_from = l1_block(DERIVED_FROM_L1 + 1); + node.actor.pipeline.reorg_when_exhausted = true; + + node.derive_and_confirm_one_block().await; +} + +/// After a reset, the engine reports the safe head even when unchanged, and an L1 head can +/// arrive first and set derivation going. The late report must not consume the attributes +/// derivation sent meanwhile, or the engine's confirmation of them is lost. +#[rstest] +#[case::attributes_awaiting_confirmation(true)] +#[case::awaiting_l1_data(false)] +#[tokio::test] +async fn a_late_report_of_the_kept_safe_head_is_ignored(#[case] derives_after_reset: bool) { + let mut node = RecordingActor::new(engine_client_accepting_attributes()); + node.derive_and_confirm_one_block().await; + + // A reset that keeps the safe head, then an L1 head before the engine reports it. + node.signal(Signal::Reset(kona_derive::ResetSignal { l2_safe_head: l2_block(DERIVED_HEAD) })) + .await; + node.actor.pipeline.attributes.clear(); + let next_l1 = DERIVED_FROM_L1 + 1; + if derives_after_reset { + node.actor.pipeline.attributes.push_back(OpAttributesWithParent { + attributes: OpPayloadAttributes::default(), + parent: l2_block(DERIVED_HEAD), + derived_from: Some(l1_block(next_l1)), + is_last_in_span: true, + }); + } + node.handle(DerivationActorRequest::ProcessL1HeadUpdateRequest(Box::default())).await; + node.confirm_safe_head(DERIVED_HEAD).await; + + if derives_after_reset { + node.confirm_safe_head(DERIVED_HEAD + 1).await; + assert_eq!(node.db.last_entry().unwrap(), record(next_l1, DERIVED_HEAD + 1)); + } else { + assert_eq!( + node.actor.derivation_state_machine.current_state(), + DerivationState::AwaitingL1Data + ); + } +} + #[tokio::test] async fn a_confirmed_block_is_recorded_against_its_derivation_source() { let mut node = RecordingActor::new(engine_client_accepting_attributes()); diff --git a/rust/kona/crates/node/service/src/actors/derivation/engine_client.rs b/rust/kona/crates/node/service/src/actors/derivation/engine_client.rs index ad024f605ad..fa85c69f9f5 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/engine_client.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/engine_client.rs @@ -1,7 +1,7 @@ -use crate::{EngineActorRequest, EngineClientError, EngineClientResult, ResetRequest}; +use crate::{EngineActorRequest, EngineClientError, EngineClientResult, ResetKind, ResetRequest}; use async_trait::async_trait; use derive_more::Constructor; -use kona_engine::{ConsolidateInput, FinalizeBlockId}; +use kona_engine::{ConsolidateInput, DerivedSafeHead, FinalizeBlockId}; use std::fmt::Debug; use tokio::sync::mpsc; @@ -12,6 +12,14 @@ pub trait DerivationEngineClient: Debug + Send + Sync { /// Resets the engine's forkchoice. async fn reset_engine_forkchoice(&self) -> EngineClientResult<()>; + /// Resets the engine after derivation detected an L1 reorg. `recent_safe_heads` are the safe + /// heads derivation recently produced, newest first. The engine keeps the newest one derived + /// from L1 data the reorg left canonical, and falls back to a full reset otherwise. + async fn reset_engine_after_l1_reorg( + &self, + recent_safe_heads: Vec, + ) -> EngineClientResult<()>; + /// Sends a request to finalize the L2 block identified by the provided [`FinalizeBlockId`]. /// Note: This does not wait for the engine to process it. async fn send_finalized_l2_block(&self, block_id: FinalizeBlockId) -> EngineClientResult<()>; @@ -33,13 +41,12 @@ pub struct QueuedDerivationEngineClient { pub engine_actor_request_tx: mpsc::Sender, } -#[async_trait] -impl DerivationEngineClient for QueuedDerivationEngineClient { - async fn reset_engine_forkchoice(&self) -> EngineClientResult<()> { +impl QueuedDerivationEngineClient { + async fn reset(&self, kind: ResetKind) -> EngineClientResult<()> { let (result_tx, mut result_rx) = mpsc::channel(1); self.engine_actor_request_tx - .send(EngineActorRequest::Reset(Box::new(ResetRequest { result_tx }))) + .send(EngineActorRequest::Reset(Box::new(ResetRequest { result_tx, kind }))) .await .map_err(|_| EngineClientError::RequestError("request channel closed.".to_string()))?; @@ -48,6 +55,20 @@ impl DerivationEngineClient for QueuedDerivationEngineClient { EngineClientError::ResponseError("response channel closed.".to_string()) })? } +} + +#[async_trait] +impl DerivationEngineClient for QueuedDerivationEngineClient { + async fn reset_engine_forkchoice(&self) -> EngineClientResult<()> { + self.reset(ResetKind::Full).await + } + + async fn reset_engine_after_l1_reorg( + &self, + recent_safe_heads: Vec, + ) -> EngineClientResult<()> { + self.reset(ResetKind::AfterL1Reorg(recent_safe_heads)).await + } async fn send_finalized_l2_block(&self, block_id: FinalizeBlockId) -> EngineClientResult<()> { self.engine_actor_request_tx diff --git a/rust/kona/crates/node/service/src/actors/derivation/mod.rs b/rust/kona/crates/node/service/src/actors/derivation/mod.rs index af3263147e0..996b8aa6218 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/mod.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/mod.rs @@ -13,6 +13,9 @@ pub use engine_client::{DerivationEngineClient, QueuedDerivationEngineClient}; mod finalizer; pub(crate) use finalizer::L2Finalizer; +mod safe_head_history; +use safe_head_history::SafeHeadHistory; + mod request; pub use request::{DerivationActorRequest, DerivationClientError, DerivationClientResult}; diff --git a/rust/kona/crates/node/service/src/actors/derivation/safe_head_history.rs b/rust/kona/crates/node/service/src/actors/derivation/safe_head_history.rs new file mode 100644 index 00000000000..ac65c79b520 --- /dev/null +++ b/rust/kona/crates/node/service/src/actors/derivation/safe_head_history.rs @@ -0,0 +1,195 @@ +//! The [`SafeHeadHistory`] for the derivation actor. + +use alloy_eips::BlockNumHash; +use kona_engine::DerivedSafeHead; +use kona_protocol::L2BlockInfo; +use std::collections::BTreeMap; + +/// How many safe heads are kept, one per L1 block that produced one. A reorg that reaches below +/// the oldest kept one falls back to a full engine reset. +const MAX_ENTRIES: usize = 64; + +/// The safe heads derivation produced from the most recent L1 blocks, each with an L1 block that +/// all of the L1 data it depends on is at or below. After an L1 reorg, the engine keeps the newest +/// one whose L1 block the reorg left canonical, instead of rewinding a full sequencing window; +/// see [`kona_engine::find_reorg_safe_head`]. +/// +/// This is kept in memory whether or not the safe head database is enabled, and is bounded to +/// the depth an L1 reorg can plausibly reach. +/// +/// A safe head depends on the L1 data its own attributes were derived from, and on everything its +/// parent depends on. The pipeline's origin bounds only the former: after a reset the pipeline +/// replays from before the safe head's L1 origin, and can produce the next block from an L1 block +/// below the one its parent needed. So the history keeps a floor that is at least the L1 block +/// the current safe head depends on, and records a safe head only if its attributes came from at +/// or above the floor. +#[derive(Debug, Default)] +pub(crate) struct SafeHeadHistory { + /// L1 block number -> the highest safe head produced with the pipeline at that L1 block. + by_l1_block: BTreeMap, + /// The lowest L1 block a new record may claim: at least the L1 block the current safe head + /// depends on. + floor: u64, +} + +impl SafeHeadHistory { + /// Records `safe_head`, confirmed from attributes the pipeline produced at `derived_from`, if + /// that is at or above the floor. + pub(crate) fn record(&mut self, safe_head: L2BlockInfo, derived_from: BlockNumHash) { + if derived_from.number < self.floor { + // The safe head depends on no more than the floor, which stays put. + return; + } + + self.floor = derived_from.number; + self.by_l1_block.insert(derived_from.number, DerivedSafeHead { safe_head, derived_from }); + while self.by_l1_block.len() > MAX_ENTRIES { + self.by_l1_block.pop_first(); + } + } + + /// The recorded safe heads, newest first. + pub(crate) fn newest_first(&self) -> Vec { + self.by_l1_block.values().rev().copied().collect() + } + + /// Derivation restarts from `safe_head`: forgets the safe heads above it. + /// + /// If `safe_head` is recorded, the floor is the L1 block it was recorded with. Otherwise + /// nothing kept is known to lead to it, so the history is cleared and the floor is the L1 + /// block by which derivation must have decided `safe_head`: one past the end of its epoch's + /// sequencing window. + pub(crate) fn reset_to(&mut self, safe_head: L2BlockInfo, seq_window_size: u64) { + self.by_l1_block.retain(|_, derived| { + derived.safe_head.block_info.number <= safe_head.block_info.number + }); + + match self.by_l1_block.last_key_value() { + Some((&number, derived)) if derived.safe_head == safe_head => self.floor = number, + _ => { + self.by_l1_block.clear(); + self.floor = safe_head.l1_origin.number + seq_window_size + 1; + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use alloy_primitives::B256; + use kona_protocol::BlockInfo; + + const SEQ_WINDOW_SIZE: u64 = 10; + + fn l2_block(number: u64) -> L2BlockInfo { + L2BlockInfo { + block_info: BlockInfo { + number, + hash: B256::with_last_byte(number as u8), + parent_hash: B256::with_last_byte(number.wrapping_sub(1) as u8), + timestamp: number, + }, + ..Default::default() + } + } + + fn l1_block(number: u64) -> BlockNumHash { + BlockNumHash { number, hash: B256::repeat_byte(number as u8) } + } + + fn record(history: &mut SafeHeadHistory, number: u64, derived_from: u64) { + history.record(l2_block(number), l1_block(derived_from)); + } + + /// `(safe head number, L1 block number)` of each record, newest first. + fn recorded(history: &SafeHeadHistory) -> Vec<(u64, u64)> { + history + .newest_first() + .iter() + .map(|d| (d.safe_head.block_info.number, d.derived_from.number)) + .collect() + } + + #[test] + fn keeps_the_highest_safe_head_per_l1_block_newest_first() { + let mut history = SafeHeadHistory::default(); + record(&mut history, 11, 100); + record(&mut history, 12, 100); + record(&mut history, 13, 102); + + assert_eq!( + history.newest_first(), + vec![ + DerivedSafeHead { safe_head: l2_block(13), derived_from: l1_block(102) }, + DerivedSafeHead { safe_head: l2_block(12), derived_from: l1_block(100) }, + ] + ); + } + + #[test] + fn bounds_the_entries_kept() { + let mut history = SafeHeadHistory::default(); + for i in 0..(MAX_ENTRIES as u64 + 10) { + record(&mut history, 1 + i, 100 + i); + } + + let recorded = history.newest_first(); + assert_eq!(recorded.len(), MAX_ENTRIES); + assert_eq!(recorded.last().unwrap().derived_from.number, 110); + } + + /// After a reset to a recorded safe head, the pipeline replays from before its L1 origin and + /// can produce the next block from an L1 block below the one that safe head was recorded + /// with. That block depends on its parent, so it must not be recorded as derived from the + /// lower L1 block, and must not overwrite the record already at that L1 block. + #[test] + fn replay_after_reset_does_not_record_below_the_kept_safe_head() { + let mut history = SafeHeadHistory::default(); + record(&mut history, 11, 100); + record(&mut history, 12, 106); + record(&mut history, 13, 107); + + history.reset_to(l2_block(12), SEQ_WINDOW_SIZE); + record(&mut history, 13, 100); + assert_eq!(recorded(&history), vec![(12, 106), (11, 100)]); + + record(&mut history, 14, 108); + assert_eq!(recorded(&history), vec![(14, 108), (12, 106), (11, 100)]); + } + + /// A reset to a safe head the history did not record, such as one from a full reset, leaves + /// that safe head's L1 dependency unknown beyond the end of its sequencing window. + #[test] + fn reset_to_an_unrecorded_safe_head_floors_at_its_sequencing_window() { + let mut history = SafeHeadHistory::default(); + record(&mut history, 11, 100); + record(&mut history, 12, 101); + + let mut safe_head = l2_block(11); + safe_head.l1_origin.number = 95; + safe_head.block_info.hash = B256::repeat_byte(0xaa); + history.reset_to(safe_head, SEQ_WINDOW_SIZE); + assert!(history.newest_first().is_empty()); + + // Derived below the end of the sequencing window: not recorded. + record(&mut history, 12, 105); + assert!(history.newest_first().is_empty()); + + // From the floor on, recording resumes. + record(&mut history, 13, 106); + assert_eq!(recorded(&history), vec![(13, 106)]); + } + + #[test] + fn reset_forgets_safe_heads_above_the_reset_point() { + let mut history = SafeHeadHistory::default(); + record(&mut history, 11, 100); + record(&mut history, 12, 101); + record(&mut history, 13, 102); + + history.reset_to(l2_block(12), SEQ_WINDOW_SIZE); + + assert_eq!(recorded(&history), vec![(12, 101), (11, 100)]); + } +} diff --git a/rust/kona/crates/node/service/src/actors/derivation/state_machine.rs b/rust/kona/crates/node/service/src/actors/derivation/state_machine.rs index 3b5cef5d88b..8b22b3cc383 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/state_machine.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/state_machine.rs @@ -197,7 +197,7 @@ impl DerivationStateMachine { if let DerivationStateUpdate::NewAttributesConfirmed(safe_head) = state_update && safe_head.block_info.hash == self.confirmed_safe_head.block_info.hash { - info!(target: "derivation", ?safe_head, "Re-received safe head. Skipping state transition."); + info!(target: "derivation", ?safe_head, "Re-received the confirmed safe head."); } info!(target: "derivation", state=?self.state, ?state_update, "Executing derivation state update."); diff --git a/rust/kona/crates/node/service/src/actors/engine/actor.rs b/rust/kona/crates/node/service/src/actors/engine/actor.rs index 35144ead0e5..8c7b46d119b 100644 --- a/rust/kona/crates/node/service/src/actors/engine/actor.rs +++ b/rust/kona/crates/node/service/src/actors/engine/actor.rs @@ -1,4 +1,7 @@ -use crate::{BuildRequest, EngineDerivationClient, EngineError, NodeActor, SealRequest}; +use crate::{ + BuildRequest, EngineDerivationClient, EngineError, NodeActor, ResetKind, ResetRequest, + SealRequest, +}; use async_trait::async_trait; use kona_derive::{ResetSignal, Signal}; use kona_engine::{ @@ -50,7 +53,7 @@ where /// instead of fetching it back from the execution layer. block_sink: Arc, /// A reset is retained until successful; RPC failure must not lose the request. - reset_pending: bool, + pending_reset: Option, /// Callers waiting for the retained reset to finish. reset_waiters: Vec>>, /// Earliest next attempt after a dependency failure. @@ -83,7 +86,7 @@ where unsafe_head_tx, inbound_request_rx, block_sink, - reset_pending: false, + pending_reset: None, reset_waiters: Vec::new(), retry_at: Instant::now(), retry_delay: Duration::from_millis(100), @@ -95,18 +98,22 @@ where self.retry_delay = (self.retry_delay * 2).min(Duration::from_secs(5)); } - fn begin_reset(&mut self) { - if !self.reset_pending { - self.engine.clear(); - self.reset_pending = true; - self.retry_at = Instant::now(); - } + /// Retains a reset of `kind`, combined with any reset already retained. + fn begin_reset(&mut self, kind: ResetKind) { + self.pending_reset = Some(match self.pending_reset.take() { + Some(pending) => pending.merge(kind), + None => { + self.engine.clear(); + self.retry_at = Instant::now(); + kind + } + }); } async fn attempt_reset(&mut self) -> Result<(), EngineError> { match self.reset().await { Ok(()) => { - self.reset_pending = false; + self.pending_reset = None; self.retry_delay = Duration::from_millis(100); if !self.el_sync_complete && self.engine.state().el_sync_finished { self.mark_el_sync_complete_and_notify_derivation_actor().await?; @@ -124,7 +131,18 @@ where /// Resets the inner [`Engine`] and propagates the reset to the derivation actor. async fn reset(&mut self) -> Result<(), EngineError> { // Reset the engine. - let l2_safe_head = self.engine.reset(self.client.clone(), self.rollup.clone()).await?; + let l2_safe_head = match &self.pending_reset { + Some(ResetKind::AfterL1Reorg(recent_safe_heads)) => { + self.engine + .reset_after_l1_reorg( + self.client.clone(), + self.rollup.clone(), + recent_safe_heads, + ) + .await? + } + _ => self.engine.reset(self.client.clone(), self.rollup.clone()).await?, + }; // Derivation may be awaiting this reply with its inbound queue full of L1 updates. // Release those callers before awaiting delivery to that same queue. They transition @@ -143,6 +161,10 @@ where } } + // Send the safe head even if the reset left it unchanged, as a reset after an L1 reorg + // usually does. Derivation waits after a reset for a safe head or a new L1 head, and + // drops the L1 heads that arrived while it awaited the reset. + self.last_engine_safe_head_sent = L2BlockInfo::default(); self.send_derivation_actor_safe_head_if_updated().await?; Ok(()) @@ -163,7 +185,7 @@ where } EngineTaskErrorSeverity::Reset => { warn!(target: "engine", ?err, "Received reset request"); - self.begin_reset(); + self.begin_reset(ResetKind::Full); return Ok(()); } EngineTaskErrorSeverity::Flush => { @@ -196,7 +218,7 @@ where if !self.el_sync_complete && self.engine.state().el_sync_finished { if self.engine.state().sync_state.finalized_head() == L2BlockInfo::default() { - self.begin_reset(); + self.begin_reset(ResetKind::Full); } else { self.mark_el_sync_complete_and_notify_derivation_actor().await?; } @@ -252,7 +274,7 @@ where async fn step(&mut self) -> Result<(), Self::Error> { if Instant::now() >= self.retry_at { - if self.reset_pending { + if self.pending_reset.is_some() { self.attempt_reset().await?; } else { self.drain().await?; @@ -276,7 +298,7 @@ where request.ok_or(EngineError::ChannelClosed)? } _ = time::sleep_until(self.retry_at), - if self.reset_pending || !self.engine.is_empty() => return Ok(()), + if self.pending_reset.is_some() || !self.engine.is_empty() => return Ok(()), }; match request { @@ -322,8 +344,9 @@ where EngineActorRequest::Reset(reset_request) => { warn!(target: "engine", "Received reset request"); - self.begin_reset(); - self.reset_waiters.push(reset_request.result_tx); + let ResetRequest { result_tx, kind } = *reset_request; + self.begin_reset(kind); + self.reset_waiters.push(result_tx); } EngineActorRequest::Seal(seal_request) => { let SealRequest { payload_id, attributes, result_tx } = *seal_request; diff --git a/rust/kona/crates/node/service/src/actors/engine/mod.rs b/rust/kona/crates/node/service/src/actors/engine/mod.rs index 533727d043e..11a2b810e1f 100644 --- a/rust/kona/crates/node/service/src/actors/engine/mod.rs +++ b/rust/kona/crates/node/service/src/actors/engine/mod.rs @@ -14,7 +14,7 @@ pub use error::EngineError; pub use kona_engine::{ BuildRequest, EngineActorRequest, EngineRequestError as EngineClientError, - EngineRequestResult as EngineClientResult, ResetRequest, SealRequest, + EngineRequestResult as EngineClientResult, ResetKind, ResetRequest, SealRequest, }; #[cfg(test)] diff --git a/rust/kona/crates/node/service/src/actors/engine/tests.rs b/rust/kona/crates/node/service/src/actors/engine/tests.rs index eded3d472ca..32c89655b97 100644 --- a/rust/kona/crates/node/service/src/actors/engine/tests.rs +++ b/rust/kona/crates/node/service/src/actors/engine/tests.rs @@ -3,8 +3,8 @@ use crate::{DerivationActorRequest, NodeActor}; use alloy_primitives::B256; use alloy_rpc_types_engine::{ForkchoiceUpdated, PayloadStatus, PayloadStatusEnum}; use kona_engine::{ - Engine, EngineActorRequest, EngineState, EngineSyncStateUpdate, FinalizeBlockId, NoopBlockSink, - test_utils::test_engine_client, + DerivedSafeHead, Engine, EngineActorRequest, EngineState, EngineSyncStateUpdate, + FinalizeBlockId, NoopBlockSink, ResetKind, ResetRequest, test_utils::test_engine_client, }; use kona_genesis::RollupConfig; use kona_protocol::{BlockInfo, L2BlockInfo}; @@ -171,3 +171,92 @@ async fn derivation_is_handed_the_local_safe_head() { }; assert_eq!(*sent, local_safe_head); } + +/// Derivation waits after a reset for a safe head or a new L1 head. A reset after an L1 reorg +/// usually keeps the safe head, and must still report it, or derivation stalls until L1 advances. +#[tokio::test(start_paused = true)] +async fn reset_after_l1_reorg_reports_an_unchanged_safe_head() { + let l1_hash = |number: u64| B256::repeat_byte(number as u8); + let safe_head = L2BlockInfo { + l1_origin: alloy_eips::BlockNumHash { number: 100, hash: l1_hash(100) }, + ..l2_block(10) + }; + let derived_from = alloy_eips::BlockNumHash { number: 105, hash: l1_hash(105) }; + let config = Arc::new(RollupConfig::default()); + let (client, l1, l2) = test_engine_client(config.clone()); + let state = EngineState { + sync_state: EngineState::default().sync_state.apply_update(EngineSyncStateUpdate { + unsafe_head: Some(safe_head), + local_safe_head: Some(safe_head), + safe_head: Some(safe_head), + finalized_head: Some(safe_head), + }), + el_sync_finished: true, + ..Default::default() + }; + let (derivation_tx, mut derivation_rx) = mpsc::channel(16); + let derivation = QueuedEngineDerivationClient { derivation_actor_request_tx: derivation_tx }; + let (requests, request_rx) = mpsc::channel(8); + let (state_tx, _state_rx) = tokio::sync::watch::channel(state); + let mut actor = EngineActor::new( + Arc::new(client), + config, + derivation, + Engine::new(state, state_tx), + None, + request_rx, + Arc::new(NoopBlockSink), + ); + + for _ in 0..2 { + // The reset checks the unsafe head's L1 origin, the safe head's L1 block, and the safe + // head on the engine's chain, and keeps it without a forkchoice update. + for number in [100_u64, 105] { + let mut block = + alloy_rpc_types_eth::Block::::default(); + block.header.hash = l1_hash(number); + l1.expect_params( + "eth_getBlockByNumber", + serde_json::json!([format!("{number:#x}"), false]), + block, + ); + } + let mut block: ::BlockResponse = + Default::default(); + block.header.hash = safe_head.block_info.hash; + l2.expect_params("eth_getBlockByNumber", serde_json::json!(["0xa", false]), block); + + let (reply, mut reply_rx) = mpsc::channel(1); + let kind = ResetKind::AfterL1Reorg(vec![DerivedSafeHead { safe_head, derived_from }]); + requests + .send(EngineActorRequest::Reset(Box::new(ResetRequest { result_tx: reply, kind }))) + .await + .unwrap(); + actor.step().await.unwrap(); // retain the reset + // Perform it. The step then waits for a request that never comes. + let _ = time::timeout(Duration::from_secs(1), actor.step()).await; + reply_rx.recv().await.unwrap().unwrap(); + l1.assert_finished(); + l2.assert_finished(); + + // Before the first reset, the actor also reports its starting safe head and EL sync. + let mut received = Vec::new(); + while let Ok(request) = derivation_rx.try_recv() { + received.push(request); + } + let reset_at = received + .iter() + .position(|r| matches!(r, DerivationActorRequest::ProcessEngineSignalRequest(_))) + .unwrap_or_else(|| panic!("no reset signal in {received:?}")); + assert!( + matches!( + &received[reset_at..], + [ + DerivationActorRequest::ProcessEngineSignalRequest(_), + DerivationActorRequest::ProcessEngineSafeHeadUpdateRequest(head), + ] if **head == safe_head + ), + "got {received:?}" + ); + } +} diff --git a/rust/kona/crates/node/service/src/actors/mod.rs b/rust/kona/crates/node/service/src/actors/mod.rs index 478171bf6b5..5bf1aecc362 100644 --- a/rust/kona/crates/node/service/src/actors/mod.rs +++ b/rust/kona/crates/node/service/src/actors/mod.rs @@ -14,8 +14,8 @@ pub use clock::{Clock, SystemClock}; mod engine; pub use engine::{ BuildRequest, EngineActor, EngineActorRequest, EngineClientError, EngineClientResult, - EngineConfig, EngineDerivationClient, EngineError, QueuedEngineDerivationClient, ResetRequest, - SealRequest, + EngineConfig, EngineDerivationClient, EngineError, QueuedEngineDerivationClient, ResetKind, + ResetRequest, SealRequest, }; mod rpc; diff --git a/rust/kona/crates/node/service/src/actors/sequencer/engine_client.rs b/rust/kona/crates/node/service/src/actors/sequencer/engine_client.rs index e07babf85da..ddf16147b00 100644 --- a/rust/kona/crates/node/service/src/actors/sequencer/engine_client.rs +++ b/rust/kona/crates/node/service/src/actors/sequencer/engine_client.rs @@ -1,6 +1,6 @@ use crate::{ EngineClientError, EngineClientResult, - actors::engine::{BuildRequest, EngineActorRequest, ResetRequest, SealRequest}, + actors::engine::{BuildRequest, EngineActorRequest, ResetKind, ResetRequest, SealRequest}, }; use alloy_rpc_types_engine::PayloadId; use async_trait::async_trait; @@ -61,7 +61,10 @@ impl SequencerEngineClient for QueuedSequencerEngineClient { let (result_tx, mut result_rx) = mpsc::channel(1); self.engine_actor_request_tx - .send(EngineActorRequest::Reset(Box::new(ResetRequest { result_tx }))) + .send(EngineActorRequest::Reset(Box::new(ResetRequest { + result_tx, + kind: ResetKind::Full, + }))) .await .map_err(|_| EngineClientError::RequestError("request channel closed.".to_string()))?; diff --git a/rust/kona/crates/node/service/src/lib.rs b/rust/kona/crates/node/service/src/lib.rs index 0956c53a3bd..8031647fa04 100644 --- a/rust/kona/crates/node/service/src/lib.rs +++ b/rust/kona/crates/node/service/src/lib.rs @@ -29,9 +29,9 @@ pub use actors::{ NetworkDriverError, NetworkEngineClient, NetworkHandler, NodeActor, OriginSelector, QueuedDerivationEngineClient, QueuedEngineDerivationClient, QueuedL1WatcherDerivationClient, QueuedNetworkEngineClient, QueuedSequencerEngineClient, QueuedUnsafePayloadGossipClient, - ResetRequest, RpcActor, RpcActorError, SealRequest, SequencerActor, SequencerActorError, - SequencerConfig, SequencerEngineClient, SignedPayload, SignerActor, SignerActorError, - SystemClock, UnsafePayloadGossipClient, UnsafePayloadGossipClientError, + ResetKind, ResetRequest, RpcActor, RpcActorError, SealRequest, SequencerActor, + SequencerActorError, SequencerConfig, SequencerEngineClient, SignedPayload, SignerActor, + SignerActorError, SystemClock, UnsafePayloadGossipClient, UnsafePayloadGossipClientError, }; mod metrics; From 5ab40d43e6deffe915d0c94bed11a02d09d02501 Mon Sep 17 00:00:00 2001 From: Teddy Knox Date: Sat, 24 Oct 2026 16:58:14 +0200 Subject: [PATCH 2/8] kona-node: resume derivation itself after a reset that keeps its safe head The engine reports its safe head only when it changes, so derivation now resumes on its own when a reset keeps the safe head it has confirmed, instead of the engine resending an unchanged safe head. That removes the resend and the guard against it racing an L1 head. Also inlines the pending-reset merge rule and consolidates overlapping tests. Co-Authored-By: Claude Opus 5.5 --- rust/kona/crates/node/engine/src/request.rs | 30 --- .../crates/node/engine/src/task_queue/core.rs | 183 +++++++----------- .../service/src/actors/derivation/actor.rs | 28 +-- .../src/actors/derivation/actor/tests.rs | 46 ++--- .../actors/derivation/safe_head_history.rs | 12 -- .../node/service/src/actors/engine/actor.rs | 8 +- .../node/service/src/actors/engine/tests.rs | 93 +-------- 7 files changed, 118 insertions(+), 282 deletions(-) diff --git a/rust/kona/crates/node/engine/src/request.rs b/rust/kona/crates/node/engine/src/request.rs index 13372a59418..3b98ca20602 100644 --- a/rust/kona/crates/node/engine/src/request.rs +++ b/rust/kona/crates/node/engine/src/request.rs @@ -84,17 +84,6 @@ pub enum ResetKind { AfterL1Reorg(Vec), } -impl ResetKind { - /// The reset to perform when `next` is requested while `self` is still retained: a full reset - /// wins, and otherwise the newer request does. - pub fn merge(self, next: Self) -> Self { - match self { - Self::Full => Self::Full, - Self::AfterL1Reorg(_) => next, - } - } -} - /// A request to seal and canonicalize a payload. /// Contains the `PayloadId`, attributes, and a channel to send back the result. #[derive(Debug)] @@ -106,22 +95,3 @@ pub struct SealRequest { /// The channel on which the result, successful or not, will be sent. pub result_tx: mpsc::Sender>, } - -#[cfg(test)] -mod tests { - use super::ResetKind; - use crate::DerivedSafeHead; - - #[test] - fn a_full_reset_wins_and_otherwise_the_newer_request_does() { - let older = ResetKind::AfterL1Reorg(vec![]); - let newer = ResetKind::AfterL1Reorg(vec![DerivedSafeHead { - safe_head: Default::default(), - derived_from: Default::default(), - }]); - - assert_eq!(older.clone().merge(ResetKind::Full), ResetKind::Full); - assert_eq!(ResetKind::Full.merge(older.clone()), ResetKind::Full); - assert_eq!(older.merge(newer.clone()), newer); - } -} diff --git a/rust/kona/crates/node/engine/src/task_queue/core.rs b/rust/kona/crates/node/engine/src/task_queue/core.rs index a5315552c9a..8a254a78d6a 100644 --- a/rust/kona/crates/node/engine/src/task_queue/core.rs +++ b/rust/kona/crates/node/engine/src/task_queue/core.rs @@ -465,62 +465,6 @@ mod l1_reorg_reset_tests { (Engine::new(state, state_tx), Arc::new(client), l1, l2, config) } - /// Each case scripts exactly the lookups expected, in order: the unsafe head's L1 origin, - /// then each recent safe head's L1 block until one is canonical, then that safe head on the - /// engine's chain. - #[rstest] - #[case::tip_reorg_keeps_the_safe_head(108, UNSAFE_ORIGIN, &[UNSAFE_ORIGIN, 106], Some(20))] - #[case::reorg_of_the_safe_heads_l1_block_keeps_the_previous_one( - 105, UNSAFE_ORIGIN, &[UNSAFE_ORIGIN, 106, 103], Some(18) - )] - #[case::reorg_of_every_recent_safe_heads_l1_block( - 103, UNSAFE_ORIGIN, &[UNSAFE_ORIGIN, 106, 103], None - )] - // L2 block 20's L1 block 106 is still canonical; only the unsafe head's origin was replaced. - #[case::reorg_of_the_unsafe_heads_l1_origin(107, 107, &[107], None)] - #[tokio::test] - async fn finds_the_newest_safe_head_derived_from_canonical_l1( - #[case] reorg_height: u64, - #[case] unsafe_origin: u64, - #[case] l1_lookups: &[u64], - #[case] kept: Option, - ) { - let (engine, client, l1, l2, _) = setup(unsafe_origin); - for &number in l1_lookups { - expect_l1(&l1, number, reorg_height); - } - if let Some(kept) = kept { - expect_l2(&l2, kept, l2_block(kept, 0).block_info.hash); - } - - let safe_head = - find_reorg_safe_head(&client, engine.state(), &recent_safe_heads()).await.unwrap(); - - assert_eq!(safe_head.map(|head| head.block_info.number), kept); - l1.assert_finished(); - l2.assert_finished(); - } - - /// A safe head outside the engine's local-safe and finalized heads is not kept, even if its L1 - /// block is canonical. - #[rstest] - #[case::above_the_local_safe_head(25)] - #[case::below_the_finalized_head(FINALIZED - 1)] - #[tokio::test] - async fn does_not_keep_a_safe_head_outside_the_engines_safe_range(#[case] number: u64) { - let (engine, client, l1, l2, _) = setup(UNSAFE_ORIGIN); - expect_l1(&l1, UNSAFE_ORIGIN, 108); - expect_l1(&l1, 106, 108); - let candidate = - DerivedSafeHead { safe_head: l2_block(number, 99), derived_from: l1_id(106, 0) }; - - let safe_head = find_reorg_safe_head(&client, engine.state(), &[candidate]).await.unwrap(); - - assert_eq!(safe_head, None); - l1.assert_finished(); - l2.assert_finished(); - } - /// Scripts the forkchoice update that moves the safe labels, checking the heads it sends. fn expect_forkchoice_update(l2: &RpcMock, head: L2BlockInfo, safe: L2BlockInfo) { let state = alloy_rpc_types_engine::ForkchoiceState { @@ -535,83 +479,104 @@ mod l1_reorg_reset_tests { ); } - /// A safe head no longer on the engine's chain is not kept, even if its L1 block is canonical. - #[tokio::test] - async fn does_not_keep_a_safe_head_off_the_engines_chain() { - let (engine, client, l1, l2, _) = setup(UNSAFE_ORIGIN); - expect_l1(&l1, UNSAFE_ORIGIN, 108); - expect_l1(&l1, 106, 108); - expect_l2(&l2, 20, B256::repeat_byte(0xaa)); - - let safe_head = - find_reorg_safe_head(&client, engine.state(), &recent_safe_heads()).await.unwrap(); - - assert_eq!(safe_head, None); - l1.assert_finished(); - l2.assert_finished(); - } - + /// The reset keeps the newest recent safe head whose L1 block is canonical. Its lookups are + /// scripted in order: the unsafe head's L1 origin, each recent safe head's L1 block until one + /// is canonical, then that safe head on the engine's chain. Only the safe labels move, with a + /// forkchoice update only if they do; a cross-safe head already below the kept one stays. + #[rstest] + #[case::tip_reorg_keeps_the_safe_head(108, None, 20)] + #[case::reorg_of_its_l1_block_keeps_the_previous_one(105, None, 18)] + #[case::cross_safe_below_the_kept_one_stays(105, Some(15), 18)] #[tokio::test] - async fn tip_reorg_keeps_every_head_without_a_forkchoice_update() { + async fn reset_keeps_the_newest_safe_head_derived_from_canonical_l1( + #[case] reorg_height: u64, + #[case] cross_safe: Option, + #[case] kept: u64, + ) { let (mut engine, client, l1, l2, config) = setup(UNSAFE_ORIGIN); + if let Some(cross_safe) = cross_safe { + engine.state.sync_state = engine.state.sync_state.apply_update(EngineSyncStateUpdate { + safe_head: Some(l2_block(cross_safe, 98)), + ..Default::default() + }); + } let before = engine.state().sync_state; - expect_l1(&l1, UNSAFE_ORIGIN, 108); - expect_l1(&l1, 106, 108); - expect_l2(&l2, 20, l2_block(20, 100).block_info.hash); + let kept = recent_safe_heads() + .into_iter() + .map(|derived| derived.safe_head) + .find(|head| head.block_info.number == kept) + .unwrap(); + let safe_after = if cross_safe.is_some() { before.safe_head() } else { kept }; + expect_l1(&l1, UNSAFE_ORIGIN, reorg_height); + expect_l1(&l1, 106, reorg_height); + if reorg_height <= 106 { + expect_l1(&l1, 103, reorg_height); + } + expect_l2(&l2, kept.block_info.number, kept.block_info.hash); + if kept != before.local_safe_head() { + expect_forkchoice_update(&l2, before.unsafe_head(), safe_after); + } let safe_head = engine.reset_after_l1_reorg(client, config, &recent_safe_heads()).await.unwrap(); - assert_eq!(safe_head, l2_block(20, 100)); - assert_eq!(engine.state().sync_state, before); + assert_eq!(safe_head, kept); + let after = engine.state().sync_state; + assert_eq!(after.local_safe_head(), kept); + assert_eq!(after.safe_head(), safe_after); + assert_eq!(after.unsafe_head(), before.unsafe_head()); + assert_eq!(after.finalized_head(), before.finalized_head()); l1.assert_finished(); l2.assert_finished(); } + /// No recent safe head survives a reorg of every one's L1 block, and none is kept once the + /// unsafe head's L1 origin was replaced: there L2 block 20's L1 block 106 is still canonical. + #[rstest] + #[case::reorg_of_every_recent_safe_heads_l1_block(103, UNSAFE_ORIGIN, &[UNSAFE_ORIGIN, 106, 103])] + #[case::reorg_of_the_unsafe_heads_l1_origin(107, 107, &[107])] #[tokio::test] - async fn reorg_of_the_safe_heads_l1_block_rewinds_only_the_safe_heads() { - let (mut engine, client, l1, l2, config) = setup(UNSAFE_ORIGIN); - for number in [UNSAFE_ORIGIN, 106, 103] { - expect_l1(&l1, number, 105); + async fn finds_no_safe_head_to_keep( + #[case] reorg_height: u64, + #[case] unsafe_origin: u64, + #[case] l1_lookups: &[u64], + ) { + let (engine, client, l1, l2, _) = setup(unsafe_origin); + for &number in l1_lookups { + expect_l1(&l1, number, reorg_height); } - expect_l2(&l2, 18, l2_block(18, 99).block_info.hash); - expect_forkchoice_update(&l2, l2_block(UNSAFE, UNSAFE_ORIGIN), l2_block(18, 99)); let safe_head = - engine.reset_after_l1_reorg(client, config, &recent_safe_heads()).await.unwrap(); + find_reorg_safe_head(&client, engine.state(), &recent_safe_heads()).await.unwrap(); - assert_eq!(safe_head, l2_block(18, 99)); - let sync_state = engine.state().sync_state; - assert_eq!(sync_state.local_safe_head(), safe_head); - assert_eq!(sync_state.safe_head(), safe_head); - assert_eq!(sync_state.unsafe_head(), l2_block(UNSAFE, UNSAFE_ORIGIN)); - assert_eq!(sync_state.finalized_head(), l2_block(FINALIZED, 90)); + assert_eq!(safe_head, None); l1.assert_finished(); l2.assert_finished(); } - /// A cross-safe head already below the kept safe head stays where it is: only local-safe moves. + /// A safe head with a canonical L1 block is still not kept unless it is on the engine's chain, + /// between the finalized and local-safe heads. + #[rstest] + #[case::above_the_local_safe_head(25, None)] + #[case::below_the_finalized_head(FINALIZED - 1, None)] + #[case::off_the_engines_chain(20, Some(B256::repeat_byte(0xaa)))] #[tokio::test] - async fn a_cross_safe_head_below_the_kept_one_is_left_alone() { - let (mut engine, client, l1, l2, config) = setup(UNSAFE_ORIGIN); - let cross_safe = l2_block(15, 98); - engine.state.sync_state = engine.state.sync_state.apply_update(EngineSyncStateUpdate { - unsafe_head: None, - local_safe_head: None, - safe_head: Some(cross_safe), - finalized_head: None, - }); - for number in [UNSAFE_ORIGIN, 106, 103] { - expect_l1(&l1, number, 105); + async fn does_not_keep_a_safe_head_outside_the_engines_safe_chain( + #[case] number: u64, + #[case] engine_hash: Option, + ) { + let (engine, client, l1, l2, _) = setup(UNSAFE_ORIGIN); + expect_l1(&l1, UNSAFE_ORIGIN, 108); + expect_l1(&l1, 106, 108); + if let Some(hash) = engine_hash { + expect_l2(&l2, number, hash); } - expect_l2(&l2, 18, l2_block(18, 99).block_info.hash); - expect_forkchoice_update(&l2, l2_block(UNSAFE, UNSAFE_ORIGIN), cross_safe); + let candidate = + DerivedSafeHead { safe_head: l2_block(number, 99), derived_from: l1_id(106, 0) }; - let safe_head = - engine.reset_after_l1_reorg(client, config, &recent_safe_heads()).await.unwrap(); + let safe_head = find_reorg_safe_head(&client, engine.state(), &[candidate]).await.unwrap(); - assert_eq!(engine.state().sync_state.local_safe_head(), safe_head); - assert_eq!(engine.state().sync_state.safe_head(), cross_safe); + assert_eq!(safe_head, None); l1.assert_finished(); l2.assert_finished(); } diff --git a/rust/kona/crates/node/service/src/actors/derivation/actor.rs b/rust/kona/crates/node/service/src/actors/derivation/actor.rs index 90b662433e9..73f2ce4d265 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/actor.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/actor.rs @@ -244,6 +244,22 @@ where self.derivation_state_machine .update(&DerivationStateUpdate::SignalProcessed)?; } + // The engine reports its safe head only when it changes, so a reset that keeps + // the confirmed safe head, as one after a shallow L1 reorg usually does, gets no + // report. Resume from it instead of waiting for the next L1 head. + if let Signal::Reset(reset) = *signal && + reset.l2_safe_head == + self.derivation_state_machine.last_confirmed_safe_head() && + self.derivation_state_machine.current_state() == + DerivationState::AwaitingUpdateAfterSignal + { + self.derivation_state_machine.update( + &DerivationStateUpdate::NewAttributesConfirmed(Box::new( + reset.l2_safe_head, + )), + )?; + self.attempt_derivation().await?; + } } DerivationActorRequest::ProcessFinalizedL1Block(finalized_l1_block) => { // Attempt to finalize the block. If successful, notify engine. @@ -266,18 +282,6 @@ where } DerivationActorRequest::ProcessEngineSafeHeadUpdateRequest(safe_head) => { info!(target: "derivation", safe_head = ?*safe_head, "Received safe head from engine."); - // The engine reports its safe head after every reset, even when unchanged, and an - // L1 head can arrive between the reset signal and that report. Unless derivation - // is still waiting for it, a head it has already confirmed carries nothing new: it - // must not consume attributes awaiting confirmation or move the state machine. - if safe_head.block_info.hash == - self.derivation_state_machine.last_confirmed_safe_head().block_info.hash && - self.derivation_state_machine.current_state() != - DerivationState::AwaitingUpdateAfterSignal - { - debug!(target: "derivation", safe_head = ?*safe_head, "Ignoring an already confirmed safe head."); - return Ok(()); - } self.derivation_state_machine .update(&DerivationStateUpdate::NewAttributesConfirmed(safe_head.clone()))?; self.settle_derived_attributes(*safe_head)?; diff --git a/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs b/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs index 8c95355bd40..6d3a67f5bcb 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs @@ -546,40 +546,40 @@ async fn el_sync_completion_floors_the_history_at_the_sequencing_window() { node.derive_and_confirm_one_block().await; } -/// After a reset, the engine reports the safe head even when unchanged, and an L1 head can -/// arrive first and set derivation going. The late report must not consume the attributes -/// derivation sent meanwhile, or the engine's confirmation of them is lost. +/// The engine reports its safe head only when it changes. After a reset that keeps the confirmed +/// safe head, derivation resumes at once, deriving and recording the next block. After a reset +/// to another head, it waits for the engine to report that head. #[rstest] -#[case::attributes_awaiting_confirmation(true)] -#[case::awaiting_l1_data(false)] +#[case::keeps_the_confirmed_safe_head(DERIVED_HEAD, true)] +#[case::moves_the_safe_head(RESUME_HEAD, false)] #[tokio::test] -async fn a_late_report_of_the_kept_safe_head_is_ignored(#[case] derives_after_reset: bool) { +async fn a_reset_that_keeps_the_confirmed_safe_head_resumes_derivation( + #[case] reset_to: u64, + #[case] resumes: bool, +) { let mut node = RecordingActor::new(engine_client_accepting_attributes()); node.derive_and_confirm_one_block().await; - - // A reset that keeps the safe head, then an L1 head before the engine reports it. - node.signal(Signal::Reset(kona_derive::ResetSignal { l2_safe_head: l2_block(DERIVED_HEAD) })) - .await; - node.actor.pipeline.attributes.clear(); let next_l1 = DERIVED_FROM_L1 + 1; - if derives_after_reset { - node.actor.pipeline.attributes.push_back(OpAttributesWithParent { - attributes: OpPayloadAttributes::default(), - parent: l2_block(DERIVED_HEAD), - derived_from: Some(l1_block(next_l1)), - is_last_in_span: true, - }); - } - node.handle(DerivationActorRequest::ProcessL1HeadUpdateRequest(Box::default())).await; - node.confirm_safe_head(DERIVED_HEAD).await; + node.actor.pipeline.attributes.push_back(OpAttributesWithParent { + attributes: OpPayloadAttributes::default(), + parent: l2_block(reset_to), + derived_from: Some(l1_block(next_l1)), + is_last_in_span: true, + }); + + node.signal(Signal::Reset(kona_derive::ResetSignal { l2_safe_head: l2_block(reset_to) })).await; - if derives_after_reset { + if resumes { + assert_eq!( + node.actor.derivation_state_machine.current_state(), + DerivationState::AwaitingSafeHeadConfirmation + ); node.confirm_safe_head(DERIVED_HEAD + 1).await; assert_eq!(node.db.last_entry().unwrap(), record(next_l1, DERIVED_HEAD + 1)); } else { assert_eq!( node.actor.derivation_state_machine.current_state(), - DerivationState::AwaitingL1Data + DerivationState::AwaitingUpdateAfterSignal ); } } diff --git a/rust/kona/crates/node/service/src/actors/derivation/safe_head_history.rs b/rust/kona/crates/node/service/src/actors/derivation/safe_head_history.rs index ac65c79b520..8b173822e26 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/safe_head_history.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/safe_head_history.rs @@ -180,16 +180,4 @@ mod tests { record(&mut history, 13, 106); assert_eq!(recorded(&history), vec![(13, 106)]); } - - #[test] - fn reset_forgets_safe_heads_above_the_reset_point() { - let mut history = SafeHeadHistory::default(); - record(&mut history, 11, 100); - record(&mut history, 12, 101); - record(&mut history, 13, 102); - - history.reset_to(l2_block(12), SEQ_WINDOW_SIZE); - - assert_eq!(recorded(&history), vec![(12, 101), (11, 100)]); - } } diff --git a/rust/kona/crates/node/service/src/actors/engine/actor.rs b/rust/kona/crates/node/service/src/actors/engine/actor.rs index 8c7b46d119b..5176c7136f5 100644 --- a/rust/kona/crates/node/service/src/actors/engine/actor.rs +++ b/rust/kona/crates/node/service/src/actors/engine/actor.rs @@ -101,7 +101,9 @@ where /// Retains a reset of `kind`, combined with any reset already retained. fn begin_reset(&mut self, kind: ResetKind) { self.pending_reset = Some(match self.pending_reset.take() { - Some(pending) => pending.merge(kind), + // A pending full reset stays full; a pending reset after an L1 reorg is replaced. + Some(ResetKind::Full) => ResetKind::Full, + Some(ResetKind::AfterL1Reorg(_)) => kind, None => { self.engine.clear(); self.retry_at = Instant::now(); @@ -161,10 +163,6 @@ where } } - // Send the safe head even if the reset left it unchanged, as a reset after an L1 reorg - // usually does. Derivation waits after a reset for a safe head or a new L1 head, and - // drops the L1 heads that arrived while it awaited the reset. - self.last_engine_safe_head_sent = L2BlockInfo::default(); self.send_derivation_actor_safe_head_if_updated().await?; Ok(()) diff --git a/rust/kona/crates/node/service/src/actors/engine/tests.rs b/rust/kona/crates/node/service/src/actors/engine/tests.rs index 32c89655b97..eded3d472ca 100644 --- a/rust/kona/crates/node/service/src/actors/engine/tests.rs +++ b/rust/kona/crates/node/service/src/actors/engine/tests.rs @@ -3,8 +3,8 @@ use crate::{DerivationActorRequest, NodeActor}; use alloy_primitives::B256; use alloy_rpc_types_engine::{ForkchoiceUpdated, PayloadStatus, PayloadStatusEnum}; use kona_engine::{ - DerivedSafeHead, Engine, EngineActorRequest, EngineState, EngineSyncStateUpdate, - FinalizeBlockId, NoopBlockSink, ResetKind, ResetRequest, test_utils::test_engine_client, + Engine, EngineActorRequest, EngineState, EngineSyncStateUpdate, FinalizeBlockId, NoopBlockSink, + test_utils::test_engine_client, }; use kona_genesis::RollupConfig; use kona_protocol::{BlockInfo, L2BlockInfo}; @@ -171,92 +171,3 @@ async fn derivation_is_handed_the_local_safe_head() { }; assert_eq!(*sent, local_safe_head); } - -/// Derivation waits after a reset for a safe head or a new L1 head. A reset after an L1 reorg -/// usually keeps the safe head, and must still report it, or derivation stalls until L1 advances. -#[tokio::test(start_paused = true)] -async fn reset_after_l1_reorg_reports_an_unchanged_safe_head() { - let l1_hash = |number: u64| B256::repeat_byte(number as u8); - let safe_head = L2BlockInfo { - l1_origin: alloy_eips::BlockNumHash { number: 100, hash: l1_hash(100) }, - ..l2_block(10) - }; - let derived_from = alloy_eips::BlockNumHash { number: 105, hash: l1_hash(105) }; - let config = Arc::new(RollupConfig::default()); - let (client, l1, l2) = test_engine_client(config.clone()); - let state = EngineState { - sync_state: EngineState::default().sync_state.apply_update(EngineSyncStateUpdate { - unsafe_head: Some(safe_head), - local_safe_head: Some(safe_head), - safe_head: Some(safe_head), - finalized_head: Some(safe_head), - }), - el_sync_finished: true, - ..Default::default() - }; - let (derivation_tx, mut derivation_rx) = mpsc::channel(16); - let derivation = QueuedEngineDerivationClient { derivation_actor_request_tx: derivation_tx }; - let (requests, request_rx) = mpsc::channel(8); - let (state_tx, _state_rx) = tokio::sync::watch::channel(state); - let mut actor = EngineActor::new( - Arc::new(client), - config, - derivation, - Engine::new(state, state_tx), - None, - request_rx, - Arc::new(NoopBlockSink), - ); - - for _ in 0..2 { - // The reset checks the unsafe head's L1 origin, the safe head's L1 block, and the safe - // head on the engine's chain, and keeps it without a forkchoice update. - for number in [100_u64, 105] { - let mut block = - alloy_rpc_types_eth::Block::::default(); - block.header.hash = l1_hash(number); - l1.expect_params( - "eth_getBlockByNumber", - serde_json::json!([format!("{number:#x}"), false]), - block, - ); - } - let mut block: ::BlockResponse = - Default::default(); - block.header.hash = safe_head.block_info.hash; - l2.expect_params("eth_getBlockByNumber", serde_json::json!(["0xa", false]), block); - - let (reply, mut reply_rx) = mpsc::channel(1); - let kind = ResetKind::AfterL1Reorg(vec![DerivedSafeHead { safe_head, derived_from }]); - requests - .send(EngineActorRequest::Reset(Box::new(ResetRequest { result_tx: reply, kind }))) - .await - .unwrap(); - actor.step().await.unwrap(); // retain the reset - // Perform it. The step then waits for a request that never comes. - let _ = time::timeout(Duration::from_secs(1), actor.step()).await; - reply_rx.recv().await.unwrap().unwrap(); - l1.assert_finished(); - l2.assert_finished(); - - // Before the first reset, the actor also reports its starting safe head and EL sync. - let mut received = Vec::new(); - while let Ok(request) = derivation_rx.try_recv() { - received.push(request); - } - let reset_at = received - .iter() - .position(|r| matches!(r, DerivationActorRequest::ProcessEngineSignalRequest(_))) - .unwrap_or_else(|| panic!("no reset signal in {received:?}")); - assert!( - matches!( - &received[reset_at..], - [ - DerivationActorRequest::ProcessEngineSignalRequest(_), - DerivationActorRequest::ProcessEngineSafeHeadUpdateRequest(head), - ] if **head == safe_head - ), - "got {received:?}" - ); - } -} From eeef88ab0817668dfd9a9de85d46768dec3e301f Mon Sep 17 00:00:00 2001 From: Teddy Knox Date: Sun, 25 Oct 2026 23:19:51 +0100 Subject: [PATCH 3/8] kona-node: rewind only the unsafe chain when a reorg replaces its L1 origin A reorg that replaced the unsafe head's L1 origin fell back to the full reset. Sequencers pick origins only 4-5 L1 blocks behind the tip, so that covered every reorg deeper than about 5 blocks, the ones most likely during a finality stall. The engine now keeps the safe head it chooses and walks the unsafe chain back to its last block with a canonical L1 origin, never past that safe head. find_reorg_safe_head becomes find_reorg_forkchoice and returns both heads. Co-Authored-By: Claude Opus 5.5 --- rust/kona/crates/node/engine/src/lib.rs | 2 +- rust/kona/crates/node/engine/src/sync/mod.rs | 2 +- .../kona/crates/node/engine/src/sync/reorg.rs | 99 +++++-- .../crates/node/engine/src/task_queue/core.rs | 280 +++++++++++++----- .../actors/derivation/safe_head_history.rs | 2 +- 5 files changed, 287 insertions(+), 98 deletions(-) diff --git a/rust/kona/crates/node/engine/src/lib.rs b/rust/kona/crates/node/engine/src/lib.rs index 45eae8c70d7..98dd79f1efc 100644 --- a/rust/kona/crates/node/engine/src/lib.rs +++ b/rust/kona/crates/node/engine/src/lib.rs @@ -75,7 +75,7 @@ pub use metrics::Metrics; mod sync; pub use sync::{ - DerivedSafeHead, L2ForkchoiceState, SyncStartError, find_reorg_safe_head, + DerivedSafeHead, L2ForkchoiceState, SyncStartError, find_reorg_forkchoice, find_starting_forkchoice, }; diff --git a/rust/kona/crates/node/engine/src/sync/mod.rs b/rust/kona/crates/node/engine/src/sync/mod.rs index fb570a6a162..d15a4dc828d 100644 --- a/rust/kona/crates/node/engine/src/sync/mod.rs +++ b/rust/kona/crates/node/engine/src/sync/mod.rs @@ -10,7 +10,7 @@ mod error; pub use error::SyncStartError; mod reorg; -pub use reorg::{DerivedSafeHead, find_reorg_safe_head}; +pub use reorg::{DerivedSafeHead, find_reorg_forkchoice}; use tracing::info; diff --git a/rust/kona/crates/node/engine/src/sync/reorg.rs b/rust/kona/crates/node/engine/src/sync/reorg.rs index d1eb07c7f7f..aabce5c1ee3 100644 --- a/rust/kona/crates/node/engine/src/sync/reorg.rs +++ b/rust/kona/crates/node/engine/src/sync/reorg.rs @@ -1,4 +1,4 @@ -//! Choosing a safe head to keep across an L1 reorg. +//! Choosing the forkchoice to keep across an L1 reorg. //! //! [`find_starting_forkchoice`] cannot know which L1 block made the engine's safe head safe, so it //! walks the safe head back a full sequencing window, stopping early only at the finalized head. @@ -14,9 +14,15 @@ //! as sound as any reset: the pipeline restarts from a known-correct safe head, a channel timeout //! behind its L1 origin, as it does after [`find_starting_forkchoice`]. //! +//! The unsafe chain is checked on its own terms. An unsafe block whose L1 origin the reorg +//! replaced commits to a non-canonical L1 block and can never become safe, so the unsafe head +//! moves back to the highest block whose L1 origin is still canonical. That walk visits only the +//! blocks built on replaced L1 blocks, and never passes the kept safe head, whose L1 data is +//! canonical. +//! //! [`find_starting_forkchoice`]: crate::find_starting_forkchoice -use crate::{EngineClient, EngineState, SyncStartError}; +use crate::{EngineClient, EngineState, EngineSyncState, L2ForkchoiceState, SyncStartError}; use alloy_eips::{BlockId, BlockNumHash}; use kona_protocol::L2BlockInfo; use tracing::info; @@ -31,30 +37,44 @@ pub struct DerivedSafeHead { pub derived_from: BlockNumHash, } -/// Picks the safe head to keep after an L1 reorg, or `None` when only a full reset is sound. -/// -/// `candidates` must be ordered newest first. The first candidate whose `derived_from` is still -/// canonical is chosen. It is kept only if it is still on the engine's chain, at or below the -/// local safe head and at or above the finalized head. No candidate is kept if the unsafe head's -/// L1 origin was reorged out: [`find_starting_forkchoice`] must rewind that unsafe chain. +/// Picks the forkchoice to keep after an L1 reorg, or `None` when only a full reset is sound. /// -/// [`find_starting_forkchoice`]: crate::find_starting_forkchoice -pub async fn find_reorg_safe_head( +/// `candidates` must be ordered newest first. The safe head is the first candidate whose +/// `derived_from` is still canonical, kept only if it is still on the engine's chain, at or below +/// the local safe head and at or above the finalized head. The unsafe head is the highest block +/// between it and the current unsafe head whose L1 origin is still canonical. The finalized head +/// is unchanged. +pub async fn find_reorg_forkchoice( engine_client: &EngineClient, state: &EngineState, candidates: &[DerivedSafeHead], -) -> Result, SyncStartError> { +) -> Result, SyncStartError> { let sync_state = state.sync_state; - let unsafe_head = sync_state.unsafe_head(); - if !is_canonical_l1(engine_client, unsafe_head.l1_origin).await? { - info!( - target: "sync_start", - l1_origin = ?unsafe_head.l1_origin, - "Unsafe head's L1 origin is no longer canonical; a full reset is needed" - ); + let Some(safe) = find_reorg_safe_head(engine_client, &sync_state, candidates).await? else { return Ok(None); - } + }; + let Some(un_safe) = + find_reorg_unsafe_head(engine_client, sync_state.unsafe_head(), safe).await? + else { + return Ok(None); + }; + info!( + target: "sync_start", + safe_head = %safe.block_info.number, + unsafe_head = %un_safe.block_info.number, + "Keeping heads derived from canonical L1 data across the L1 reorg" + ); + Ok(Some(L2ForkchoiceState { un_safe, safe, finalized: sync_state.finalized_head() })) +} + +/// The first of `candidates` whose L1 data is canonical, if the engine can keep it as its safe +/// head. +async fn find_reorg_safe_head( + engine_client: &EngineClient, + sync_state: &EngineSyncState, + candidates: &[DerivedSafeHead], +) -> Result, SyncStartError> { for candidate in candidates { if !is_canonical_l1(engine_client, candidate.derived_from).await? { continue; @@ -71,20 +91,41 @@ pub async fn find_reorg_safe_head( .get_l2_block(BlockId::number(number)) .await? .is_some_and(|block| block.header.hash == safe_head.block_info.hash); - if !on_chain { - return Ok(None); + return Ok(on_chain.then_some(safe_head)); + } + + Ok(None) +} + +/// Walks back from `unsafe_head` to the highest block whose L1 origin is canonical, stopping at +/// `safe_head`. `None` if the walk does not lead to `safe_head`. +async fn find_reorg_unsafe_head( + engine_client: &EngineClient, + mut unsafe_head: L2BlockInfo, + safe_head: L2BlockInfo, +) -> Result, SyncStartError> { + let genesis = &engine_client.cfg().genesis; + // Consecutive blocks mostly share an L1 origin; check each replaced one only once. + let mut replaced_origin = None; + while unsafe_head.block_info.number > safe_head.block_info.number { + if replaced_origin != Some(unsafe_head.l1_origin) { + if is_canonical_l1(engine_client, unsafe_head.l1_origin).await? { + return Ok(Some(unsafe_head)); + } + replaced_origin = Some(unsafe_head.l1_origin); } - info!( - target: "sync_start", - safe_head = %number, - derived_from = ?candidate.derived_from, - "Keeping a safe head derived from canonical L1 data across the L1 reorg" - ); - return Ok(Some(safe_head)); + let parent_hash = unsafe_head.block_info.parent_hash; + if unsafe_head.block_info.number == safe_head.block_info.number + 1 { + return Ok((parent_hash == safe_head.block_info.hash).then_some(safe_head)); + } + let Some(parent) = engine_client.get_l2_block(parent_hash.into()).full().await? else { + return Ok(None); + }; + unsafe_head = L2BlockInfo::from_block_and_genesis(&parent.into_consensus(), genesis)?; } - Ok(None) + Ok((unsafe_head == safe_head).then_some(unsafe_head)) } /// Whether `block` is the canonical L1 block at its height. A block the L1 node does not have at diff --git a/rust/kona/crates/node/engine/src/task_queue/core.rs b/rust/kona/crates/node/engine/src/task_queue/core.rs index 8a254a78d6a..dc44e37f58a 100644 --- a/rust/kona/crates/node/engine/src/task_queue/core.rs +++ b/rust/kona/crates/node/engine/src/task_queue/core.rs @@ -3,8 +3,9 @@ use super::EngineTaskExt; use crate::{ DerivedSafeHead, EngineClient, EngineState, EngineSyncStateUpdate, EngineTask, EngineTaskError, - EngineTaskErrorSeverity, Metrics, SyncStartError, SynchronizeTask, SynchronizeTaskError, - find_reorg_safe_head, find_starting_forkchoice, task_queue::EngineTaskErrors, + EngineTaskErrorSeverity, L2ForkchoiceState, Metrics, SyncStartError, SynchronizeTask, + SynchronizeTaskError, find_reorg_forkchoice, find_starting_forkchoice, + task_queue::EngineTaskErrors, }; use kona_genesis::RollupConfig; use kona_protocol::L2BlockInfo; @@ -132,32 +133,33 @@ impl Engine { Ok(start.safe) } - /// Resets the engine after an L1 reorg, keeping the newest of `candidates` (ordered newest - /// first) that [`find_reorg_safe_head`] shows was derived from L1 data the reorg left - /// canonical. The unsafe head is kept, and the safe heads are rewound to the kept one if they - /// are above it. Falls back to [`Engine::reset`] when no candidate can be kept. + /// Resets the engine after an L1 reorg to the forkchoice [`find_reorg_forkchoice`] keeps: the + /// newest of `candidates` (ordered newest first) derived from L1 data the reorg left canonical, + /// and the unsafe chain back to its last block with a canonical L1 origin. The safe heads are + /// rewound to the kept one if they are above it. Falls back to [`Engine::reset`] when no + /// candidate can be kept. pub async fn reset_after_l1_reorg( &mut self, client: Arc, config: Arc, candidates: &[DerivedSafeHead], ) -> Result { - let Some(safe_head) = - find_reorg_safe_head(client.as_ref(), &self.state, candidates).await? + let Some(L2ForkchoiceState { un_safe, safe, .. }) = + find_reorg_forkchoice(client.as_ref(), &self.state, candidates).await? else { return self.reset(client, config).await; }; let sync_state = self.state.sync_state; - if safe_head != sync_state.local_safe_head() { - let above = |head: L2BlockInfo| head.block_info.number > safe_head.block_info.number; + if un_safe != sync_state.unsafe_head() || safe != sync_state.local_safe_head() { + let above = |head: L2BlockInfo| head.block_info.number > safe.block_info.number; let synchronize = SynchronizeTask::new( client, config, EngineSyncStateUpdate { - unsafe_head: None, - local_safe_head: Some(safe_head), - safe_head: above(sync_state.safe_head()).then_some(safe_head), + unsafe_head: (un_safe != sync_state.unsafe_head()).then_some(un_safe), + local_safe_head: Some(safe), + safe_head: above(sync_state.safe_head()).then_some(safe), finalized_head: None, }, ); @@ -166,7 +168,7 @@ impl Engine { metrics::counter!(Metrics::ENGINE_REORG_RESET_COUNT).increment(1); - Ok(safe_head) + Ok(safe) } /// Clears the task queue. @@ -387,12 +389,14 @@ mod l1_reorg_reset_tests { use super::*; use crate::test_utils::{RpcMock, test_engine_client}; use alloy_eips::BlockNumHash; - use alloy_primitives::B256; + use alloy_primitives::{Address, B256, U256}; use alloy_rpc_types_engine::{ForkchoiceUpdated, PayloadStatus, PayloadStatusEnum}; - use kona_protocol::BlockInfo; + use kona_protocol::{BlockInfo, L1BlockInfoBedrock}; use rstest::rstest; use serde_json::json; + type L2RpcBlock = ::BlockResponse; + /// L1 block `number` on `fork`: fork 0 is the chain derivation read, fork 1 replaced it from /// the reorg height on. fn l1_id(number: u64, fork: u8) -> BlockNumHash { @@ -424,15 +428,29 @@ mod l1_reorg_reset_tests { /// Scripts the engine's lookup of L2 block `number`, answered with `hash`. fn expect_l2(l2: &RpcMock, number: u64, hash: B256) { - let mut block: ::BlockResponse = - Default::default(); + let mut block = L2RpcBlock::default(); block.header.hash = hash; block.header.inner.number = number; l2.expect_params("eth_getBlockByNumber", json!([format!("{number:#x}"), false]), block); } + /// Scripts the forkchoice update that moves the heads, checking the heads it sends. + fn expect_forkchoice_update(l2: &RpcMock, head: L2BlockInfo, safe: L2BlockInfo) { + let state = alloy_rpc_types_engine::ForkchoiceState { + head_block_hash: head.block_info.hash, + safe_block_hash: safe.block_info.hash, + finalized_block_hash: l2_block(FINALIZED, 90).block_info.hash, + }; + l2.expect_params( + "engine_forkchoiceUpdatedV3", + json!([state, null]), + ForkchoiceUpdated::new(PayloadStatus::from_status(PayloadStatusEnum::Valid)), + ); + } + const FINALIZED: u64 = 5; const UNSAFE: u64 = 40; + /// Older than every reorg in the tests that keep the unsafe head. const UNSAFE_ORIGIN: u64 = 102; /// Derivation's recent safe heads, newest first: L2 block 20 depends on L1 data up to block @@ -444,10 +462,10 @@ mod l1_reorg_reset_tests { ] } - /// An engine at the local safe head L2 block 20 and the unsafe head L2 block 40, whose L1 - /// origin is `unsafe_origin`. - fn setup( - unsafe_origin: u64, + /// An engine with the given heads and finalized head [`FINALIZED`]. + fn engine_at( + unsafe_head: L2BlockInfo, + safe_head: L2BlockInfo, ) -> (Engine, Arc, RpcMock, RpcMock, Arc) { let mut config = RollupConfig::default(); config.hardforks.ecotone_time = Some(0); @@ -456,32 +474,24 @@ mod l1_reorg_reset_tests { let mut state = EngineState::default(); state.sync_state = state.sync_state.apply_update(EngineSyncStateUpdate { - unsafe_head: Some(l2_block(UNSAFE, unsafe_origin)), - local_safe_head: Some(l2_block(20, 100)), - safe_head: Some(l2_block(20, 100)), + unsafe_head: Some(unsafe_head), + local_safe_head: Some(safe_head), + safe_head: Some(safe_head), finalized_head: Some(l2_block(FINALIZED, 90)), }); let (state_tx, _) = tokio::sync::watch::channel(state); (Engine::new(state, state_tx), Arc::new(client), l1, l2, config) } - /// Scripts the forkchoice update that moves the safe labels, checking the heads it sends. - fn expect_forkchoice_update(l2: &RpcMock, head: L2BlockInfo, safe: L2BlockInfo) { - let state = alloy_rpc_types_engine::ForkchoiceState { - head_block_hash: head.block_info.hash, - safe_block_hash: safe.block_info.hash, - finalized_block_hash: l2_block(FINALIZED, 90).block_info.hash, - }; - l2.expect_params( - "engine_forkchoiceUpdatedV3", - json!([state, null]), - ForkchoiceUpdated::new(PayloadStatus::from_status(PayloadStatusEnum::Valid)), - ); + /// An engine at the local safe head L2 block 20 and the unsafe head L2 block 40, whose L1 + /// origin [`UNSAFE_ORIGIN`] every reorg below leaves canonical. + fn setup() -> (Engine, Arc, RpcMock, RpcMock, Arc) { + engine_at(l2_block(UNSAFE, UNSAFE_ORIGIN), l2_block(20, 100)) } /// The reset keeps the newest recent safe head whose L1 block is canonical. Its lookups are - /// scripted in order: the unsafe head's L1 origin, each recent safe head's L1 block until one - /// is canonical, then that safe head on the engine's chain. Only the safe labels move, with a + /// scripted in order: each recent safe head's L1 block until one is canonical, that safe head + /// on the engine's chain, then the unsafe head's L1 origin. Only the safe labels move, with a /// forkchoice update only if they do; a cross-safe head already below the kept one stays. #[rstest] #[case::tip_reorg_keeps_the_safe_head(108, None, 20)] @@ -493,7 +503,7 @@ mod l1_reorg_reset_tests { #[case] cross_safe: Option, #[case] kept: u64, ) { - let (mut engine, client, l1, l2, config) = setup(UNSAFE_ORIGIN); + let (mut engine, client, l1, l2, config) = setup(); if let Some(cross_safe) = cross_safe { engine.state.sync_state = engine.state.sync_state.apply_update(EngineSyncStateUpdate { safe_head: Some(l2_block(cross_safe, 98)), @@ -507,12 +517,12 @@ mod l1_reorg_reset_tests { .find(|head| head.block_info.number == kept) .unwrap(); let safe_after = if cross_safe.is_some() { before.safe_head() } else { kept }; - expect_l1(&l1, UNSAFE_ORIGIN, reorg_height); expect_l1(&l1, 106, reorg_height); if reorg_height <= 106 { expect_l1(&l1, 103, reorg_height); } expect_l2(&l2, kept.block_info.number, kept.block_info.hash); + expect_l1(&l1, UNSAFE_ORIGIN, reorg_height); if kept != before.local_safe_head() { expect_forkchoice_update(&l2, before.unsafe_head(), safe_after); } @@ -530,26 +540,17 @@ mod l1_reorg_reset_tests { l2.assert_finished(); } - /// No recent safe head survives a reorg of every one's L1 block, and none is kept once the - /// unsafe head's L1 origin was replaced: there L2 block 20's L1 block 106 is still canonical. - #[rstest] - #[case::reorg_of_every_recent_safe_heads_l1_block(103, UNSAFE_ORIGIN, &[UNSAFE_ORIGIN, 106, 103])] - #[case::reorg_of_the_unsafe_heads_l1_origin(107, 107, &[107])] + /// No recent safe head survives a reorg of every one's L1 block. #[tokio::test] - async fn finds_no_safe_head_to_keep( - #[case] reorg_height: u64, - #[case] unsafe_origin: u64, - #[case] l1_lookups: &[u64], - ) { - let (engine, client, l1, l2, _) = setup(unsafe_origin); - for &number in l1_lookups { - expect_l1(&l1, number, reorg_height); - } + async fn finds_nothing_to_keep_when_every_recent_safe_heads_l1_block_was_replaced() { + let (engine, client, l1, l2, _) = setup(); + expect_l1(&l1, 106, 103); + expect_l1(&l1, 103, 103); - let safe_head = - find_reorg_safe_head(&client, engine.state(), &recent_safe_heads()).await.unwrap(); + let forkchoice = + find_reorg_forkchoice(&client, engine.state(), &recent_safe_heads()).await.unwrap(); - assert_eq!(safe_head, None); + assert_eq!(forkchoice, None); l1.assert_finished(); l2.assert_finished(); } @@ -565,8 +566,7 @@ mod l1_reorg_reset_tests { #[case] number: u64, #[case] engine_hash: Option, ) { - let (engine, client, l1, l2, _) = setup(UNSAFE_ORIGIN); - expect_l1(&l1, UNSAFE_ORIGIN, 108); + let (engine, client, l1, l2, _) = setup(); expect_l1(&l1, 106, 108); if let Some(hash) = engine_hash { expect_l2(&l2, number, hash); @@ -574,9 +574,158 @@ mod l1_reorg_reset_tests { let candidate = DerivedSafeHead { safe_head: l2_block(number, 99), derived_from: l1_id(106, 0) }; - let safe_head = find_reorg_safe_head(&client, engine.state(), &[candidate]).await.unwrap(); + let forkchoice = + find_reorg_forkchoice(&client, engine.state(), &[candidate]).await.unwrap(); + + assert_eq!(forkchoice, None); + l1.assert_finished(); + l2.assert_finished(); + } + + /// A real L2 chain from block 18 to block 26, so the unsafe walk can fetch and decode its + /// blocks. Blocks 18 to 20 have L1 origins 99, 100 and 100, blocks 21 to 24 have L1 origins + /// 101 to 104, and blocks 25 and the unsafe head 26 share L1 origin 105. + fn real_chain() -> Vec<(L2BlockInfo, L2RpcBlock)> { + let mut chain: Vec<(L2BlockInfo, L2RpcBlock)> = Vec::new(); + for number in 18..=26 { + let origin = match number { + 18 => 99, + 19 | 20 => 100, + 25 | 26 => 105, + n => 80 + n, + }; + let info = L1BlockInfoBedrock::new( + origin, + 0, + 0, + l1_id(origin, 0).hash, + 0, + Address::ZERO, + U256::ZERO, + U256::ZERO, + ); + let deposit = op_alloy_consensus::OpTxEnvelope::Deposit(alloy_primitives::Sealed::new( + op_alloy_consensus::TxDeposit { + input: info.encode_calldata(), + ..Default::default() + }, + )); + let header = alloy_consensus::Header { + number, + parent_hash: chain.last().map_or(B256::ZERO, |(parent, _)| parent.block_info.hash), + timestamp: number, + ..Default::default() + }; + let consensus = alloy_consensus::Block:: { + header: header.clone(), + body: alloy_consensus::BlockBody { + transactions: vec![deposit.clone()], + ..Default::default() + }, + }; + let block_info = + L2BlockInfo::from_block_and_genesis(&consensus, &RollupConfig::default().genesis) + .unwrap(); + let rpc = L2RpcBlock { + header: alloy_rpc_types_eth::Header { + hash: block_info.block_info.hash, + inner: header, + ..Default::default() + }, + transactions: alloy_rpc_types_eth::BlockTransactions::Full(vec![ + op_alloy_rpc_types::Transaction { + inner: alloy_rpc_types_eth::Transaction { + inner: alloy_consensus::transaction::Recovered::new_unchecked( + deposit, + Address::ZERO, + ), + block_hash: Some(block_info.block_info.hash), + block_number: Some(number), + effective_gas_price: Some(0), + transaction_index: Some(0), + block_timestamp: None, + }, + deposit_nonce: None, + deposit_receipt_version: None, + }, + ]), + ..Default::default() + }; + chain.push((block_info, rpc)); + } + chain + } + + /// When the reorg replaced the unsafe head's L1 origin, the unsafe head moves back to the + /// highest block whose L1 origin is canonical, never below the kept safe head. Each walk step + /// fetches a parent by hash and checks its L1 origin, skipping an origin already found + /// replaced; the block just above the safe head is linked to it by its parent hash alone. + #[rstest] + #[case::tip_reorg(105, &[(20, 101), (18, 100)], 20, 24, &[101, 105, 104])] + #[case::deeper_reorg(103, &[(20, 101), (18, 100)], 20, 22, &[101, 105, 104, 103, 102])] + #[case::reorg_of_the_safe_heads_l1_data( + 101, &[(20, 101), (18, 100)], 18, 20, &[101, 100, 105, 104, 103, 102, 101, 100] + )] + #[case::back_to_the_safe_head(101, &[(20, 100)], 20, 20, &[100, 105, 104, 103, 102, 101])] + #[tokio::test] + async fn reset_walks_the_unsafe_chain_back_to_a_canonical_l1_origin( + #[case] reorg_height: u64, + #[case] candidates: &[(u64, u64)], + #[case] kept: u64, + #[case] unsafe_after: u64, + #[case] l1_lookups: &[u64], + ) { + let chain = real_chain(); + let block = |number: u64| chain.iter().find(|(info, _)| info.block_info.number == number); + let head = |number: u64| block(number).unwrap().0; + let (mut engine, client, l1, l2, config) = engine_at(head(26), head(20)); + let candidates: Vec<_> = candidates + .iter() + .map(|&(safe_head, derived_from)| DerivedSafeHead { + safe_head: head(safe_head), + derived_from: l1_id(derived_from, 0), + }) + .collect(); + for &number in l1_lookups { + expect_l1(&l1, number, reorg_height); + } + expect_l2(&l2, kept, head(kept).block_info.hash); + for number in (unsafe_after.max(kept + 1)..26).rev() { + let (info, rpc) = block(number).unwrap(); + l2.expect_params("eth_getBlockByHash", json!([info.block_info.hash, true]), rpc); + } + let safe_after = head(kept); + expect_forkchoice_update(&l2, head(unsafe_after), safe_after); + + let safe_head = engine.reset_after_l1_reorg(client, config, &candidates).await.unwrap(); - assert_eq!(safe_head, None); + assert_eq!(safe_head, safe_after); + let after = engine.state().sync_state; + assert_eq!(after.unsafe_head(), head(unsafe_after)); + assert_eq!(after.local_safe_head(), safe_after); + assert_eq!(after.safe_head(), safe_after); + l1.assert_finished(); + l2.assert_finished(); + } + + /// An unsafe chain that does not lead back to the kept safe head gives nothing to keep. + #[tokio::test] + async fn finds_nothing_to_keep_when_the_unsafe_chain_does_not_lead_to_the_safe_head() { + let chain = real_chain(); + let head = + |number: u64| chain.iter().find(|(info, _)| info.block_info.number == number).unwrap(); + let mut other_20 = head(20).0; + other_20.block_info.hash = B256::repeat_byte(0xaa); + let (engine, client, l1, l2, _) = engine_at(head(21).0, other_20); + expect_l1(&l1, 100, 101); + expect_l2(&l2, 20, other_20.block_info.hash); + expect_l1(&l1, 101, 101); + + let candidate = DerivedSafeHead { safe_head: other_20, derived_from: l1_id(100, 0) }; + let forkchoice = + find_reorg_forkchoice(&client, engine.state(), &[candidate]).await.unwrap(); + + assert_eq!(forkchoice, None); l1.assert_finished(); l2.assert_finished(); } @@ -585,10 +734,9 @@ mod l1_reorg_reset_tests { /// the engine's finalized head. #[tokio::test] async fn falls_back_to_the_full_walk() { - let (mut engine, client, l1, l2, config) = setup(UNSAFE_ORIGIN); - for number in [UNSAFE_ORIGIN, 106, 103] { - expect_l1(&l1, number, 103); - } + let (mut engine, client, l1, l2, config) = setup(); + expect_l1(&l1, 106, 103); + expect_l1(&l1, 103, 103); l2.expect_error("eth_getBlockByNumber"); let err = diff --git a/rust/kona/crates/node/service/src/actors/derivation/safe_head_history.rs b/rust/kona/crates/node/service/src/actors/derivation/safe_head_history.rs index 8b173822e26..92311d5c749 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/safe_head_history.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/safe_head_history.rs @@ -12,7 +12,7 @@ const MAX_ENTRIES: usize = 64; /// The safe heads derivation produced from the most recent L1 blocks, each with an L1 block that /// all of the L1 data it depends on is at or below. After an L1 reorg, the engine keeps the newest /// one whose L1 block the reorg left canonical, instead of rewinding a full sequencing window; -/// see [`kona_engine::find_reorg_safe_head`]. +/// see [`kona_engine::find_reorg_forkchoice`]. /// /// This is kept in memory whether or not the safe head database is enabled, and is bounded to /// the depth an L1 reorg can plausibly reach. From 12911b0b19f5f8ad45c16e032c098a87e87484e5 Mon Sep 17 00:00:00 2001 From: Teddy Knox Date: Mon, 26 Oct 2026 18:08:24 +0100 Subject: [PATCH 4/8] kona-node: record each safe head at one floor-corrected L1 block After a reset the pipeline replays from before the safe head's L1 origin and can produce the next block from an L1 block below the one its parent was recorded at. The in-memory SafeHeadHistory skipped such a block; the safe head database recorded it there, out of ascending L1 order, so it answered that block for L1 heights where its parent was not yet safe. The history now raises such a block to its parent's L1 block and returns that block, which the database records too, so the two records agree. Co-Authored-By: Claude Opus 5.5 --- .../service/src/actors/derivation/actor.rs | 23 ++-- .../src/actors/derivation/actor/tests.rs | 45 ++++++++ .../actors/derivation/safe_head_history.rs | 101 ++++++++++++------ 3 files changed, 126 insertions(+), 43 deletions(-) diff --git a/rust/kona/crates/node/service/src/actors/derivation/actor.rs b/rust/kona/crates/node/service/src/actors/derivation/actor.rs index 73f2ce4d265..36842b1801f 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/actor.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/actor.rs @@ -325,15 +325,20 @@ where ); return Ok(()); } - if let Some(derived_from) = attributes.derived_from { - self.safe_head_history.record(safe_head, derived_from.id()); - } - if let (Some(database), Some(derived_from)) = (&self.safe_db, attributes.derived_from) { - debug!(target: "derivation", l1_block = derived_from.number, safe_head = safe_head.block_info.number, "Recording safe head"); - database.safe_head_updated(safe_head, derived_from.id()).inspect_err(|error| { - error!(target: "derivation", %error, l1_block = derived_from.number, safe_head = safe_head.block_info.number, "Failed to record safe head"); - })?; - debug!(target: "derivation", l1_block = derived_from.number, safe_head = safe_head.block_info.number, "Recorded safe head"); + if let Some(derived_from) = attributes.derived_from.map(|block| block.id()) { + // Both records take the L1 block the history works out: the derivation source, + // raised to the L1 block the parent was recorded at, since a block cannot become safe + // before its parent. Below a floor known only as a bound, the history records nothing + // and the database keeps the derivation source. + let safe_at = + self.safe_head_history.record(safe_head, derived_from).unwrap_or(derived_from); + if let Some(database) = &self.safe_db { + debug!(target: "derivation", l1_block = safe_at.number, safe_head = safe_head.block_info.number, "Recording safe head"); + database.safe_head_updated(safe_head, safe_at).inspect_err(|error| { + error!(target: "derivation", %error, l1_block = safe_at.number, safe_head = safe_head.block_info.number, "Failed to record safe head"); + })?; + debug!(target: "derivation", l1_block = safe_at.number, safe_head = safe_head.block_info.number, "Recorded safe head"); + } } self.finalizer.enqueue_for_finalization(&attributes); Ok(()) diff --git a/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs b/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs index 6d3a67f5bcb..046d17ed2a3 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs @@ -584,6 +584,51 @@ async fn a_reset_that_keeps_the_confirmed_safe_head_resumes_derivation( } } +/// Attributes for the child of `parent`, derived from L1 block `derived_from`. +fn attributes_after(parent: u64, derived_from: u64) -> OpAttributesWithParent { + OpAttributesWithParent { + attributes: OpPayloadAttributes::default(), + parent: l2_block(parent), + derived_from: Some(l1_block(derived_from)), + is_last_in_span: true, + } +} + +/// After a reset, the pipeline replays from before the safe head's L1 origin and can produce the +/// next block from an L1 block below the one the safe head became safe at. That block became safe +/// only once its parent did, so the database records it at its parent's L1 block: updates stay +/// in ascending L1 order, and no query answers it for an L1 block where the parent was not safe. +#[tokio::test] +async fn replay_after_a_reset_records_no_safe_head_below_its_parents_l1_block() { + let mut node = RecordingActor::new(engine_client_accepting_attributes()); + node.derive_and_confirm_one_block().await; + let parent_l1 = DERIVED_FROM_L1 + 6; + node.actor.pipeline.attributes.push_back(attributes_after(DERIVED_HEAD, parent_l1)); + node.handle(DerivationActorRequest::ProcessL1HeadUpdateRequest(Box::default())).await; + node.confirm_safe_head(DERIVED_HEAD + 1).await; + assert_eq!(node.db.last_entry().unwrap(), record(parent_l1, DERIVED_HEAD + 1)); + + // A reset that keeps that safe head, then a replay that finds the next block's batch lower. + node.actor.pipeline.attributes.push_back(attributes_after(DERIVED_HEAD + 1, parent_l1 - 3)); + node.signal(Signal::Reset(kona_derive::ResetSignal { + l2_safe_head: l2_block(DERIVED_HEAD + 1), + })) + .await; + node.actor + .pipeline + .attributes + .retain(|attributes| attributes.parent == l2_block(DERIVED_HEAD + 1)); + node.confirm_safe_head(DERIVED_HEAD + 2).await; + + assert_eq!(node.db.last_entry().unwrap(), record(parent_l1, DERIVED_HEAD + 2)); + // Below the parent's L1 block, the safe head is still the grandparent. + assert_eq!( + node.db.safe_head_at_l1(parent_l1 - 1).unwrap(), + record(DERIVED_FROM_L1, DERIVED_HEAD) + ); + assert_eq!(node.db.l1_at_safe_head(DERIVED_HEAD + 1).unwrap().l1, l1_block(parent_l1).id()); +} + #[tokio::test] async fn a_confirmed_block_is_recorded_against_its_derivation_source() { let mut node = RecordingActor::new(engine_client_accepting_attributes()); diff --git a/rust/kona/crates/node/service/src/actors/derivation/safe_head_history.rs b/rust/kona/crates/node/service/src/actors/derivation/safe_head_history.rs index 92311d5c749..5fc55c84dd6 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/safe_head_history.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/safe_head_history.rs @@ -15,37 +15,68 @@ const MAX_ENTRIES: usize = 64; /// see [`kona_engine::find_reorg_forkchoice`]. /// /// This is kept in memory whether or not the safe head database is enabled, and is bounded to -/// the depth an L1 reorg can plausibly reach. +/// the depth an L1 reorg can plausibly reach. The database records each safe head at the L1 block +/// [`SafeHeadHistory::record`] returns, so the two never disagree. /// /// A safe head depends on the L1 data its own attributes were derived from, and on everything its /// parent depends on. The pipeline's origin bounds only the former: after a reset the pipeline /// replays from before the safe head's L1 origin, and can produce the next block from an L1 block -/// below the one its parent needed. So the history keeps a floor that is at least the L1 block -/// the current safe head depends on, and records a safe head only if its attributes came from at -/// or above the floor. +/// below the one its parent was recorded at, as when its batch landed first and was buffered. +/// So the history keeps a [`Floor`] that is at least the L1 block the current safe head depends +/// on, and records no safe head below it. #[derive(Debug, Default)] pub(crate) struct SafeHeadHistory { /// L1 block number -> the highest safe head produced with the pipeline at that L1 block. by_l1_block: BTreeMap, - /// The lowest L1 block a new record may claim: at least the L1 block the current safe head - /// depends on. - floor: u64, + /// The lowest L1 block a new record may claim. + floor: Floor, } -impl SafeHeadHistory { - /// Records `safe_head`, confirmed from attributes the pipeline produced at `derived_from`, if - /// that is at or above the floor. - pub(crate) fn record(&mut self, safe_head: L2BlockInfo, derived_from: BlockNumHash) { - if derived_from.number < self.floor { - // The safe head depends on no more than the floor, which stays put. - return; - } +/// The lowest L1 block a new record may claim: at least the L1 block the current safe head +/// depends on. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Floor { + /// The L1 block the current safe head was recorded at. A safe head derived below it cannot + /// have become safe before its parent, so it is recorded there. + Recorded(BlockNumHash), + /// Only a bound on the L1 blocks the current safe head depends on, with no block to record + /// at. A safe head derived below it is not recorded. + Bound(u64), +} - self.floor = derived_from.number; - self.by_l1_block.insert(derived_from.number, DerivedSafeHead { safe_head, derived_from }); +impl Default for Floor { + fn default() -> Self { + Self::Bound(0) + } +} + +impl SafeHeadHistory { + /// Records `safe_head`, confirmed from attributes the pipeline produced at `derived_from`, at + /// `derived_from` raised to the floor, and returns that L1 block. `None` when it is below a + /// floor known only as a bound, and so is not recorded. + /// + /// The returned block bounds the L1 data `safe_head` depends on, and is the earliest L1 block + /// it can have become safe at. Before Holocene that is where it became safe. After Holocene a + /// node that reset can derive it from a batch its first pass dropped as a future batch, and + /// the first pass took it from a later one. + pub(crate) fn record( + &mut self, + safe_head: L2BlockInfo, + derived_from: BlockNumHash, + ) -> Option { + let safe_at = match self.floor { + Floor::Recorded(floor) if derived_from.number < floor.number => floor, + Floor::Bound(bound) if derived_from.number < bound => return None, + _ => derived_from, + }; + + self.floor = Floor::Recorded(safe_at); + self.by_l1_block + .insert(safe_at.number, DerivedSafeHead { safe_head, derived_from: safe_at }); while self.by_l1_block.len() > MAX_ENTRIES { self.by_l1_block.pop_first(); } + Some(safe_at) } /// The recorded safe heads, newest first. @@ -55,20 +86,22 @@ impl SafeHeadHistory { /// Derivation restarts from `safe_head`: forgets the safe heads above it. /// - /// If `safe_head` is recorded, the floor is the L1 block it was recorded with. Otherwise - /// nothing kept is known to lead to it, so the history is cleared and the floor is the L1 - /// block by which derivation must have decided `safe_head`: one past the end of its epoch's - /// sequencing window. + /// If `safe_head` is recorded, the floor is the L1 block it was recorded at. Otherwise + /// nothing kept is known to lead to it, so the history is cleared and the floor is bounded by + /// the L1 block by which derivation must have decided `safe_head`: one past the end of its + /// epoch's sequencing window. pub(crate) fn reset_to(&mut self, safe_head: L2BlockInfo, seq_window_size: u64) { self.by_l1_block.retain(|_, derived| { derived.safe_head.block_info.number <= safe_head.block_info.number }); match self.by_l1_block.last_key_value() { - Some((&number, derived)) if derived.safe_head == safe_head => self.floor = number, + Some((_, derived)) if derived.safe_head == safe_head => { + self.floor = Floor::Recorded(derived.derived_from); + } _ => { self.by_l1_block.clear(); - self.floor = safe_head.l1_origin.number + seq_window_size + 1; + self.floor = Floor::Bound(safe_head.l1_origin.number + seq_window_size + 1); } } } @@ -98,8 +131,8 @@ mod tests { BlockNumHash { number, hash: B256::repeat_byte(number as u8) } } - fn record(history: &mut SafeHeadHistory, number: u64, derived_from: u64) { - history.record(l2_block(number), l1_block(derived_from)); + fn record(history: &mut SafeHeadHistory, number: u64, derived_from: u64) -> Option { + history.record(l2_block(number), l1_block(derived_from)).map(|safe_at| safe_at.number) } /// `(safe head number, L1 block number)` of each record, newest first. @@ -141,21 +174,21 @@ mod tests { /// After a reset to a recorded safe head, the pipeline replays from before its L1 origin and /// can produce the next block from an L1 block below the one that safe head was recorded - /// with. That block depends on its parent, so it must not be recorded as derived from the - /// lower L1 block, and must not overwrite the record already at that L1 block. + /// with. That block depends on its parent, so it became safe at its parent's L1 block, where + /// it is recorded instead. #[test] - fn replay_after_reset_does_not_record_below_the_kept_safe_head() { + fn replay_after_reset_records_at_the_kept_safe_heads_l1_block() { let mut history = SafeHeadHistory::default(); record(&mut history, 11, 100); record(&mut history, 12, 106); record(&mut history, 13, 107); history.reset_to(l2_block(12), SEQ_WINDOW_SIZE); - record(&mut history, 13, 100); - assert_eq!(recorded(&history), vec![(12, 106), (11, 100)]); + assert_eq!(record(&mut history, 13, 100), Some(106)); + assert_eq!(recorded(&history), vec![(13, 106), (11, 100)]); - record(&mut history, 14, 108); - assert_eq!(recorded(&history), vec![(14, 108), (12, 106), (11, 100)]); + assert_eq!(record(&mut history, 14, 108), Some(108)); + assert_eq!(recorded(&history), vec![(14, 108), (13, 106), (11, 100)]); } /// A reset to a safe head the history did not record, such as one from a full reset, leaves @@ -173,11 +206,11 @@ mod tests { assert!(history.newest_first().is_empty()); // Derived below the end of the sequencing window: not recorded. - record(&mut history, 12, 105); + assert_eq!(record(&mut history, 12, 105), None); assert!(history.newest_first().is_empty()); // From the floor on, recording resumes. - record(&mut history, 13, 106); + assert_eq!(record(&mut history, 13, 106), Some(106)); assert_eq!(recorded(&history), vec![(13, 106)]); } } From 60e4baa035c8bfb3bf480c297470f1c211d53461 Mon Sep 17 00:00:00 2001 From: Teddy Knox Date: Mon, 26 Oct 2026 18:45:38 +0100 Subject: [PATCH 5/8] kona-node: share the unsafe-chain walk and check L1 origins by number The cold-start reset kept an unsafe head whenever its L1 origin could be fetched by hash. An L1 node serves a block by hash after a reorg replaces it at the same height, so an unsafe head built on a replaced L1 block survived a cold start. find_starting_forkchoice and the L1-reorg reset now share one walk: it checks each origin by number, checks a replaced origin once, and never walks past a floor, the finalized head on a cold start and the kept safe head on an L1 reorg. Co-Authored-By: Claude Opus 5.5 --- .../kona/crates/node/engine/src/sync/error.rs | 16 ++ rust/kona/crates/node/engine/src/sync/mod.rs | 207 ++++++++++++++---- .../kona/crates/node/engine/src/sync/reorg.rs | 63 ++---- .../node/engine/src/sync/unsafe_head.rs | 90 ++++++++ .../crates/node/engine/src/task_queue/core.rs | 155 +++---------- .../node/engine/src/test_utils/chain.rs | 144 ++++++++++++ .../crates/node/engine/src/test_utils/mod.rs | 3 + .../node/service/src/actors/engine/tests.rs | 30 ++- 8 files changed, 481 insertions(+), 227 deletions(-) create mode 100644 rust/kona/crates/node/engine/src/sync/unsafe_head.rs create mode 100644 rust/kona/crates/node/engine/src/test_utils/chain.rs diff --git a/rust/kona/crates/node/engine/src/sync/error.rs b/rust/kona/crates/node/engine/src/sync/error.rs index d98b69d642b..f852bc171b5 100644 --- a/rust/kona/crates/node/engine/src/sync/error.rs +++ b/rust/kona/crates/node/engine/src/sync/error.rs @@ -29,6 +29,22 @@ pub enum SyncStartError { /// Finalized block mismatch #[error("Finalized block mismatch. Expected {0}, Got {1}")] MismatchedFinalizedBlock(B256, B256), + /// The unsafe chain does not lead back to a block it must descend from. + #[error("Unsafe chain does not lead back to {expected}; found {found}")] + UnsafeChainMismatch { + /// The block the unsafe chain must descend from. + expected: B256, + /// The block the unsafe chain has at that height. + found: B256, + }, + /// The unsafe head is below a block it must descend from. + #[error("Unsafe head {unsafe_head} is below block {floor}, which it must descend from")] + UnsafeHeadBelowFloor { + /// The unsafe head's number. + unsafe_head: u64, + /// The number of the block it must descend from. + floor: u64, + }, /// L1 origin mismatch. #[error("L1 origin mismatch")] L1OriginMismatch, diff --git a/rust/kona/crates/node/engine/src/sync/mod.rs b/rust/kona/crates/node/engine/src/sync/mod.rs index d15a4dc828d..9cfbf3a11ae 100644 --- a/rust/kona/crates/node/engine/src/sync/mod.rs +++ b/rust/kona/crates/node/engine/src/sync/mod.rs @@ -12,22 +12,21 @@ pub use error::SyncStartError; mod reorg; pub use reorg::{DerivedSafeHead, find_reorg_forkchoice}; +mod unsafe_head; +use unsafe_head::find_canonical_unsafe_head; + use tracing::info; use crate::EngineClient; /// Searches for the latest [`L2ForkchoiceState`] that we can use to start the sync process with. /// -/// - The *unsafe L2 block*: This is the highest L2 block whose L1 origin is a *plausible* -/// extension of the canonical L1 chain (as known to the rollup node). +/// - The *unsafe L2 block*: This is the highest L2 block, at or above the finalized one, whose L1 +/// origin is canonical: the L1 node has that block at that height. /// - The *safe L2 block*: This is the highest L2 block whose epoch's sequencing window is /// complete within the canonical L1 chain (as known to the rollup node). /// - The *finalized L2 block*: This is the L2 block which is known to be fully derived from /// finalized L1 block data. -/// -/// Plausible: meaning that the blockhash of the L2 block's L1 origin -/// (as reported in the L1 Attributes deposit within the L2 block) is not canonical at another -/// height in the L1 chain, and the same holds for all its ancestors. pub async fn find_starting_forkchoice( cfg: &RollupConfig, engine_client: &EngineClient, @@ -41,40 +40,15 @@ pub async fn find_starting_forkchoice( "Loaded current L2 EL forkchoice state" ); - // Search for the highest `unsafe` block, relative to the initial `unsafe` block's L1 origin, - loop { - let l1_origin = - engine_client.get_l1_block(current_fc.un_safe.l1_origin.hash.into()).await?; - info!( - target: "sync_start", - l1_origin = %current_fc.un_safe.l1_origin.number, - l2_unsafe = %current_fc.un_safe.block_info.number, - "Searching for L2 unsafe block with canonical L1 origin" - ); - - match l1_origin { - Some(_) => { - // Unsafe block has existing L1 origin. Continue with this head. - info!( - target: "sync_start", - l2_unsafe = %current_fc.un_safe.block_info.number, - "Found L2 unsafe block with canonical L1 origin" - ); - break; - } - None => { - let l2_parent_hash = current_fc.un_safe.block_info.parent_hash.into(); - let l2_parent = engine_client - .get_l2_block(l2_parent_hash) - .full() - .await? - .ok_or(SyncStartError::BlockNotFound(l2_parent_hash))?; - - current_fc.un_safe = - L2BlockInfo::from_block_and_genesis(&l2_parent.into_consensus(), &cfg.genesis)?; - } - } - } + // Search for the highest `unsafe` block whose L1 origin is canonical. The finalized head's + // is, so the walk stops there at the latest. + current_fc.un_safe = + find_canonical_unsafe_head(engine_client, current_fc.un_safe, current_fc.finalized).await?; + info!( + target: "sync_start", + l2_unsafe = %current_fc.un_safe.block_info.number, + "Found L2 unsafe block with canonical L1 origin" + ); // Search for the highest `safe` block that's L1 origin is at least older than the sequencing // window, relative to the L1 origin of the `unsafe` block. @@ -149,3 +123,156 @@ mod test { assert_eq!(rpc_reported_hash, l2_block_info.block_info.hash); } } + +#[cfg(test)] +mod cold_start_tests { + use super::*; + use crate::{ + EngineClient, EngineResetError, + test_utils::{L2RpcBlock, RpcMock, TestChain, expect_l1_block, test_engine_client}, + }; + use rstest::rstest; + use serde_json::json; + use std::sync::Arc; + + /// The test chain, and an engine client over scripted L1 and L2 nodes. The sequencing window + /// is long enough that the safe head search always stops at the finalized head. + fn setup() -> (TestChain, Arc, EngineClient, RpcMock, RpcMock) { + let config = Arc::new(RollupConfig { seq_window_size: 1000, ..Default::default() }); + let (client, l1, l2) = test_engine_client(config.clone()); + (TestChain::new(), config, client, l1, l2) + } + + /// How the L1 node answers for L1 block 105, the unsafe head's L1 origin. + #[derive(Debug, Clone, Copy)] + enum Origin { + Canonical, + ReplacedAtTheSameHeight, + NotSeenYet, + } + + /// A cold start keeps the unsafe head only if the L1 node has its L1 origin at that height. + /// Otherwise it walks back to the highest block with a canonical L1 origin: the replaced + /// origin's other block is still served by hash, so finding it by hash would keep a head + /// that can never become safe. Blocks 25 and 26 share L1 origin 105, checked once; block 24's + /// is 104. The safe head is then searched down to the finalized block 18. + #[rstest] + #[case::canonical(Origin::Canonical, 26)] + #[case::replaced_at_the_same_height(Origin::ReplacedAtTheSameHeight, 24)] + #[case::not_seen_yet(Origin::NotSeenYet, 24)] + #[tokio::test] + async fn cold_start_keeps_only_an_unsafe_head_with_a_canonical_l1_origin( + #[case] origin: Origin, + #[case] unsafe_head: u64, + ) { + let (chain, config, client, l1, l2) = setup(); + chain.expect_heads(&l2, 18, 20, 26); + match origin { + Origin::Canonical => expect_l1_block(&l1, 105, 106), + Origin::ReplacedAtTheSameHeight => expect_l1_block(&l1, 105, 105), + Origin::NotSeenYet => l1.expect_params( + "eth_getBlockByNumber", + json!(["0x69", false]), + Option::::None, + ), + } + if unsafe_head < 26 { + chain.expect_by_hash(&l2, 25); + chain.expect_by_hash(&l2, 24); + expect_l1_block(&l1, 104, 105); + } + for number in (18..unsafe_head).rev() { + chain.expect_by_hash(&l2, number); + } + + let forkchoice = find_starting_forkchoice(&config, &client).await.unwrap(); + + assert_eq!(forkchoice.un_safe, chain.info(unsafe_head)); + assert_eq!(forkchoice.safe, chain.info(18)); + assert_eq!(forkchoice.finalized, chain.info(18)); + l1.assert_finished(); + l2.assert_finished(); + } + + /// Block 18 with a different header, so a different block at the finalized height. + fn other_18(chain: &TestChain) -> L2RpcBlock { + let mut block = chain.rpc(18).clone(); + block.header.inner.timestamp += 1; + block.header.hash = block.header.inner.hash_slow(); + block + } + + /// The walk never passes the finalized head. With every L1 origin above it replaced, the + /// unsafe head becomes the finalized head, which the block above it must link to; an unsafe + /// chain that does not lead there fails the reset for good. + #[rstest] + #[case::leads_to_the_finalized_head(false)] + #[case::does_not_lead_to_the_finalized_head(true)] + #[tokio::test] + async fn cold_start_walks_back_no_further_than_the_finalized_head( + #[case] other_finalized: bool, + ) { + let (chain, config, client, l1, l2) = setup(); + let finalized = if other_finalized { other_18(&chain) } else { chain.rpc(18).clone() }; + l2.expect_params("eth_getBlockByNumber", json!(["finalized", true]), &finalized); + l2.expect_params("eth_getBlockByNumber", json!(["safe", true]), chain.rpc(20)); + l2.expect_params("eth_getBlockByNumber", json!(["latest", true]), chain.rpc(26)); + for number in [105, 104, 103, 102, 101, 100] { + expect_l1_block(&l1, number, 100); + } + for number in (19..26).rev() { + chain.expect_by_hash(&l2, number); + } + + let forkchoice = find_starting_forkchoice(&config, &client).await; + + if other_finalized { + let err = forkchoice.unwrap_err(); + assert!( + matches!(err, SyncStartError::UnsafeChainMismatch { found, .. } if found == chain.info(18).block_info.hash), + "got {err:?}" + ); + assert!(!EngineResetError::SyncStart(err).is_temporary()); + } else { + let forkchoice = forkchoice.unwrap(); + assert_eq!(forkchoice.un_safe, chain.info(18)); + assert_eq!(forkchoice.safe, chain.info(18)); + } + l1.assert_finished(); + l2.assert_finished(); + } + + /// An unsafe head at or below the finalized head needs no L1 lookup: at the finalized head it + /// is kept, and as another block at that height or below it, the reset fails for good. + #[rstest] + #[case::the_finalized_head(18, false)] + #[case::another_block_at_its_height(18, true)] + #[case::below_it(20, false)] + #[tokio::test] + async fn cold_start_checks_an_unsafe_head_at_or_below_the_finalized_head( + #[case] finalized: u64, + #[case] other_unsafe: bool, + ) { + let (chain, config, client, l1, l2) = setup(); + let unsafe_head = if other_unsafe { other_18(&chain) } else { chain.rpc(18).clone() }; + l2.expect_params("eth_getBlockByNumber", json!(["finalized", true]), chain.rpc(finalized)); + l2.expect_params("eth_getBlockByNumber", json!(["safe", true]), chain.rpc(finalized)); + l2.expect_params("eth_getBlockByNumber", json!(["latest", true]), &unsafe_head); + + let forkchoice = find_starting_forkchoice(&config, &client).await; + + match (finalized, other_unsafe) { + (18, false) => assert_eq!(forkchoice.unwrap().un_safe, chain.info(18)), + (18, true) => assert!(matches!( + forkchoice.unwrap_err(), + SyncStartError::UnsafeChainMismatch { expected, .. } if expected == chain.info(18).block_info.hash + )), + _ => assert!(matches!( + forkchoice.unwrap_err(), + SyncStartError::UnsafeHeadBelowFloor { unsafe_head: 18, floor: 20 } + )), + } + l1.assert_finished(); + l2.assert_finished(); + } +} diff --git a/rust/kona/crates/node/engine/src/sync/reorg.rs b/rust/kona/crates/node/engine/src/sync/reorg.rs index aabce5c1ee3..0ba28ac3e91 100644 --- a/rust/kona/crates/node/engine/src/sync/reorg.rs +++ b/rust/kona/crates/node/engine/src/sync/reorg.rs @@ -22,6 +22,7 @@ //! //! [`find_starting_forkchoice`]: crate::find_starting_forkchoice +use super::unsafe_head::{find_canonical_unsafe_head, is_canonical_l1}; use crate::{EngineClient, EngineState, EngineSyncState, L2ForkchoiceState, SyncStartError}; use alloy_eips::{BlockId, BlockNumHash}; use kona_protocol::L2BlockInfo; @@ -53,11 +54,20 @@ pub async fn find_reorg_forkchoice( let Some(safe) = find_reorg_safe_head(engine_client, &sync_state, candidates).await? else { return Ok(None); }; - let Some(un_safe) = - find_reorg_unsafe_head(engine_client, sync_state.unsafe_head(), safe).await? - else { - return Ok(None); - }; + let un_safe = + match find_canonical_unsafe_head(engine_client, sync_state.unsafe_head(), safe).await { + Ok(un_safe) => un_safe, + // An unsafe chain that misses a block or does not lead to the kept safe head is left to + // the full reset. + Err( + SyncStartError::BlockNotFound(_) | + SyncStartError::UnsafeChainMismatch { .. } | + SyncStartError::UnsafeHeadBelowFloor { .. }, + ) => { + return Ok(None); + } + Err(err) => return Err(err), + }; info!( target: "sync_start", @@ -96,46 +106,3 @@ async fn find_reorg_safe_head( Ok(None) } - -/// Walks back from `unsafe_head` to the highest block whose L1 origin is canonical, stopping at -/// `safe_head`. `None` if the walk does not lead to `safe_head`. -async fn find_reorg_unsafe_head( - engine_client: &EngineClient, - mut unsafe_head: L2BlockInfo, - safe_head: L2BlockInfo, -) -> Result, SyncStartError> { - let genesis = &engine_client.cfg().genesis; - // Consecutive blocks mostly share an L1 origin; check each replaced one only once. - let mut replaced_origin = None; - while unsafe_head.block_info.number > safe_head.block_info.number { - if replaced_origin != Some(unsafe_head.l1_origin) { - if is_canonical_l1(engine_client, unsafe_head.l1_origin).await? { - return Ok(Some(unsafe_head)); - } - replaced_origin = Some(unsafe_head.l1_origin); - } - - let parent_hash = unsafe_head.block_info.parent_hash; - if unsafe_head.block_info.number == safe_head.block_info.number + 1 { - return Ok((parent_hash == safe_head.block_info.hash).then_some(safe_head)); - } - let Some(parent) = engine_client.get_l2_block(parent_hash.into()).full().await? else { - return Ok(None); - }; - unsafe_head = L2BlockInfo::from_block_and_genesis(&parent.into_consensus(), genesis)?; - } - - Ok((unsafe_head == safe_head).then_some(unsafe_head)) -} - -/// Whether `block` is the canonical L1 block at its height. A block the L1 node does not have at -/// that height yet is not known to be canonical. -async fn is_canonical_l1( - engine_client: &EngineClient, - block: BlockNumHash, -) -> Result { - Ok(engine_client - .get_l1_block(BlockId::number(block.number)) - .await? - .is_some_and(|canonical| canonical.header.hash == block.hash)) -} diff --git a/rust/kona/crates/node/engine/src/sync/unsafe_head.rs b/rust/kona/crates/node/engine/src/sync/unsafe_head.rs new file mode 100644 index 00000000000..915718e6ff2 --- /dev/null +++ b/rust/kona/crates/node/engine/src/sync/unsafe_head.rs @@ -0,0 +1,90 @@ +//! Finding the highest unsafe block whose L1 origin is canonical. + +use crate::{EngineClient, SyncStartError}; +use alloy_eips::{BlockId, BlockNumHash}; +use kona_protocol::L2BlockInfo; +use tracing::info; + +/// Walks back from `unsafe_head` to the highest block whose L1 origin is canonical, never below +/// `floor`, a block whose L1 data is canonical. +/// +/// An L1 origin is canonical when the L1 node has that block at that height. A block built on an +/// origin the L1 node has replaced cannot become safe. One built on an origin the L1 node has not +/// seen yet is not known to be canonical, and is walked back from too; op-node keeps such a head +/// as plausible. The walk visits only blocks built on such origins, checking each origin once, +/// and links the block just above `floor` to it by its parent hash. +/// +/// Fails with [`SyncStartError::UnsafeHeadBelowFloor`] if `unsafe_head` is below `floor`, with +/// [`SyncStartError::BlockNotFound`] if the engine lacks a block on the way, and with +/// [`SyncStartError::UnsafeChainMismatch`] if the chain does not lead back to `floor`. +pub(super) async fn find_canonical_unsafe_head( + engine_client: &EngineClient, + mut unsafe_head: L2BlockInfo, + floor: L2BlockInfo, +) -> Result { + let (head, floor_number) = (unsafe_head.block_info, floor.block_info.number); + if head.number < floor_number { + return Err(SyncStartError::UnsafeHeadBelowFloor { + unsafe_head: head.number, + floor: floor_number, + }); + } + if head.number == floor_number { + return if head.hash == floor.block_info.hash { + Ok(floor) + } else { + Err(SyncStartError::UnsafeChainMismatch { + expected: floor.block_info.hash, + found: head.hash, + }) + }; + } + + let genesis = &engine_client.cfg().genesis; + // Consecutive blocks mostly share an L1 origin; check each replaced one only once. + let mut replaced_origin = None; + loop { + if replaced_origin != Some(unsafe_head.l1_origin) { + if is_canonical_l1(engine_client, unsafe_head.l1_origin).await? { + return Ok(unsafe_head); + } + info!( + target: "sync_start", + l2_unsafe = %unsafe_head.block_info.number, + l1_origin = ?unsafe_head.l1_origin, + "Unsafe block's L1 origin is not canonical; walking back" + ); + replaced_origin = Some(unsafe_head.l1_origin); + } + + let parent_hash = unsafe_head.block_info.parent_hash; + if unsafe_head.block_info.number == floor_number + 1 { + return if parent_hash == floor.block_info.hash { + Ok(floor) + } else { + Err(SyncStartError::UnsafeChainMismatch { + expected: floor.block_info.hash, + found: parent_hash, + }) + }; + } + let parent = engine_client + .get_l2_block(parent_hash.into()) + .full() + .await? + .ok_or(SyncStartError::BlockNotFound(parent_hash.into()))?; + unsafe_head = L2BlockInfo::from_block_and_genesis(&parent.into_consensus(), genesis)?; + } +} + +/// Whether `block` is the canonical L1 block at its height. A block the L1 node does not have at +/// that height yet is not known to be canonical. +pub(super) async fn is_canonical_l1( + engine_client: &EngineClient, + block: BlockNumHash, +) -> Result { + Ok(engine_client + .get_l1_block(BlockId::number(block.number)) + .await? + .is_some_and(|canonical| canonical.header.hash == block.hash)) +} diff --git a/rust/kona/crates/node/engine/src/task_queue/core.rs b/rust/kona/crates/node/engine/src/task_queue/core.rs index dc44e37f58a..864e1202102 100644 --- a/rust/kona/crates/node/engine/src/task_queue/core.rs +++ b/rust/kona/crates/node/engine/src/task_queue/core.rs @@ -387,22 +387,15 @@ mod recovery_tests { #[cfg(test)] mod l1_reorg_reset_tests { use super::*; - use crate::test_utils::{RpcMock, test_engine_client}; - use alloy_eips::BlockNumHash; - use alloy_primitives::{Address, B256, U256}; + use crate::test_utils::{ + L2RpcBlock, RpcMock, TestChain, expect_l1_block, test_engine_client, test_l1_block, + }; + use alloy_primitives::B256; use alloy_rpc_types_engine::{ForkchoiceUpdated, PayloadStatus, PayloadStatusEnum}; - use kona_protocol::{BlockInfo, L1BlockInfoBedrock}; + use kona_protocol::BlockInfo; use rstest::rstest; use serde_json::json; - type L2RpcBlock = ::BlockResponse; - - /// L1 block `number` on `fork`: fork 0 is the chain derivation read, fork 1 replaced it from - /// the reorg height on. - fn l1_id(number: u64, fork: u8) -> BlockNumHash { - BlockNumHash { number, hash: B256::left_padding_from(&[fork, number as u8]) } - } - fn l2_block(number: u64, l1_origin: u64) -> L2BlockInfo { let hash = |n: u64| B256::left_padding_from(&[0xee, n as u8]); L2BlockInfo { @@ -412,20 +405,11 @@ mod l1_reorg_reset_tests { parent_hash: hash(number - 1), timestamp: number, }, - l1_origin: l1_id(l1_origin, 0), + l1_origin: test_l1_block(l1_origin, 0), seq_num: 0, } } - /// Scripts the L1 lookup of block `number`, answered from the chain reorged at - /// `reorg_height`. - fn expect_l1(l1: &RpcMock, number: u64, reorg_height: u64) { - let mut block = alloy_rpc_types_eth::Block::::default(); - block.header.hash = l1_id(number, u8::from(number >= reorg_height)).hash; - block.header.inner.number = number; - l1.expect_params("eth_getBlockByNumber", json!([format!("{number:#x}"), false]), block); - } - /// Scripts the engine's lookup of L2 block `number`, answered with `hash`. fn expect_l2(l2: &RpcMock, number: u64, hash: B256) { let mut block = L2RpcBlock::default(); @@ -457,8 +441,8 @@ mod l1_reorg_reset_tests { /// 106, and L2 block 18 on L1 data up to block 103. fn recent_safe_heads() -> [DerivedSafeHead; 2] { [ - DerivedSafeHead { safe_head: l2_block(20, 100), derived_from: l1_id(106, 0) }, - DerivedSafeHead { safe_head: l2_block(18, 99), derived_from: l1_id(103, 0) }, + DerivedSafeHead { safe_head: l2_block(20, 100), derived_from: test_l1_block(106, 0) }, + DerivedSafeHead { safe_head: l2_block(18, 99), derived_from: test_l1_block(103, 0) }, ] } @@ -517,12 +501,12 @@ mod l1_reorg_reset_tests { .find(|head| head.block_info.number == kept) .unwrap(); let safe_after = if cross_safe.is_some() { before.safe_head() } else { kept }; - expect_l1(&l1, 106, reorg_height); + expect_l1_block(&l1, 106, reorg_height); if reorg_height <= 106 { - expect_l1(&l1, 103, reorg_height); + expect_l1_block(&l1, 103, reorg_height); } expect_l2(&l2, kept.block_info.number, kept.block_info.hash); - expect_l1(&l1, UNSAFE_ORIGIN, reorg_height); + expect_l1_block(&l1, UNSAFE_ORIGIN, reorg_height); if kept != before.local_safe_head() { expect_forkchoice_update(&l2, before.unsafe_head(), safe_after); } @@ -544,8 +528,8 @@ mod l1_reorg_reset_tests { #[tokio::test] async fn finds_nothing_to_keep_when_every_recent_safe_heads_l1_block_was_replaced() { let (engine, client, l1, l2, _) = setup(); - expect_l1(&l1, 106, 103); - expect_l1(&l1, 103, 103); + expect_l1_block(&l1, 106, 103); + expect_l1_block(&l1, 103, 103); let forkchoice = find_reorg_forkchoice(&client, engine.state(), &recent_safe_heads()).await.unwrap(); @@ -567,12 +551,14 @@ mod l1_reorg_reset_tests { #[case] engine_hash: Option, ) { let (engine, client, l1, l2, _) = setup(); - expect_l1(&l1, 106, 108); + expect_l1_block(&l1, 106, 108); if let Some(hash) = engine_hash { expect_l2(&l2, number, hash); } - let candidate = - DerivedSafeHead { safe_head: l2_block(number, 99), derived_from: l1_id(106, 0) }; + let candidate = DerivedSafeHead { + safe_head: l2_block(number, 99), + derived_from: test_l1_block(106, 0), + }; let forkchoice = find_reorg_forkchoice(&client, engine.state(), &[candidate]).await.unwrap(); @@ -582,80 +568,6 @@ mod l1_reorg_reset_tests { l2.assert_finished(); } - /// A real L2 chain from block 18 to block 26, so the unsafe walk can fetch and decode its - /// blocks. Blocks 18 to 20 have L1 origins 99, 100 and 100, blocks 21 to 24 have L1 origins - /// 101 to 104, and blocks 25 and the unsafe head 26 share L1 origin 105. - fn real_chain() -> Vec<(L2BlockInfo, L2RpcBlock)> { - let mut chain: Vec<(L2BlockInfo, L2RpcBlock)> = Vec::new(); - for number in 18..=26 { - let origin = match number { - 18 => 99, - 19 | 20 => 100, - 25 | 26 => 105, - n => 80 + n, - }; - let info = L1BlockInfoBedrock::new( - origin, - 0, - 0, - l1_id(origin, 0).hash, - 0, - Address::ZERO, - U256::ZERO, - U256::ZERO, - ); - let deposit = op_alloy_consensus::OpTxEnvelope::Deposit(alloy_primitives::Sealed::new( - op_alloy_consensus::TxDeposit { - input: info.encode_calldata(), - ..Default::default() - }, - )); - let header = alloy_consensus::Header { - number, - parent_hash: chain.last().map_or(B256::ZERO, |(parent, _)| parent.block_info.hash), - timestamp: number, - ..Default::default() - }; - let consensus = alloy_consensus::Block:: { - header: header.clone(), - body: alloy_consensus::BlockBody { - transactions: vec![deposit.clone()], - ..Default::default() - }, - }; - let block_info = - L2BlockInfo::from_block_and_genesis(&consensus, &RollupConfig::default().genesis) - .unwrap(); - let rpc = L2RpcBlock { - header: alloy_rpc_types_eth::Header { - hash: block_info.block_info.hash, - inner: header, - ..Default::default() - }, - transactions: alloy_rpc_types_eth::BlockTransactions::Full(vec![ - op_alloy_rpc_types::Transaction { - inner: alloy_rpc_types_eth::Transaction { - inner: alloy_consensus::transaction::Recovered::new_unchecked( - deposit, - Address::ZERO, - ), - block_hash: Some(block_info.block_info.hash), - block_number: Some(number), - effective_gas_price: Some(0), - transaction_index: Some(0), - block_timestamp: None, - }, - deposit_nonce: None, - deposit_receipt_version: None, - }, - ]), - ..Default::default() - }; - chain.push((block_info, rpc)); - } - chain - } - /// When the reorg replaced the unsafe head's L1 origin, the unsafe head moves back to the /// highest block whose L1 origin is canonical, never below the kept safe head. Each walk step /// fetches a parent by hash and checks its L1 origin, skipping an origin already found @@ -675,24 +587,22 @@ mod l1_reorg_reset_tests { #[case] unsafe_after: u64, #[case] l1_lookups: &[u64], ) { - let chain = real_chain(); - let block = |number: u64| chain.iter().find(|(info, _)| info.block_info.number == number); - let head = |number: u64| block(number).unwrap().0; + let chain = TestChain::new(); + let head = |number: u64| chain.info(number); let (mut engine, client, l1, l2, config) = engine_at(head(26), head(20)); let candidates: Vec<_> = candidates .iter() .map(|&(safe_head, derived_from)| DerivedSafeHead { safe_head: head(safe_head), - derived_from: l1_id(derived_from, 0), + derived_from: test_l1_block(derived_from, 0), }) .collect(); for &number in l1_lookups { - expect_l1(&l1, number, reorg_height); + expect_l1_block(&l1, number, reorg_height); } expect_l2(&l2, kept, head(kept).block_info.hash); for number in (unsafe_after.max(kept + 1)..26).rev() { - let (info, rpc) = block(number).unwrap(); - l2.expect_params("eth_getBlockByHash", json!([info.block_info.hash, true]), rpc); + chain.expect_by_hash(&l2, number); } let safe_after = head(kept); expect_forkchoice_update(&l2, head(unsafe_after), safe_after); @@ -711,17 +621,16 @@ mod l1_reorg_reset_tests { /// An unsafe chain that does not lead back to the kept safe head gives nothing to keep. #[tokio::test] async fn finds_nothing_to_keep_when_the_unsafe_chain_does_not_lead_to_the_safe_head() { - let chain = real_chain(); - let head = - |number: u64| chain.iter().find(|(info, _)| info.block_info.number == number).unwrap(); - let mut other_20 = head(20).0; + let chain = TestChain::new(); + let mut other_20 = chain.info(20); other_20.block_info.hash = B256::repeat_byte(0xaa); - let (engine, client, l1, l2, _) = engine_at(head(21).0, other_20); - expect_l1(&l1, 100, 101); + let (engine, client, l1, l2, _) = engine_at(chain.info(21), other_20); + expect_l1_block(&l1, 100, 101); expect_l2(&l2, 20, other_20.block_info.hash); - expect_l1(&l1, 101, 101); + expect_l1_block(&l1, 101, 101); - let candidate = DerivedSafeHead { safe_head: other_20, derived_from: l1_id(100, 0) }; + let candidate = + DerivedSafeHead { safe_head: other_20, derived_from: test_l1_block(100, 0) }; let forkchoice = find_reorg_forkchoice(&client, engine.state(), &[candidate]).await.unwrap(); @@ -735,8 +644,8 @@ mod l1_reorg_reset_tests { #[tokio::test] async fn falls_back_to_the_full_walk() { let (mut engine, client, l1, l2, config) = setup(); - expect_l1(&l1, 106, 103); - expect_l1(&l1, 103, 103); + expect_l1_block(&l1, 106, 103); + expect_l1_block(&l1, 103, 103); l2.expect_error("eth_getBlockByNumber"); let err = diff --git a/rust/kona/crates/node/engine/src/test_utils/chain.rs b/rust/kona/crates/node/engine/src/test_utils/chain.rs new file mode 100644 index 00000000000..f77b554f440 --- /dev/null +++ b/rust/kona/crates/node/engine/src/test_utils/chain.rs @@ -0,0 +1,144 @@ +//! A real L2 chain over a scripted L1 chain, for tests of the sync start and reorg walks. + +use crate::test_utils::RpcMock; +use alloy_eips::BlockNumHash; +use alloy_primitives::{Address, B256, U256}; +use kona_genesis::RollupConfig; +use kona_protocol::{L1BlockInfoBedrock, L2BlockInfo}; +use serde_json::json; + +/// An L2 block as the engine's RPC returns it. +pub type L2RpcBlock = ::BlockResponse; + +/// L1 block `number` on `fork`: fork 0 is the chain derivation read, fork 1 replaced it from +/// some height on. +pub fn test_l1_block(number: u64, fork: u8) -> BlockNumHash { + BlockNumHash { number, hash: B256::left_padding_from(&[fork, number as u8]) } +} + +/// Scripts the L1 node's lookup of block `number`, on the chain whose blocks from +/// `replaced_from` on are on fork 1. +pub fn expect_l1_block(l1: &RpcMock, number: u64, replaced_from: u64) { + let mut block = alloy_rpc_types_eth::Block::::default(); + block.header.hash = test_l1_block(number, u8::from(number >= replaced_from)).hash; + block.header.inner.number = number; + l1.expect_params("eth_getBlockByNumber", json!([format!("{number:#x}"), false]), block); +} + +/// A real L2 chain from block 18 to block 26, so the unsafe walks can fetch and decode its +/// blocks. Blocks 18 to 20 have L1 origins 99, 100 and 100, blocks 21 to 24 have L1 origins +/// 101 to 104, and blocks 25 and 26 share L1 origin 105. +#[derive(Debug)] +pub struct TestChain { + blocks: Vec<(L2BlockInfo, L2RpcBlock)>, +} + +impl Default for TestChain { + fn default() -> Self { + Self::new() + } +} + +impl TestChain { + /// Builds the chain. + pub fn new() -> Self { + let mut blocks: Vec<(L2BlockInfo, L2RpcBlock)> = Vec::new(); + for number in 18..=26 { + let origin = match number { + 18 => 99, + 19 | 20 => 100, + 25 | 26 => 105, + n => 80 + n, + }; + let info = L1BlockInfoBedrock::new( + origin, + 0, + 0, + test_l1_block(origin, 0).hash, + 0, + Address::ZERO, + U256::ZERO, + U256::ZERO, + ); + let deposit = op_alloy_consensus::OpTxEnvelope::Deposit(alloy_primitives::Sealed::new( + op_alloy_consensus::TxDeposit { + input: info.encode_calldata(), + ..Default::default() + }, + )); + let header = alloy_consensus::Header { + number, + parent_hash: blocks.last().map_or(B256::ZERO, |(parent, _)| parent.block_info.hash), + timestamp: number, + ..Default::default() + }; + let consensus = alloy_consensus::Block:: { + header: header.clone(), + body: alloy_consensus::BlockBody { + transactions: vec![deposit.clone()], + ..Default::default() + }, + }; + let block_info = + L2BlockInfo::from_block_and_genesis(&consensus, &RollupConfig::default().genesis) + .unwrap(); + let rpc = L2RpcBlock { + header: alloy_rpc_types_eth::Header { + hash: block_info.block_info.hash, + inner: header, + ..Default::default() + }, + transactions: alloy_rpc_types_eth::BlockTransactions::Full(vec![ + op_alloy_rpc_types::Transaction { + inner: alloy_rpc_types_eth::Transaction { + inner: alloy_consensus::transaction::Recovered::new_unchecked( + deposit, + Address::ZERO, + ), + block_hash: Some(block_info.block_info.hash), + block_number: Some(number), + effective_gas_price: Some(0), + transaction_index: Some(0), + block_timestamp: None, + }, + deposit_nonce: None, + deposit_receipt_version: None, + }, + ]), + ..Default::default() + }; + blocks.push((block_info, rpc)); + } + Self { blocks } + } + + /// Block `number`'s info. + pub fn info(&self, number: u64) -> L2BlockInfo { + self.block(number).0 + } + + /// Block `number` as the engine's RPC returns it. + pub fn rpc(&self, number: u64) -> &L2RpcBlock { + &self.block(number).1 + } + + /// Scripts the engine's lookup of block `number` by hash, with transactions. + pub fn expect_by_hash(&self, l2: &RpcMock, number: u64) { + let (info, rpc) = self.block(number); + l2.expect_params("eth_getBlockByHash", json!([info.block_info.hash, true]), rpc); + } + + /// Scripts the engine's lookups of its finalized, safe and latest blocks. + pub fn expect_heads(&self, l2: &RpcMock, finalized: u64, safe: u64, latest: u64) { + for (tag, number) in [("finalized", finalized), ("safe", safe), ("latest", latest)] { + l2.expect_params("eth_getBlockByNumber", json!([tag, true]), self.rpc(number)); + } + } + + fn block(&self, number: u64) -> &(L2BlockInfo, L2RpcBlock) { + self.blocks + .iter() + .find(|(info, _)| info.block_info.number == number) + .unwrap_or_else(|| panic!("no block {number} in the test chain")) + } +} diff --git a/rust/kona/crates/node/engine/src/test_utils/mod.rs b/rust/kona/crates/node/engine/src/test_utils/mod.rs index a5575963a67..af89fc92067 100644 --- a/rust/kona/crates/node/engine/src/test_utils/mod.rs +++ b/rust/kona/crates/node/engine/src/test_utils/mod.rs @@ -9,3 +9,6 @@ pub use engine_state::TestEngineStateBuilder; mod misc; pub use misc::test_block_info; + +mod chain; +pub use chain::{L2RpcBlock, TestChain, expect_l1_block, test_l1_block}; diff --git a/rust/kona/crates/node/service/src/actors/engine/tests.rs b/rust/kona/crates/node/service/src/actors/engine/tests.rs index eded3d472ca..74096f9f848 100644 --- a/rust/kona/crates/node/service/src/actors/engine/tests.rs +++ b/rust/kona/crates/node/service/src/actors/engine/tests.rs @@ -4,7 +4,7 @@ use alloy_primitives::B256; use alloy_rpc_types_engine::{ForkchoiceUpdated, PayloadStatus, PayloadStatusEnum}; use kona_engine::{ Engine, EngineActorRequest, EngineState, EngineSyncStateUpdate, FinalizeBlockId, NoopBlockSink, - test_utils::test_engine_client, + test_utils::{TestChain, expect_l1_block, test_engine_client}, }; use kona_genesis::RollupConfig; use kona_protocol::{BlockInfo, L2BlockInfo}; @@ -26,18 +26,16 @@ async fn admin_reset_recovers_and_completes_original_request( #[case] l1_failure: bool, #[case] read_delay: Duration, ) { - let block: ::BlockResponse = - Default::default(); - let mut config = RollupConfig::default(); - config.genesis.l2.hash = block.header.inner.hash_slow(); - let config = Arc::new(config); + // Finalized, safe and unsafe heads 18, 20 and 26 of a real chain, so the reset checks the + // unsafe head's L1 origin. The sequencing window is long enough that the safe head search + // stops at the finalized head. + let chain = TestChain::new(); + let config = Arc::new(RollupConfig { seq_window_size: 1000, ..Default::default() }); let (client, l1, l2) = test_engine_client(config.clone()); let client = Arc::new(client); if l1_failure { - for tag in ["finalized", "safe", "latest"] { - l2.expect_params("eth_getBlockByNumber", serde_json::json!([tag, true]), &block); - } - l1.expect_error("eth_getBlockByHash"); + chain.expect_heads(&l2, 18, 20, 26); + l1.expect_error("eth_getBlockByNumber"); } else { l2.expect_error("eth_getBlockByNumber"); } @@ -65,13 +63,13 @@ async fn admin_reset_recovers_and_completes_original_request( actor.step().await.unwrap(); // retain the reset actor.step().await.unwrap(); // failed attempt, then wait for backoff assert!(!reset.is_finished()); - for _ in 0..3 { - l2.expect_with_delay("eth_getBlockByNumber", &block, read_delay); + for number in [18, 20, 26] { + l2.expect_with_delay("eth_getBlockByNumber", chain.rpc(number), read_delay); + } + expect_l1_block(&l1, 105, 106); + for number in (18..26).rev() { + chain.expect_by_hash(&l2, number); } - l1.expect( - "eth_getBlockByHash", - alloy_rpc_types_eth::Block::::default(), - ); l2.expect( "engine_forkchoiceUpdatedV3", ForkchoiceUpdated::new(PayloadStatus::from_status(PayloadStatusEnum::Valid)), From ef7b8692e4979d70ff93ae8faee429553e1205dd Mon Sep 17 00:00:00 2001 From: Teddy Knox Date: Tue, 27 Oct 2026 07:02:30 +0100 Subject: [PATCH 6/8] kona-node: retrigger CI after a conductor leadership-transfer flake TestLeadershipTransferMovesActiveSequencer failed in memory-all-kona-op-reth-fusaka on a Raft/unsafe-head race during the transfer, a path this branch does not touch; its sibling conductor tests are flagged flaky in the same job. Co-Authored-By: Claude Opus 5.5 From a57f5fdd83bdf62ab6d014e596cdaf8784520576 Mon Sep 17 00:00:00 2001 From: Teddy Knox Date: Wed, 28 Oct 2026 16:33:45 +0100 Subject: [PATCH 7/8] kona-engine: never raise the safe head on a cold-start reset The cold-start safe head search walked down from the unsafe head and stopped at the first block more than a sequencing window behind it, without checking the previous safe head. When the unsafe head is more than a window ahead of derivation, that marks blocks safe that derivation never produced, and that a batcher outage may have made invalid. op-node's FindL2Heads also requires the safe head to be at or below the previous one; do the same. Co-Authored-By: Claude Opus 5.5 --- rust/kona/crates/node/engine/src/sync/mod.rs | 35 +++++++++++++++++--- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/rust/kona/crates/node/engine/src/sync/mod.rs b/rust/kona/crates/node/engine/src/sync/mod.rs index 9cfbf3a11ae..4af254c60c5 100644 --- a/rust/kona/crates/node/engine/src/sync/mod.rs +++ b/rust/kona/crates/node/engine/src/sync/mod.rs @@ -23,8 +23,9 @@ use crate::EngineClient; /// /// - The *unsafe L2 block*: This is the highest L2 block, at or above the finalized one, whose L1 /// origin is canonical: the L1 node has that block at that height. -/// - The *safe L2 block*: This is the highest L2 block whose epoch's sequencing window is -/// complete within the canonical L1 chain (as known to the rollup node). +/// - The *safe L2 block*: This is the highest L2 block, at or below the previous safe one, whose +/// epoch's sequencing window is complete within the canonical L1 chain (as known to the rollup +/// node). /// - The *finalized L2 block*: This is the L2 block which is known to be fully derived from /// finalized L1 block data. pub async fn find_starting_forkchoice( @@ -51,7 +52,10 @@ pub async fn find_starting_forkchoice( ); // Search for the highest `safe` block that's L1 origin is at least older than the sequencing - // window, relative to the L1 origin of the `unsafe` block. + // window, relative to the L1 origin of the `unsafe` block, and that is no higher than the + // previous safe head: blocks above it were never derived, and may never be. Like op-node's + // `FindL2Heads`. + let previous_safe = current_fc.safe.block_info.number; let mut safe_cursor = current_fc.un_safe; loop { info!( @@ -64,9 +68,10 @@ pub async fn find_starting_forkchoice( let is_behind_sequence_window = current_fc.un_safe.l1_origin.number.saturating_sub(cfg.seq_window_size) > safe_cursor.l1_origin.number; + let is_derived = safe_cursor.block_info.number <= previous_safe; let is_finalized = safe_cursor.block_info.hash == current_fc.finalized.block_info.hash; let is_genesis = safe_cursor.block_info.hash == cfg.genesis.l2.hash; - if is_behind_sequence_window || is_finalized || is_genesis { + if (is_behind_sequence_window && is_derived) || is_finalized || is_genesis { info!( target: "sync_start", l2_safe = %safe_cursor.block_info.number, @@ -242,6 +247,28 @@ mod cold_start_tests { l2.assert_finished(); } + /// The safe head never moves up. With a sequencing window of 2, block 22 (L1 origin 102) is + /// the first block below the unsafe head (L1 origin 105) more than a window behind it, but + /// derivation has only reached block 20, so the search keeps going down to block 20. + #[tokio::test] + async fn cold_start_keeps_the_safe_head_at_or_below_the_previous_one() { + let config = Arc::new(RollupConfig { seq_window_size: 2, ..Default::default() }); + let (client, l1, l2) = test_engine_client(config.clone()); + let chain = TestChain::new(); + chain.expect_heads(&l2, 18, 20, 26); + expect_l1_block(&l1, 105, 106); + for number in (20..26).rev() { + chain.expect_by_hash(&l2, number); + } + + let forkchoice = find_starting_forkchoice(&config, &client).await.unwrap(); + + assert_eq!(forkchoice.un_safe, chain.info(26)); + assert_eq!(forkchoice.safe, chain.info(20)); + l1.assert_finished(); + l2.assert_finished(); + } + /// An unsafe head at or below the finalized head needs no L1 lookup: at the finalized head it /// is kept, and as another block at that height or below it, the reset fails for good. #[rstest] From 8b8a49d94fecf95cc9d30c6a8635f0c79a929aff Mon Sep 17 00:00:00 2001 From: Teddy Knox Date: Thu, 29 Oct 2026 16:16:31 +0100 Subject: [PATCH 8/8] kona-engine: leave the safe head search uncapped just after EL sync The previous-safe-head cap from the last commit regressed the reset after EL sync. There the engine's safe and finalized heads are still at genesis, so the capped search walked every L2 block back to genesis and restarted derivation from there. Like op-node's RecoverMinSeqWindows check, skip the cap when the safe and finalized heads are at genesis and the unsafe head's L1 origin is more than 14 sequencing windows past L1 genesis. Co-Authored-By: Claude Opus 5.5 --- rust/kona/crates/node/engine/src/sync/mod.rs | 46 ++++++++++++++++---- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/rust/kona/crates/node/engine/src/sync/mod.rs b/rust/kona/crates/node/engine/src/sync/mod.rs index 4af254c60c5..0fe5a5e7537 100644 --- a/rust/kona/crates/node/engine/src/sync/mod.rs +++ b/rust/kona/crates/node/engine/src/sync/mod.rs @@ -19,6 +19,11 @@ use tracing::info; use crate::EngineClient; +/// How many sequencing windows the unsafe head's L1 origin must be past L1 genesis, with the safe +/// and finalized heads still at genesis, for the safe head search to treat the node as just +/// EL-synced. Matches op-node's `RecoverMinSeqWindows`. +const RECOVER_MIN_SEQ_WINDOWS: u64 = 14; + /// Searches for the latest [`L2ForkchoiceState`] that we can use to start the sync process with. /// /// - The *unsafe L2 block*: This is the highest L2 block, at or above the finalized one, whose L1 @@ -54,8 +59,18 @@ pub async fn find_starting_forkchoice( // Search for the highest `safe` block that's L1 origin is at least older than the sequencing // window, relative to the L1 origin of the `unsafe` block, and that is no higher than the // previous safe head: blocks above it were never derived, and may never be. Like op-node's - // `FindL2Heads`. - let previous_safe = current_fc.safe.block_info.number; + // `FindL2Heads`. After EL sync, though, the safe and finalized heads can still be at genesis + // with the unsafe head far ahead; there the cap would walk the whole chain back to genesis, + // so, like op-node's `RecoverMinSeqWindows` check, the search is left uncapped. + let el_synced_without_safe_head = current_fc.finalized.block_info.hash == cfg.genesis.l2.hash && + current_fc.safe.block_info.hash == cfg.genesis.l2.hash && + current_fc.un_safe.l1_origin.number > + cfg.genesis + .l1 + .number + .saturating_add(RECOVER_MIN_SEQ_WINDOWS * cfg.seq_window_size); + let previous_safe = + if el_synced_without_safe_head { u64::MAX } else { current_fc.safe.block_info.number }; let mut safe_cursor = current_fc.un_safe; loop { info!( @@ -249,22 +264,35 @@ mod cold_start_tests { /// The safe head never moves up. With a sequencing window of 2, block 22 (L1 origin 102) is /// the first block below the unsafe head (L1 origin 105) more than a window behind it, but - /// derivation has only reached block 20, so the search keeps going down to block 20. + /// derivation has only reached block 20, so the search keeps going down to block 20. Just + /// after EL sync, with the safe and finalized heads at genesis (here block 18) and the unsafe + /// head's L1 origin more than 14 windows past L1 genesis, it stops at block 22 instead of + /// walking the chain back to genesis. + #[rstest] + #[case::derivation_reached_block_20(false, 20)] + #[case::just_el_synced(true, 22)] #[tokio::test] - async fn cold_start_keeps_the_safe_head_at_or_below_the_previous_one() { - let config = Arc::new(RollupConfig { seq_window_size: 2, ..Default::default() }); - let (client, l1, l2) = test_engine_client(config.clone()); + async fn cold_start_keeps_the_safe_head_at_or_below_the_previous_one( + #[case] el_synced: bool, + #[case] expected_safe: u64, + ) { let chain = TestChain::new(); - chain.expect_heads(&l2, 18, 20, 26); + let mut config = RollupConfig { seq_window_size: 2, ..Default::default() }; + if el_synced { + config.genesis.l2 = chain.info(18).block_info.id(); + } + let config = Arc::new(config); + let (client, l1, l2) = test_engine_client(config.clone()); + chain.expect_heads(&l2, 18, if el_synced { 18 } else { 20 }, 26); expect_l1_block(&l1, 105, 106); - for number in (20..26).rev() { + for number in (expected_safe..26).rev() { chain.expect_by_hash(&l2, number); } let forkchoice = find_starting_forkchoice(&config, &client).await.unwrap(); assert_eq!(forkchoice.un_safe, chain.info(26)); - assert_eq!(forkchoice.safe, chain.info(20)); + assert_eq!(forkchoice.safe, chain.info(expected_safe)); l1.assert_finished(); l2.assert_finished(); }