From c1c8522c0dd6632d52e55e0df9e3eacf1416a7c6 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Fri, 6 Nov 2026 20:36:49 -0800 Subject: [PATCH 01/14] kona-engine: deliver and retry delegated forkchoice updates A delegated safe head below the unsafe head was treated like an intermediate span-batch block: it moved the consensus labels without a forkchoice update. An update that left the labels unchanged was also skipped, but a SYNCING answer moves the labels, so an identical retry never reached the execution layer. And while the execution layer was still syncing and lacked the delegated safe block, consolidation reset the engine, which restarted the sync (the delegated case of #23931). Only derived attributes inside a span batch now defer the update. The delegated consolidation paths and the reset always send their forkchoice, so each delegate poll is the retry after a SYNCING answer or an execution-layer restart. A delegated safe head whose block the syncing execution layer does not have yet moves only the safe labels, as in op-node's follow-source mode, instead of resetting; the missing block is logged at warn. One helper applies a delegated safe head with or without the unsafe head. The finalize that follows such a poll still fails on the same syncing execution layer until #23908 skips it, so this lands together with #23899 and #23908. `test_utils::l1_info_deposit` replaces three inline copies of the L1-info deposit, and the delegated consolidation table asserts what reaches the block sink. Co-Authored-By: Claude Opus 5.5 Co-Authored-By: Claude Fable 5.1 --- .../crates/node/engine/src/task_queue/core.rs | 40 ++- .../src/task_queue/tasks/canonicalize_test.rs | 14 +- .../src/task_queue/tasks/consolidate/task.rs | 317 ++++++++++++++++-- .../src/task_queue/tasks/finalize/task.rs | 4 +- .../src/task_queue/tasks/insert/task.rs | 4 +- .../node/engine/src/task_queue/tasks/mod.rs | 2 +- .../task_queue/tasks/synchronize/helper.rs | 25 +- .../src/task_queue/tasks/synchronize/mod.rs | 5 +- .../src/task_queue/tasks/synchronize/tests.rs | 134 ++++++++ .../node/engine/src/test_utils/deposits.rs | 14 + .../crates/node/engine/src/test_utils/mod.rs | 3 + .../crates/node/engine/src/test_utils/rpc.rs | 17 + .../service/src/actors/engine/queue_tests.rs | 71 +++- 13 files changed, 588 insertions(+), 62 deletions(-) create mode 100644 rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/tests.rs create mode 100644 rust/kona/crates/node/engine/src/test_utils/deposits.rs diff --git a/rust/kona/crates/node/engine/src/task_queue/core.rs b/rust/kona/crates/node/engine/src/task_queue/core.rs index 816c8d0519e..acc3fa27d40 100644 --- a/rust/kona/crates/node/engine/src/task_queue/core.rs +++ b/rust/kona/crates/node/engine/src/task_queue/core.rs @@ -5,7 +5,7 @@ use crate::{ EngineClient, EngineState, EngineSyncStateUpdate, EngineTask, EngineTaskError, EngineTaskErrorSeverity, Metrics, SyncStartError, SynchronizeTaskError, find_starting_forkchoice, - task_queue::{EngineTaskErrors, synchronize}, + task_queue::{EngineTaskErrors, ForkchoiceDelivery, synchronize}, }; use kona_genesis::RollupConfig; use kona_protocol::L2BlockInfo; @@ -115,7 +115,9 @@ impl Engine { ) -> Result { let start = find_starting_forkchoice(&config, client.as_ref()).await?; - // One attempt. The actor retains the reset request and retries transient failures. + // One attempt. The actor retains the reset request and retries transient failures. The + // execution layer may have lost or changed its forkchoice, so the reset sends one even when + // the recovered labels equal the current ones. synchronize( client.as_ref(), &mut self.state, @@ -125,6 +127,7 @@ impl Engine { safe_head: Some(start.safe), finalized_head: Some(start.finalized), }, + ForkchoiceDelivery::Always, ) .await?; @@ -345,4 +348,37 @@ mod recovery_tests { imported.header.hash_slow() ); } + + // An execution layer that restarted can have lost its forkchoice, so a reset sends the + // recovered forkchoice even when the labels already equal it. + #[tokio::test] + async fn reset_resends_an_unchanged_forkchoice() { + let block: ::BlockResponse = + Default::default(); + let mut config = RollupConfig::default(); + config.genesis.l2.hash = block.header.inner.hash_slow(); + let genesis = + L2BlockInfo::from_block_and_genesis(&block.clone().into_consensus(), &config.genesis) + .unwrap(); + let config = Arc::new(config); + let state = TestEngineStateBuilder::new().with_unsafe_head(genesis).build(); + let forkchoice = state.sync_state.create_forkchoice_state(); + let (client, l1, l2) = test_engine_client(config.clone()); + for tag in ["finalized", "safe", "latest"] { + l2.expect_params("eth_getBlockByNumber", serde_json::json!([tag, true]), &block); + } + l1.expect( + "eth_getBlockByHash", + alloy_rpc_types_eth::Block::::default(), + ); + l2.expect_forkchoice_updated(forkchoice, PayloadStatusEnum::Valid); + let (state_tx, _state_rx) = tokio::sync::watch::channel(state); + let mut engine = Engine::new(state, state_tx); + + engine.reset(Arc::new(client), config).await.unwrap(); + + assert_eq!(engine.state().sync_state, state.sync_state); + l1.assert_finished(); + l2.assert_finished(); + } } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/canonicalize_test.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/canonicalize_test.rs index 9da42da2fe8..f8d73461107 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/canonicalize_test.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/canonicalize_test.rs @@ -1,11 +1,14 @@ use super::*; -use crate::{InsertTaskError, NoopBlockSink, SynchronizeTaskError, test_utils::test_engine_client}; +use crate::{ + InsertTaskError, NoopBlockSink, SynchronizeTaskError, + test_utils::{l1_info_deposit, test_engine_client}, +}; use alloy_consensus::{Block, SignableTransaction, TxEip1559}; -use alloy_primitives::{Sealed, Signature}; +use alloy_primitives::Signature; use alloy_rpc_types_engine::{ ExecutionPayloadV1, ForkchoiceUpdated, PayloadStatus, PayloadStatusEnum, }; -use op_alloy_consensus::{OpTxEnvelope, TxDeposit}; +use op_alloy_consensus::OpTxEnvelope; #[derive(Debug, Default)] pub(crate) struct RecordingSink(pub(crate) std::sync::Mutex>); @@ -106,10 +109,7 @@ async fn invalid_committed_payload_does_not_build_an_uncommitted_replacement( ) { let PayloadFixture { cfg, .. } = payload_fixture(); let mut block = Block::::default(); - block.body.transactions.push(OpTxEnvelope::Deposit(Sealed::new(TxDeposit { - input: kona_protocol::L1BlockInfoBedrock::default().encode_calldata(), - ..Default::default() - }))); + block.body.transactions.push(l1_info_deposit()); if !deposits_only { block.body.transactions.push(OpTxEnvelope::Eip1559( TxEip1559::default().into_signed(Signature::test_signature()), diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs index a9ada74d3c0..9531eac4aa3 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs @@ -3,7 +3,7 @@ use crate::{ ConsolidateTaskError, EngineClient, EngineState, ExecutableEngineTask, ImportedBlockSink, state::EngineSyncStateUpdate, - task_queue::{build_and_import, synchronize}, + task_queue::{ForkchoiceDelivery, build_and_import, synchronize}, }; use alloy_rpc_types_eth::Block; use async_trait::async_trait; @@ -52,13 +52,19 @@ impl ConsolidateInput { } } - /// Returns true if this is `Attributes` and `attributes.is_last_in_span` is true. - const fn is_attributes_last_in_span(&self) -> bool { - matches!( - self, - Self::Attributes(attributes) - if attributes.is_last_in_span - ) + /// Whether these are derived attributes for a block before the last block of its span batch. + const fn is_intermediate_span_block(&self) -> bool { + matches!(self, Self::Attributes(attributes) if !attributes.is_last_in_span) + } + + /// Derived attributes move the safe head on every consolidation, so an unchanged forkchoice was + /// already sent. A delegated safe head can repeat a forkchoice the execution layer answered + /// `SYNCING` or lost, so it is always sent. + const fn forkchoice_delivery(&self) -> ForkchoiceDelivery { + match self { + Self::Attributes(_) => ForkchoiceDelivery::WhenChanged, + Self::BlockInfo(_) => ForkchoiceDelivery::Always, + } } } @@ -120,27 +126,75 @@ impl ConsolidateTask { // self-consistent head state. This is required to correctly handle reorgs (where unsafe // may be ahead on a non-canonical fork) and to trigger EL sync when the local unsafe head // lags behind the safe head. - synchronize( - self.client.as_ref(), + self.apply_delegated_safe_head( state, + safe_l2, EngineSyncStateUpdate { - unsafe_head: Some(*safe_l2), + local_safe_head: Some(*safe_l2), safe_head: Some(*safe_l2), + unsafe_head: Some(*safe_l2), + ..Default::default() + }, + "Updated safe head via follow safe", + ) + .await + } + + /// Moves the safe labels to a delegated safe head that the execution layer does not have yet, + /// leaving the unsafe head in place. + /// + /// A block missing below the unsafe head means the execution layer is still syncing toward the + /// unsafe head. Resetting would restart that sync from the execution layer's old labels, and + /// moving the unsafe head back would retarget it. Like op-node's follow-source mode, only the + /// safe labels follow the delegate, and the forkchoice update carries them to the execution + /// layer. + async fn follow_safe_head_during_el_sync( + &self, + state: &mut EngineState, + safe_l2: &L2BlockInfo, + ) -> Result<(), ConsolidateTaskError> { + warn!( + target: "engine", + safe_l2 = %safe_l2, + unsafe_number = state.sync_state.unsafe_head().block_info.number, + "Execution layer does not have the delegated safe head yet; keeping its sync target" + ); + + self.apply_delegated_safe_head( + state, + safe_l2, + EngineSyncStateUpdate { local_safe_head: Some(*safe_l2), + safe_head: Some(*safe_l2), ..Default::default() }, + "Updated safe head while the execution layer syncs", ) .await - .map_err(|e| { - warn!(target: "engine", ?e, "Apply safe head failed"); - e - })?; + } + + /// Applies `update`, whose labels follow the delegated `safe_l2`, and always sends its + /// forkchoice update: the execution layer may have answered `SYNCING` to the same labels + /// before, or lost them in a restart. + async fn apply_delegated_safe_head( + &self, + state: &mut EngineState, + safe_l2: &L2BlockInfo, + update: EngineSyncStateUpdate, + outcome: &'static str, + ) -> Result<(), ConsolidateTaskError> { + synchronize(self.client.as_ref(), state, update, ForkchoiceDelivery::Always) + .await + .map_err(|e| { + warn!(target: "engine", ?e, safe_l2 = %safe_l2, "Applying the delegated safe head failed"); + e + })?; info!( target: "engine", hash = %safe_l2.block_info.hash, number = safe_l2.block_info.number, - "Updated safe head via follow safe" + "{outcome}" ); Ok(()) @@ -175,6 +229,9 @@ impl ConsolidateTask { let block = match self.client.l2_block_by_label(block_num.into()).await { Ok(Some(block)) => block, Ok(None) => { + if let ConsolidateInput::BlockInfo(safe_l2) = &self.input { + return self.follow_safe_head_during_el_sync(state, safe_l2).await; + } warn!(target: "engine", "Received `None` block for {}", block_num); return Err(ConsolidateTaskError::MissingUnsafeL2Block(block_num)); } @@ -194,10 +251,10 @@ impl ConsolidateTask { let consensus_block = block.into_consensus().map_transactions(|t| t.inner.inner.into_inner()); match L2BlockInfo::from_block_and_genesis(&consensus_block, &self.cfg.genesis) { - // Only issue a forkchoice update if the attributes are the last in the span - // batch. This is an optimization to avoid sending a FCU - // call for every block in the span batch. - Ok(block_info) if !self.input.is_attributes_last_in_span() => { + // A span batch's blocks before its last one only move the safe label; the last + // block's forkchoice update delivers them all. This avoids sending a FCU call for + // every block in the span batch. + Ok(block_info) if self.input.is_intermediate_span_block() => { // The next attributes built are this block's child, and ask for its config. self.block_sink.block_imported(consensus_block, block_info); @@ -229,6 +286,7 @@ impl ConsolidateTask { local_safe_head: Some(block_info), ..Default::default() }, + self.input.forkchoice_delivery(), ) .await .map_err(|e| { @@ -297,10 +355,16 @@ impl ExecutableEngineTask for ConsolidateTask { mod tests { use super::*; use crate::{ - EngineTaskError, ImportedBlockSink, SealTaskError, test_utils::test_engine_client, + EngineSyncState, EngineTaskError, ImportedBlockSink, NoopBlockSink, SealTaskError, + test_utils::{ + TestAttributesBuilder, TestEngineStateBuilder, l1_info_deposit, test_block_info, + test_engine_client, + }, }; - use alloy_eips::BlockNumHash; - use alloy_primitives::B256; + use alloy_consensus::transaction::Recovered; + use alloy_eips::{BlockNumHash, Encodable2718}; + use alloy_primitives::{Address, B256}; + use alloy_rpc_types_engine::{ForkchoiceUpdated, PayloadStatus, PayloadStatusEnum}; use alloy_rpc_types_eth::{Block as RpcBlock, BlockTransactions, Header as RpcHeader}; use kona_genesis::ChainGenesis; use kona_protocol::BlockInfo; @@ -320,21 +384,14 @@ mod tests { // Consolidation reads the unsafe block in full to compare it against the derived // attributes; the next attributes built ask for this same block's system config, so it // must reach the sink rather than being dropped. - let header = RpcHeader::new(alloy_consensus::Header::default()); - let block_hash = header.hash; - let block: RpcBlock = - RpcBlock::new(header, BlockTransactions::Full(vec![])); - - // Pin genesis to this block so its L2BlockInfo needs no L1-info deposit. - let cfg = Arc::new(RollupConfig { - genesis: ChainGenesis { - l2: BlockNumHash { hash: block_hash, number: 0 }, - ..Default::default() - }, - ..Default::default() - }); + let (block, cfg) = genesis_pinned_block(0); + let block_hash = block.header.hash; let (client, l1, l2) = test_engine_client(cfg.clone()); l2.expect_params("eth_getBlockByNumber", serde_json::json!(["0x0", true]), block); + l2.expect( + "engine_forkchoiceUpdatedV3", + ForkchoiceUpdated::new(PayloadStatus::from_status(PayloadStatusEnum::Valid)), + ); let client = Arc::new(client); let safe_l2 = L2BlockInfo { @@ -479,4 +536,192 @@ mod tests { "got {err:?}" ); } + + /// An engine state with the unsafe head at 20, the finalized head at 5 and `safe_head` between. + fn engine_state_with_safe_head(safe_head: L2BlockInfo) -> EngineState { + TestEngineStateBuilder::new() + .with_unsafe_head(test_block_info(20)) + .with_safe_head(safe_head) + .with_finalized_head(test_block_info(5)) + .build() + } + + fn with_safe_head(state: &EngineState, safe_head: L2BlockInfo) -> EngineSyncState { + state.sync_state.apply_update(EngineSyncStateUpdate { + local_safe_head: Some(safe_head), + safe_head: Some(safe_head), + ..Default::default() + }) + } + + fn block_info_of(cfg: &RollupConfig, block: &RpcBlock) -> L2BlockInfo { + let consensus_block = + block.clone().into_consensus().map_transactions(|t| t.inner.inner.into_inner()); + L2BlockInfo::from_block_and_genesis(&consensus_block, &cfg.genesis).unwrap() + } + + /// An empty block at `number`, with genesis pinned to it so its `L2BlockInfo` needs no L1-info + /// deposit. + fn genesis_pinned_block(number: u64) -> (RpcBlock, Arc) { + let header = RpcHeader::new(alloy_consensus::Header { number, ..Default::default() }); + let cfg = Arc::new(RollupConfig { + genesis: ChainGenesis { + l2: BlockNumHash { hash: header.hash, number }, + ..Default::default() + }, + ..Default::default() + }); + (RpcBlock::new(header, BlockTransactions::Full(vec![])), cfg) + } + + /// What the execution layer answers for the block at the delegated safe head's height. + fn el_block(el_has_block: bool, block: RpcBlock) -> Option> { + el_has_block.then_some(block) + } + + /// The safe head before consolidation: either already the delegated target, as a `SYNCING` + /// answer or an earlier poll leaves it, or an older block. + fn initial_safe_head(starts_at_target: bool, target: L2BlockInfo) -> L2BlockInfo { + if starts_at_target { target } else { test_block_info(5) } + } + + /// Below the unsafe head, a delegated safe head always reaches the execution layer: when the + /// labels already hold it, and when the execution layer is still syncing and does not have its + /// block yet. A missing block then moves only the safe labels instead of resetting, and only a + /// block the execution layer has reaches the block sink. + #[rstest::rstest] + #[case::new_target(true, false, PayloadStatusEnum::Valid)] + #[case::repeated_target(true, true, PayloadStatusEnum::Valid)] + #[case::new_target_missing_from_syncing_el(false, false, PayloadStatusEnum::Syncing)] + #[case::repeated_target_missing_from_syncing_el(false, true, PayloadStatusEnum::Syncing)] + #[tokio::test] + async fn delegated_safe_head_below_unsafe_head_is_always_sent( + #[case] el_has_block: bool, + #[case] starts_at_target: bool, + #[case] reply: PayloadStatusEnum, + ) { + let (block, cfg) = genesis_pinned_block(10); + let delegated_safe_head = block_info_of(&cfg, &block); + let mut state = + engine_state_with_safe_head(initial_safe_head(starts_at_target, delegated_safe_head)); + let expected_sync_state = with_safe_head(&state, delegated_safe_head); + let (client, l1, l2) = test_engine_client(cfg.clone()); + l2.expect_params( + "eth_getBlockByNumber", + serde_json::json!(["0xa", true]), + el_block(el_has_block, block), + ); + l2.expect_forkchoice_updated(expected_sync_state.create_forkchoice_state(), reply); + let sink = Arc::new(RecordingSink::default()); + let task = ConsolidateTask::new( + Arc::new(client), + cfg, + ConsolidateInput::BlockInfo(delegated_safe_head), + sink.clone(), + ); + + task.execute(&mut state).await.unwrap(); + + assert_eq!(state.sync_state, expected_sync_state); + assert_eq!(sink.0.lock().unwrap().len(), usize::from(el_has_block)); + l1.assert_finished(); + l2.assert_finished(); + } + + /// Unlike a delegated safe head, derived attributes whose block is missing below the unsafe + /// head still reset the engine. + #[tokio::test] + async fn derived_attributes_missing_from_el_reset_the_engine() { + let cfg = Arc::new(RollupConfig::default()); + let attributes = derived_attributes(true); + let mut state = engine_state_with_safe_head(attributes.parent); + let state_before = state; + let (client, l1, l2) = test_engine_client(cfg.clone()); + l2.expect_params( + "eth_getBlockByNumber", + serde_json::json!(["0xa", true]), + None::>, + ); + let task = ConsolidateTask::new( + Arc::new(client), + cfg, + ConsolidateInput::Attributes(Box::new(attributes)), + Arc::new(NoopBlockSink), + ); + + let err = task.execute(&mut state).await.unwrap_err(); + + assert!(matches!(err, ConsolidateTaskError::MissingUnsafeL2Block(10)), "got {err:?}"); + assert_eq!(err.severity(), crate::EngineTaskErrorSeverity::Reset); + assert_eq!(state, state_before); + l1.assert_finished(); + l2.assert_finished(); + } + + fn derived_attributes(is_last_in_span: bool) -> OpAttributesWithParent { + let mut attributes = TestAttributesBuilder::new() + .with_parent(test_block_info(9)) + .with_transactions(vec![l1_info_deposit().encoded_2718().into()]) + .build(); + attributes.is_last_in_span = is_last_in_span; + attributes + } + + /// The unsafe block that `attributes` derive, so consolidation finds it consistent. + fn unsafe_block_matching(attributes: &OpAttributesWithParent) -> RpcBlock { + let payload_attributes = &attributes.attributes.payload_attributes; + let header = alloy_consensus::Header { + beneficiary: payload_attributes.suggested_fee_recipient, + gas_limit: attributes.attributes.gas_limit.unwrap(), + mix_hash: payload_attributes.prev_randao, + number: attributes.block_number(), + parent_beacon_block_root: payload_attributes.parent_beacon_block_root, + parent_hash: attributes.parent.block_info.hash, + timestamp: payload_attributes.timestamp, + ..Default::default() + }; + let deposit = Transaction::from_transaction( + Recovered::new_unchecked(l1_info_deposit(), Address::ZERO), + Default::default(), + ); + RpcBlock::new(RpcHeader::new(header), BlockTransactions::Full(vec![deposit])) + } + + /// Derived blocks inside a span batch only move the safe label; the span's last block sends + /// the forkchoice update that delivers it. + #[rstest::rstest] + #[case::intermediate_block_defers_forkchoice(false, 0)] + #[case::last_block_sends_forkchoice(true, 1)] + #[tokio::test] + async fn span_batch_sends_forkchoice_only_at_its_last_block( + #[case] is_last_in_span: bool, + #[case] forkchoice_updates: usize, + ) { + let cfg = Arc::new(RollupConfig::default()); + let attributes = derived_attributes(is_last_in_span); + let block = unsafe_block_matching(&attributes); + let derived_safe_head = block_info_of(&cfg, &block); + let mut state = engine_state_with_safe_head(attributes.parent); + let expected_sync_state = with_safe_head(&state, derived_safe_head); + let (client, l1, l2) = test_engine_client(cfg.clone()); + l2.expect_params("eth_getBlockByNumber", serde_json::json!(["0xa", true]), block); + for _ in 0..forkchoice_updates { + l2.expect_forkchoice_updated( + expected_sync_state.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + } + let task = ConsolidateTask::new( + Arc::new(client), + cfg, + ConsolidateInput::Attributes(Box::new(attributes)), + Arc::new(NoopBlockSink), + ); + + task.execute(&mut state).await.unwrap(); + + assert_eq!(state.sync_state, expected_sync_state); + l1.assert_finished(); + l2.assert_finished(); + } } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs index 3461d7d860f..d00bb74b624 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs @@ -2,7 +2,8 @@ use crate::{ EngineClient, EngineState, ExecutableEngineTask, FinalizeBlockId, FinalizeTaskError, - state::EngineSyncStateUpdate, task_queue::synchronize, + state::EngineSyncStateUpdate, + task_queue::{ForkchoiceDelivery, synchronize}, }; use alloy_eips::BlockId; use async_trait::async_trait; @@ -71,6 +72,7 @@ impl ExecutableEngineTask for FinalizeTask { self.client.as_ref(), state, EngineSyncStateUpdate { finalized_head: Some(block_info), ..Default::default() }, + ForkchoiceDelivery::WhenChanged, ) .await?; diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/insert/task.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/insert/task.rs index 54d2d380709..10110e3b622 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/insert/task.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/insert/task.rs @@ -2,7 +2,8 @@ use crate::{ EngineClient, EngineState, ExecutableEngineTask, ImportedBlockSink, InsertTaskError, - state::EngineSyncStateUpdate, task_queue::synchronize, + state::EngineSyncStateUpdate, + task_queue::{ForkchoiceDelivery, synchronize}, }; use alloy_rpc_types_engine::{ExecutionPayloadInputV2, PayloadStatusEnum}; use async_trait::async_trait; @@ -125,6 +126,7 @@ pub(in crate::task_queue) async fn insert_payload( safe_head: is_payload_safe.then_some(new_unsafe_ref), ..Default::default() }, + ForkchoiceDelivery::WhenChanged, ) .await?; diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/mod.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/mod.rs index 224c7a9e34f..c5aa7a8ad47 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/mod.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/mod.rs @@ -7,7 +7,7 @@ pub use task::{ mod synchronize; pub use synchronize::SynchronizeTaskError; -pub(super) use synchronize::synchronize; +pub(super) use synchronize::{ForkchoiceDelivery, synchronize}; mod insert; pub(super) use insert::insert_payload; diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs index abae6d99464..219416bd6f1 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs @@ -4,6 +4,18 @@ use crate::{EngineClient, EngineState, SynchronizeTaskError, state::EngineSyncSt use alloy_rpc_types_engine::{INVALID_FORK_CHOICE_STATE_ERROR, PayloadStatusEnum}; use op_alloy_provider::ext::engine::OpEngineApi; +/// Whether [`synchronize`] may skip a forkchoice update that leaves the consensus labels unchanged. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(in crate::task_queue) enum ForkchoiceDelivery { + /// Send the update even when it leaves the consensus labels unchanged. + /// + /// Unchanged labels do not prove that the execution layer holds their forkchoice: a `SYNCING` + /// answer moves the labels too, and an execution layer that restarts can lose its forkchoice. + Always, + /// Skip the update when it leaves the consensus labels unchanged. + WhenChanged, +} + /// Applies `state_update` to the engine's sync state and synchronizes the execution layer's /// forkchoice with it, using an `engine_forkchoiceUpdated` call without payload attributes. /// @@ -15,26 +27,31 @@ use op_alloy_provider::ext::engine::OpEngineApi; target = "engine", level = "info", skip_all, - fields(?state_update, forkchoice = tracing::field::Empty) + fields(?delivery, ?state_update, forkchoice = tracing::field::Empty) )] pub(in crate::task_queue) async fn synchronize( client: &EngineClient, state: &mut EngineState, state_update: EngineSyncStateUpdate, + delivery: ForkchoiceDelivery, ) -> Result<(), SynchronizeTaskError> { // Apply the sync state update to the engine state. let new_sync_state = state.sync_state.apply_update(state_update); // Check if a forkchoice update is not needed, return early. // A forkchoice update is not needed if... - // 1. The engine state is not default (initial forkchoice state has been emitted), and - // 2. The new sync state is the same as the current sync state (no changes to the sync state). + // 1. The caller allows skipping an unchanged forkchoice, + // 2. The engine state is not default (initial forkchoice state has been emitted), and + // 3. The new sync state is the same as the current sync state (no changes to the sync state). // // NOTE: // We shouldn't retry the synchronization here. Since the `sync_state` is only updated inside // `synchronize` (except inside the ConsolidateTask, when the block is not the last in the // batch) - the engine would get stuck retrying the synchronization. - if state.sync_state != Default::default() && state.sync_state == new_sync_state { + if delivery == ForkchoiceDelivery::WhenChanged && + state.sync_state != Default::default() && + state.sync_state == new_sync_state + { debug!(target: "engine", ?new_sync_state, "No forkchoice update needed"); return Ok(()); } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/mod.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/mod.rs index 67c378103a7..e73923b3228 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/mod.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/mod.rs @@ -1,7 +1,10 @@ //! The forkchoice engine update shared by the engine tasks, and its error type. mod helper; -pub(in crate::task_queue) use helper::synchronize; +pub(in crate::task_queue) use helper::{ForkchoiceDelivery, synchronize}; mod error; pub use error::SynchronizeTaskError; + +#[cfg(test)] +mod tests; diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/tests.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/tests.rs new file mode 100644 index 00000000000..1ebc3a56248 --- /dev/null +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/tests.rs @@ -0,0 +1,134 @@ +use super::{ForkchoiceDelivery, synchronize}; +use crate::{ + EngineState, EngineSyncState, EngineSyncStateUpdate, EngineTaskError, + EngineTaskErrorSeverity::{self, Reset, Temporary}, + test_utils::{RpcMock, TestEngineStateBuilder, test_block_info, test_engine_client}, +}; +use alloy_rpc_types_engine::{ForkchoiceState, INVALID_FORK_CHOICE_STATE_ERROR, PayloadStatusEnum}; +use kona_genesis::RollupConfig; +use rstest::rstest; +use std::sync::Arc; + +const FORKCHOICE_UPDATED: &str = "engine_forkchoiceUpdatedV3"; + +/// How the execution layer answers the forkchoice update. +#[derive(Clone, Copy, Debug)] +enum Reply { + Invalid, + InvalidForkchoiceState, + Syncing, + TransportError, + Valid, +} + +/// Which labels the engine holds after the forkchoice update. +#[derive(Clone, Copy, Debug)] +enum Labels { + Kept, + Moved, +} + +/// An engine state that has not seen a `VALID` answer yet. +fn engine_state() -> EngineState { + TestEngineStateBuilder::new() + .with_unsafe_head(test_block_info(20)) + .with_safe_head(test_block_info(10)) + .with_finalized_head(test_block_info(5)) + .with_el_sync_finished(false) + .build() +} + +/// An update that moves the safe head, or one that leaves every label unchanged. +fn state_update(moves_safe_head: bool) -> EngineSyncStateUpdate { + EngineSyncStateUpdate { + safe_head: moves_safe_head.then(|| test_block_info(15)), + ..Default::default() + } +} + +const fn labels_after( + labels: Labels, + before: EngineSyncState, + moved: EngineSyncState, +) -> EngineSyncState { + match labels { + Labels::Kept => before, + Labels::Moved => moved, + } +} + +fn expect_reply(l2: &RpcMock, forkchoice: ForkchoiceState, reply: Reply) { + match reply { + Reply::Invalid => l2.expect_forkchoice_updated( + forkchoice, + PayloadStatusEnum::Invalid { validation_error: "invalid".into() }, + ), + Reply::InvalidForkchoiceState => { + l2.expect_error_code(FORKCHOICE_UPDATED, INVALID_FORK_CHOICE_STATE_ERROR as i64) + } + Reply::Syncing => l2.expect_forkchoice_updated(forkchoice, PayloadStatusEnum::Syncing), + Reply::TransportError => l2.expect_transport_error(FORKCHOICE_UPDATED), + Reply::Valid => l2.expect_forkchoice_updated(forkchoice, PayloadStatusEnum::Valid), + } +} + +/// An update that leaves the labels unchanged is skipped only when the caller allows it. +#[rstest] +#[case::changed_labels_are_sent(true, ForkchoiceDelivery::WhenChanged, 1)] +#[case::unchanged_labels_are_skipped(false, ForkchoiceDelivery::WhenChanged, 0)] +#[case::changed_labels_are_sent_always(true, ForkchoiceDelivery::Always, 1)] +#[case::unchanged_labels_are_sent_always(false, ForkchoiceDelivery::Always, 1)] +#[tokio::test] +async fn synchronize_skips_unchanged_labels_only_when_allowed( + #[case] moves_safe_head: bool, + #[case] delivery: ForkchoiceDelivery, + #[case] forkchoice_updates: usize, +) { + let mut state = engine_state(); + let update = state_update(moves_safe_head); + let expected_sync_state = state.sync_state.apply_update(update); + let (client, l1, l2) = test_engine_client(Arc::new(RollupConfig::default())); + for _ in 0..forkchoice_updates { + l2.expect_forkchoice_updated( + expected_sync_state.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + } + + synchronize(&client, &mut state, update, delivery).await.unwrap(); + + assert_eq!(state.sync_state, expected_sync_state); + l1.assert_finished(); + l2.assert_finished(); +} + +/// Only a `VALID` or `SYNCING` answer moves the labels, and only `VALID` marks the execution layer +/// as synced. Failures leave the engine state untouched. +#[rstest] +#[case::valid(Reply::Valid, Labels::Moved, true, None)] +#[case::syncing(Reply::Syncing, Labels::Moved, false, None)] +#[case::invalid_payload(Reply::Invalid, Labels::Kept, false, Some(Reset))] +#[case::invalid_forkchoice_state(Reply::InvalidForkchoiceState, Labels::Kept, false, Some(Reset))] +#[case::transport_error(Reply::TransportError, Labels::Kept, false, Some(Temporary))] +#[tokio::test] +async fn synchronize_applies_labels_only_when_accepted( + #[case] reply: Reply, + #[case] labels: Labels, + #[case] el_sync_finished: bool, + #[case] error_severity: Option, +) { + let mut state = engine_state(); + let update = state_update(true); + let moved_sync_state = state.sync_state.apply_update(update); + let expected_sync_state = labels_after(labels, state.sync_state, moved_sync_state); + let (client, l1, l2) = test_engine_client(Arc::new(RollupConfig::default())); + expect_reply(&l2, moved_sync_state.create_forkchoice_state(), reply); + + let result = synchronize(&client, &mut state, update, ForkchoiceDelivery::WhenChanged).await; + + assert_eq!(result.as_ref().err().map(EngineTaskError::severity), error_severity, "{result:?}"); + assert_eq!(state.sync_state, expected_sync_state); + assert_eq!(state.el_sync_finished, el_sync_finished); + l1.assert_finished(); + l2.assert_finished(); +} diff --git a/rust/kona/crates/node/engine/src/test_utils/deposits.rs b/rust/kona/crates/node/engine/src/test_utils/deposits.rs new file mode 100644 index 00000000000..78d60d4bf01 --- /dev/null +++ b/rust/kona/crates/node/engine/src/test_utils/deposits.rs @@ -0,0 +1,14 @@ +//! Deposit transaction fixtures. + +use alloy_primitives::Sealed; +use kona_protocol::L1BlockInfoBedrock; +use op_alloy_consensus::{OpTxEnvelope, TxDeposit}; + +/// The L1-info deposit a block needs for `L2BlockInfo::from_block_and_genesis` to read its L1 +/// origin. +pub fn l1_info_deposit() -> OpTxEnvelope { + OpTxEnvelope::Deposit(Sealed::new(TxDeposit { + input: L1BlockInfoBedrock::default().encode_calldata(), + ..Default::default() + })) +} diff --git a/rust/kona/crates/node/engine/src/test_utils/mod.rs b/rust/kona/crates/node/engine/src/test_utils/mod.rs index a5575963a67..e5949f23225 100644 --- a/rust/kona/crates/node/engine/src/test_utils/mod.rs +++ b/rust/kona/crates/node/engine/src/test_utils/mod.rs @@ -1,6 +1,9 @@ mod attributes; pub use attributes::TestAttributesBuilder; +mod deposits; +pub use deposits::l1_info_deposit; + mod rpc; pub use rpc::{RpcMock, test_engine_client}; diff --git a/rust/kona/crates/node/engine/src/test_utils/rpc.rs b/rust/kona/crates/node/engine/src/test_utils/rpc.rs index be93a9816fe..1cfc3995356 100644 --- a/rust/kona/crates/node/engine/src/test_utils/rpc.rs +++ b/rust/kona/crates/node/engine/src/test_utils/rpc.rs @@ -5,6 +5,9 @@ use alloy_json_rpc::{ErrorPayload, RequestPacket, ResponsePacket}; use alloy_network::Network; use alloy_provider::RootProvider; use alloy_rpc_client::RpcClient; +use alloy_rpc_types_engine::{ + ForkchoiceState, ForkchoiceUpdated, PayloadStatus, PayloadStatusEnum, +}; use alloy_transport::{ TransportError, TransportErrorKind, TransportFut, mock::{Asserter, MockResponse, MockTransport}, @@ -56,6 +59,20 @@ impl RpcMock { ); } + /// Expects an `engine_forkchoiceUpdatedV3` call for `forkchoice` without payload attributes and + /// answers it with `status`. + pub fn expect_forkchoice_updated( + &self, + forkchoice: ForkchoiceState, + status: PayloadStatusEnum, + ) { + self.expect_params( + "engine_forkchoiceUpdatedV3", + serde_json::json!([forkchoice, null]), + ForkchoiceUpdated::new(PayloadStatus::from_status(status)), + ); + } + /// Expects a call and delays its response, for recovery tests using Tokio's paused clock. pub fn expect_with_delay( &self, diff --git a/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs b/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs index 49e30d84f94..3dc21d249e1 100644 --- a/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs +++ b/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs @@ -1,15 +1,13 @@ use super::*; use crate::QueuedEngineDerivationClient; -use alloy_primitives::Sealed; use alloy_rpc_types_engine::{ ExecutionPayloadV1, ForkchoiceUpdated, PayloadStatus, PayloadStatusEnum, }; use kona_engine::{ - NoopBlockSink, - test_utils::{TestEngineStateBuilder, test_block_info, test_engine_client}, + ConsolidateInput, EngineSyncStateUpdate, NoopBlockSink, + test_utils::{TestEngineStateBuilder, l1_info_deposit, test_block_info, test_engine_client}, }; -use kona_protocol::L1BlockInfoBedrock; -use op_alloy_consensus::{OpBlock, OpTxEnvelope, TxDeposit}; +use op_alloy_consensus::OpBlock; use op_alloy_rpc_types_engine::OpExecutionPayloadEnvelope; /// Exercise the real actor request/drain loop with forty consecutive unsafe blocks, without @@ -39,10 +37,7 @@ async fn healthy_unsafe_imports_keep_engine_queue_bounded() { block.header.number = number; block.header.timestamp = number * 2; block.header.parent_hash = expected_heads.last().unwrap().hash(); - block.body.transactions.push(OpTxEnvelope::Deposit(Sealed::new(TxDeposit { - input: L1BlockInfoBedrock::default().encode_calldata(), - ..Default::default() - }))); + block.body.transactions.push(l1_info_deposit()); let payload = ExecutionPayloadV1::from_block_slow(&block); let imported: OpBlock = OpExecutionPayloadEnvelope::V1(payload.clone()).try_into_block().unwrap(); @@ -87,3 +82,61 @@ async fn healthy_unsafe_imports_keep_engine_queue_bounded() { l1.assert_finished(); l2.assert_finished(); } + +/// While the execution layer catches up to a delegated safe head ahead of the local unsafe head, +/// each delegate poll repeats the same target. Every poll sends it, after a `SYNCING` answer and +/// after a `VALID` one, since an execution layer that restarts can lose an applied forkchoice. +#[tokio::test(start_paused = true)] +async fn delegated_safe_head_is_sent_on_every_poll() { + let config = Arc::new(RollupConfig::default()); + let (client, l1, l2) = test_engine_client(config.clone()); + let state = TestEngineStateBuilder::new() + .with_unsafe_head(test_block_info(20)) + .with_safe_head(test_block_info(10)) + .with_finalized_head(test_block_info(5)) + .build(); + let delegated_safe_head = test_block_info(30); + let target = state + .sync_state + .apply_update(EngineSyncStateUpdate { + local_safe_head: Some(delegated_safe_head), + safe_head: Some(delegated_safe_head), + unsafe_head: Some(delegated_safe_head), + ..Default::default() + }) + .create_forkchoice_state(); + for status in [PayloadStatusEnum::Syncing, PayloadStatusEnum::Valid, PayloadStatusEnum::Valid] { + l2.expect_forkchoice_updated(target, status); + } + let (state_tx, state_rx) = watch::channel(state); + let (requests, request_rx) = mpsc::channel(8); + let (derivation_tx, _derivation_rx) = mpsc::channel(8); + let mut actor = EngineActor::new( + Arc::new(client), + config, + QueuedEngineDerivationClient { derivation_actor_request_tx: derivation_tx }, + Engine::new(state, state_tx), + None, + request_rx, + Arc::new(NoopBlockSink), + ); + + // Three polls report the same target. Each step drains the previous poll's task and then + // accepts the next poll. + for _ in 0..3 { + requests + .send(EngineActorRequest::ProcessSafeL2Signal(ConsolidateInput::BlockInfo( + delegated_safe_head, + ))) + .await + .unwrap(); + time::timeout(Duration::from_secs(1), actor.step()).await.unwrap().unwrap(); + } + drop(requests); + assert!(matches!(actor.step().await, Err(EngineError::ChannelClosed))); + + assert!(actor.engine.is_empty()); + assert_eq!(state_rx.borrow().sync_state.safe_head(), delegated_safe_head); + l1.assert_finished(); + l2.assert_finished(); +} From 3ca85833f58d2b22adcea5be86104d3f386c6f07 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Fri, 6 Nov 2026 20:36:49 -0800 Subject: [PATCH 02/14] kona-engine: prevent delegated finalized-head regression A lagging delegate could move finality backward, and a delegate on another fork could name a finalized block that is not the execution layer's canonical block. The first is irreversible. The second either stopped the node or failed every forkchoice update and reset the engine on each poll. The finalize task now looks the target up by number and finalizes a pinned hash only when it is the canonical block at that height. A target behind the finalized head, a different block at the finalized height, or a pinned hash that is not canonical fails with an error of the new `Skip` severity: the engine drops the task, counts it as a failure, and the engine actor warns once per distinct error until a reset. A target equal to the finalized head needs no lookup or forkchoice update, and each finality guard runs once per kind of target. A block served at another height is a distinct `BlockNumberMismatch`. The delegate actor also rejects a whole sync status whose finalized L2 block is behind, or conflicts with, the engine's, so a lagging or diverged delegate's safe head does not reach the engine either. The baseline is the finalized head the engine applied, read from the state it publishes, not the last target forwarded to it: a target the engine skipped must not make the canonical block at its height look like a conflict, which would stop the delegate's safe head too. BREAKING CHANGE: `EngineTaskErrorSeverity` has a new `Skip` variant, and `FinalizeTaskError` has new variants `BehindFinalizedHead`, `BlockNumberMismatch`, `ConflictsWithFinalizedHead` and `NotCanonical`; exhaustive matches must handle them. A block served at another height is now `BlockNumberMismatch` instead of `BlockNotFound`. `DerivationActorRequest::ProcessEngineSyncCompletionRequest` carries an `EngineSyncState`, and `EngineDerivationClient::notify_sync_completed` takes one. `DelegateDerivationActor::new` takes the engine's state receiver. Co-Authored-By: Claude Opus 5.5 Co-Authored-By: Claude Fable 5.1 --- .../crates/node/engine/src/task_queue/core.rs | 39 ++- .../src/task_queue/tasks/finalize/error.rs | 57 +++- .../src/task_queue/tasks/finalize/id.rs | 10 +- .../src/task_queue/tasks/finalize/task.rs | 134 ++++++-- .../task_queue/tasks/finalize/task_test.rs | 289 ++++++++++++++---- .../node/engine/src/task_queue/tasks/task.rs | 4 + .../node/engine/src/test_utils/blocks.rs | 40 +++ .../crates/node/engine/src/test_utils/mod.rs | 3 + .../crates/node/engine/src/test_utils/rpc.rs | 12 +- .../src/actors/derivation/actor/core.rs | 3 +- .../src/actors/derivation/actor/tests.rs | 29 +- .../actors/derivation/actor/tests/database.rs | 2 +- .../derivation/actor/tests/op_node_parity.rs | 2 +- .../src/actors/derivation/delegated/actor.rs | 60 +++- .../src/actors/derivation/delegated/mod.rs | 3 + .../src/actors/derivation/delegated/tests.rs | 250 +++++++++++++++ .../service/src/actors/derivation/request.rs | 7 +- .../node/service/src/actors/engine/actor.rs | 24 +- .../node/service/src/actors/engine/client.rs | 16 +- .../node/service/src/actors/engine/tests.rs | 2 +- .../node/service/src/service/node/rollup.rs | 3 + 21 files changed, 855 insertions(+), 134 deletions(-) create mode 100644 rust/kona/crates/node/engine/src/test_utils/blocks.rs create mode 100644 rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs diff --git a/rust/kona/crates/node/engine/src/task_queue/core.rs b/rust/kona/crates/node/engine/src/task_queue/core.rs index acc3fa27d40..e9d9dcb4f1c 100644 --- a/rust/kona/crates/node/engine/src/task_queue/core.rs +++ b/rust/kona/crates/node/engine/src/task_queue/core.rs @@ -145,7 +145,7 @@ impl Engine { /// Attempts to drain the queue by executing all [`EngineTask`]s in-order. If any task returns /// an error along the way, the error is returned and the task is not popped from the queue (in - /// case it must be retried), unless it is a flush error. + /// case it must be retried), unless it is a flush or skip error. pub async fn drain(&mut self) -> Result<(), EngineTaskErrors> { // Drain tasks in order of priority, halting on errors for a retry to be attempted. loop { @@ -156,9 +156,13 @@ impl Engine { let result = task.execute(&mut self.state).await; // A flush reports that the task replaced its block with a deposits-only block, so the - // task is done; retrying it would replace that block again. + // task is done; retrying it would replace that block again. A skip reports a task that + // no longer applies, so retrying it cannot succeed either. if let Err(err) = &result && - err.severity() != EngineTaskErrorSeverity::Flush + !matches!( + err.severity(), + EngineTaskErrorSeverity::Flush | EngineTaskErrorSeverity::Skip + ) { return result; } @@ -198,7 +202,8 @@ impl EngineResetError { mod recovery_tests { use super::*; use crate::{ - ConsolidateInput, ConsolidateTask, InsertTask, NoopBlockSink, + ConsolidateInput, ConsolidateTask, FinalizeBlockId, FinalizeTask, InsertTask, + NoopBlockSink, test_utils::{ TestAttributesBuilder, TestEngineStateBuilder, test_block_info, test_engine_client, }, @@ -381,4 +386,30 @@ mod recovery_tests { l1.assert_finished(); l2.assert_finished(); } + + // A skipped task is dropped instead of being kept for a retry, so it cannot hold the queue. + #[tokio::test] + async fn skipped_task_is_dropped_from_the_queue() { + let config = Arc::new(RollupConfig::default()); + let (client, l1, l2) = test_engine_client(config.clone()); + let state = TestEngineStateBuilder::new() + .with_unsafe_head(test_block_info(20)) + .with_finalized_head(test_block_info(10)) + .build(); + let (state_tx, _state_rx) = tokio::sync::watch::channel(state); + let mut engine = Engine::new(state, state_tx); + engine.enqueue(EngineTask::Finalize(Box::new(FinalizeTask::new( + Arc::new(client), + config, + FinalizeBlockId::ByNumber(5), + )))); + + let error = engine.drain().await.unwrap_err(); + + assert_eq!(error.severity(), EngineTaskErrorSeverity::Skip, "got {error:?}"); + assert!(engine.is_empty()); + assert_eq!(engine.state().sync_state, state.sync_state); + l1.assert_finished(); + l2.assert_finished(); + } } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/error.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/error.rs index c8444c2595e..51a7f2c1dda 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/error.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/error.rs @@ -3,6 +3,7 @@ use crate::{ EngineTaskError, SynchronizeTaskError, task_queue::tasks::task::EngineTaskErrorSeverity, }; +use alloy_eips::BlockNumHash; use alloy_transport::{RpcError, TransportErrorKind}; use kona_protocol::FromBlockError; use thiserror::Error; @@ -10,33 +11,69 @@ use thiserror::Error; /// An error that occurs when running the [`crate::FinalizeTask`]. #[derive(Debug, Error)] pub enum FinalizeTaskError { - /// The block is not safe, and therefore cannot be finalized. - #[error("Attempted to finalize a block that is not yet safe")] - BlockNotSafe, + /// The target is below the finalized head, and finality never moves backward. + #[error("Finalization target {target} is behind the finalized head {finalized}")] + BehindFinalizedHead { + /// The number of the finalized head. + finalized: u64, + /// The number of the finalization target. + target: u64, + }, /// The block to finalize was not found. #[error("The block to finalize was not found: Number {0}")] BlockNotFound(u64), + /// The block is not safe, and therefore cannot be finalized. + #[error("Attempted to finalize a block that is not yet safe")] + BlockNotSafe, + /// The engine returned a block at a different height than the one requested. + #[error("Requested block {requested} to finalize, but the engine returned block {returned}")] + BlockNumberMismatch { + /// The requested block number. + requested: u64, + /// The number of the block the engine returned. + returned: u64, + }, + /// The target is a different block at the finalized head's height. + #[error("Finalization target {target:?} conflicts with the finalized head {finalized:?}")] + ConflictsWithFinalizedHead { + /// The finalized head. + finalized: BlockNumHash, + /// The finalization target. + target: BlockNumHash, + }, + /// The forkchoice update call to finalize the block failed. + #[error(transparent)] + ForkchoiceUpdateFailed(#[from] SynchronizeTaskError), /// An error occurred while transforming the RPC block into [`L2BlockInfo`]. /// /// [`L2BlockInfo`]: kona_protocol::L2BlockInfo #[error(transparent)] FromBlock(#[from] FromBlockError), + /// The pinned target is not the engine's canonical block at its height. + #[error("Finalization target {target:?} is not canonical; the engine has {canonical:?}")] + NotCanonical { + /// The engine's canonical block at the target's height. + canonical: BlockNumHash, + /// The finalization target. + target: BlockNumHash, + }, /// A temporary RPC failure. #[error(transparent)] TransportError(#[from] RpcError), - /// The forkchoice update call to finalize the block failed. - #[error(transparent)] - ForkchoiceUpdateFailed(#[from] SynchronizeTaskError), } impl EngineTaskError for FinalizeTaskError { fn severity(&self) -> EngineTaskErrorSeverity { match self { - Self::BlockNotSafe | Self::BlockNotFound(_) | Self::FromBlock(_) => { - EngineTaskErrorSeverity::Critical - } - Self::TransportError(_) => EngineTaskErrorSeverity::Temporary, + Self::BehindFinalizedHead { .. } | + Self::ConflictsWithFinalizedHead { .. } | + Self::NotCanonical { .. } => EngineTaskErrorSeverity::Skip, + Self::BlockNotFound(_) | + Self::BlockNotSafe | + Self::BlockNumberMismatch { .. } | + Self::FromBlock(_) => EngineTaskErrorSeverity::Critical, Self::ForkchoiceUpdateFailed(inner) => inner.severity(), + Self::TransportError(_) => EngineTaskErrorSeverity::Temporary, } } } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/id.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/id.rs index ab5a92e4ddb..c7c12fd6e73 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/id.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/id.rs @@ -10,9 +10,9 @@ use alloy_eips::BlockNumHash; /// authoritative source for the block at a given height (e.g. local L1-finality driven by /// `L2Finalizer`). The number is sufficient because there is only one chain to consult. /// - [`FinalizeBlockId::ByHash`] is used when an upstream supplies `(number, hash)` and the engine -/// must finalize the specific block identified by hash. If the engine does not have that hash, -/// the task must error rather than silently finalize the engine's canonical block at the same -/// height. +/// must finalize the specific block identified by hash. If the engine's canonical block at that +/// height has a different hash, the task skips the target rather than finalize the engine's block +/// in its place. /// /// EL finalization is irreversible, so finalizing the wrong block is unrecoverable. Forcing every /// caller to pick a variant prevents accidental hash-loss at the boundary. @@ -20,8 +20,8 @@ use alloy_eips::BlockNumHash; pub enum FinalizeBlockId { /// Finalize whatever block the engine has at the given L2 block number. ByNumber(u64), - /// Finalize the block matching the given `(number, hash)` pair. The task fails if the engine - /// does not have that hash. + /// Finalize the block matching the given `(number, hash)` pair, only once it is the engine's + /// canonical block at that height. ByHash(BlockNumHash), } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs index d00bb74b624..a01d43c1608 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs @@ -5,7 +5,7 @@ use crate::{ state::EngineSyncStateUpdate, task_queue::{ForkchoiceDelivery, synchronize}, }; -use alloy_eips::BlockId; +use alloy_eips::BlockNumHash; use async_trait::async_trait; use derive_more::Constructor; use kona_genesis::RollupConfig; @@ -14,6 +14,10 @@ use std::sync::Arc; /// The [`FinalizeTask`] fetches the [`L2BlockInfo`] identified by `block_id`, updates the /// [`EngineState`], and dispatches a forkchoice update to finalize the block. +/// +/// Finality only moves forward. A target behind the finalized head, a different block at the +/// finalized height, or a pinned hash that is not the engine's canonical block fails with an error +/// that skips the task and leaves the engine state unchanged. #[derive(Debug, Clone, Constructor)] pub struct FinalizeTask { /// The engine client. @@ -24,6 +28,88 @@ pub struct FinalizeTask { pub block_id: FinalizeBlockId, } +impl FinalizeTask { + /// Rejects a target that would move finality backward. + const fn ensure_not_behind_finalized_head( + finalized_head: &L2BlockInfo, + target_number: u64, + ) -> Result<(), FinalizeTaskError> { + if target_number < finalized_head.block_info.number { + return Err(FinalizeTaskError::BehindFinalizedHead { + finalized: finalized_head.block_info.number, + target: target_number, + }); + } + Ok(()) + } + + /// Rejects a different block at the finalized head's height. + /// + /// The default head means finality has not been established yet, so any target, including the + /// configured genesis, may initialize it. + fn ensure_no_conflict_with_finalized_head( + finalized_head: &L2BlockInfo, + target: BlockNumHash, + ) -> Result<(), FinalizeTaskError> { + let finalized = finalized_head.block_info.id(); + if *finalized_head != L2BlockInfo::default() && + target.number == finalized.number && + target.hash != finalized.hash + { + return Err(FinalizeTaskError::ConflictsWithFinalizedHead { finalized, target }); + } + Ok(()) + } + + /// Whether `target` is the finalized head itself; a different block at its height is an error. + fn is_already_finalized( + finalized_head: &L2BlockInfo, + target: BlockNumHash, + ) -> Result { + if target == finalized_head.block_info.id() { + debug!(target: "engine", ?target, "Target is already finalized"); + return Ok(true); + } + Self::ensure_no_conflict_with_finalized_head(finalized_head, target)?; + Ok(false) + } + + /// Fetches the engine's canonical block at the target's height. + /// + /// A pinned hash must be that block: a forkchoice update cannot finalize a block off the + /// canonical chain, and finalizing the canonical block instead would finalize a block the + /// caller never named. The block's identity is checked before it is decoded, so a mismatch is + /// reported as such rather than as a decoding failure. + async fn fetch_canonical_block_info(&self) -> Result { + let requested = self.block_id.number(); + debug!(target: "engine", block_id = ?self.block_id, "Fetching the block to finalize"); + let block = self + .client + .get_l2_block(requested.into()) + .full() + .await + .map_err(FinalizeTaskError::TransportError)? + .ok_or(FinalizeTaskError::BlockNotFound(requested))?; + let canonical = BlockNumHash { hash: block.header.hash, number: block.header.number }; + debug!(target: "engine", ?canonical, "Fetched the block to finalize"); + + if canonical.number != requested { + return Err(FinalizeTaskError::BlockNumberMismatch { + requested, + returned: canonical.number, + }); + } + if let FinalizeBlockId::ByHash(target) = self.block_id && + target.hash != canonical.hash + { + return Err(FinalizeTaskError::NotCanonical { canonical, target }); + } + + L2BlockInfo::from_block_and_genesis(&block.into_consensus(), &self.cfg.genesis) + .map_err(FinalizeTaskError::FromBlock) + } +} + #[async_trait] impl ExecutableEngineTask for FinalizeTask { type Output = (); @@ -31,40 +117,30 @@ impl ExecutableEngineTask for FinalizeTask { type Error = FinalizeTaskError; async fn execute(&self, state: &mut EngineState) -> Result<(), FinalizeTaskError> { - let block_number = self.block_id.number(); + let finalized_head = state.sync_state.finalized_head(); + let target_number = self.block_id.number(); + + // A lagging source can report an older finalized block, or one from another chain. A + // pinned hash is enough to detect either without asking the execution layer. + Self::ensure_not_behind_finalized_head(&finalized_head, target_number)?; + if let FinalizeBlockId::ByHash(target) = self.block_id && + Self::is_already_finalized(&finalized_head, target)? + { + return Ok(()); + } // Sanity check that the block that is being finalized is at least safe. - if state.sync_state.safe_head().block_info.number < block_number { + if state.sync_state.safe_head().block_info.number < target_number { return Err(FinalizeTaskError::BlockNotSafe); } - // Look up by hash when the caller pinned a specific hash (delegated polling supplies - // `(number, hash)`); otherwise look up by number. Finalization is irreversible, so a stale - // hash must produce a hard error, not silently finalize whatever the engine has at the - // same height. - let lookup: BlockId = match self.block_id { - FinalizeBlockId::ByHash(id) => id.hash.into(), - FinalizeBlockId::ByNumber(n) => n.into(), - }; - let block = self - .client - .get_l2_block(lookup) - .full() - .await - .map_err(FinalizeTaskError::TransportError)? - .ok_or(FinalizeTaskError::BlockNotFound(block_number))? - .into_consensus(); - let block_info = L2BlockInfo::from_block_and_genesis(&block, &self.client.cfg().genesis) - .map_err(FinalizeTaskError::FromBlock)?; - - // For ByHash, also assert the returned block's height matches the caller's claim. The - // engine should never serve a block at a different height for a given hash, but a height - // mismatch indicates either an upstream protocol bug or a misuse of the API and must - // never lead to silent finalization. - if let FinalizeBlockId::ByHash(id) = self.block_id && - block_info.block_info.number != id.number + let block_info = self.fetch_canonical_block_info().await?; + + // A by-number target only learns its hash from the lookup. + if matches!(self.block_id, FinalizeBlockId::ByNumber(_)) && + Self::is_already_finalized(&finalized_head, block_info.block_info.id())? { - return Err(FinalizeTaskError::BlockNotFound(id.number)); + return Ok(()); } // Dispatch a forkchoice update. diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs index 08c4f153e37..fc2243dcb6c 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs @@ -1,62 +1,249 @@ use crate::{ + EngineState, EngineSyncState, EngineSyncStateUpdate, EngineTaskError, EngineTaskErrorSeverity, ExecutableEngineTask, FinalizeBlockId, FinalizeTask, FinalizeTaskError, - test_utils::{TestEngineStateBuilder, test_engine_client}, + test_utils::{ + RpcMock, TestEngineStateBuilder, TestRpcBlock as RpcBlock, test_block_info, + test_engine_client, test_l2_block, test_rpc_block, + }, }; +use alloy_consensus::Header; use alloy_eips::BlockNumHash; -use alloy_primitives::b256; -use kona_genesis::RollupConfig; -use kona_protocol::{BlockInfo, L2BlockInfo}; +use alloy_primitives::B256; +use alloy_rpc_types_engine::PayloadStatusEnum; +use alloy_rpc_types_eth::{BlockTransactions, Header as RpcHeader}; +use kona_genesis::{ChainGenesis, RollupConfig}; +use kona_protocol::L2BlockInfo; +use rstest::rstest; use std::sync::Arc; -/// When the engine receives a `ByHash` finalize request for a block hash it doesn't have, -/// [`FinalizeTask`] must fail with [`FinalizeTaskError::BlockNotFound`] rather than silently -/// finalize whatever it happens to have at the same height. -/// -/// Assert the exact by-hash RPC lookup, so a by-number lookup cannot silently finalize a -/// different canonical block at the same height. -#[tokio::test] -async fn finalize_task_by_hash_errors_when_engine_lacks_hash() { - const N: u64 = 10; - let hash_a = b256!("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); - let hash_b = b256!("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"); +const STALE: u64 = 5; +const FINALIZED: u64 = 10; +const ADVANCED: u64 = 15; +const SAFE: u64 = 20; +const UNSAFE: u64 = 30; + +/// The L2 genesis block, which carries no L1-info deposit. +fn genesis_block() -> RpcBlock { + RpcBlock::new(RpcHeader::new(Header::default()), BlockTransactions::Full(vec![])) +} + +fn rollup_config() -> Arc { + let genesis = BlockNumHash { hash: genesis_block().header.hash, number: 0 }; + Arc::new(RollupConfig { + genesis: ChainGenesis { l2: genesis, ..Default::default() }, + ..Default::default() + }) +} + +/// An L2 block at `number`. Blocks built from different `fork` values share a height but not a +/// hash. +fn l2_block(number: u64, fork: u8) -> RpcBlock { + test_rpc_block(&test_l2_block(number, B256::ZERO, fork)) +} + +fn canonical(number: u64) -> RpcBlock { + l2_block(number, 0) +} + +fn conflicting(number: u64) -> RpcBlock { + l2_block(number, 1) +} + +fn l2_block_info(block: &RpcBlock) -> L2BlockInfo { + L2BlockInfo::from_block_and_genesis(&block.clone().into_consensus(), &rollup_config().genesis) + .unwrap() +} + +fn by_hash(block: &RpcBlock) -> FinalizeBlockId { + FinalizeBlockId::ByHash(l2_block_info(block).block_info.id()) +} - let cfg = Arc::new(RollupConfig::default()); +fn by_number(block: &RpcBlock) -> FinalizeBlockId { + FinalizeBlockId::ByNumber(block.header.number) +} + +fn established() -> L2BlockInfo { + l2_block_info(&canonical(FINALIZED)) +} + +fn uninitialized() -> L2BlockInfo { + L2BlockInfo::default() +} + +fn engine_state(finalized_head: L2BlockInfo) -> EngineState { + TestEngineStateBuilder::new() + .with_unsafe_head(test_block_info(UNSAFE)) + .with_safe_head(test_block_info(SAFE)) + .with_finalized_head(finalized_head) + .build() +} + +fn with_finalized_head(state: &EngineState, finalized_head: L2BlockInfo) -> EngineSyncState { + state.sync_state.apply_update(EngineSyncStateUpdate { + finalized_head: Some(finalized_head), + ..Default::default() + }) +} + +async fn finalize( + target: FinalizeBlockId, + state: &mut EngineState, + configure_engine: impl FnOnce(&RpcMock), +) -> Result<(), FinalizeTaskError> { + let cfg = rollup_config(); let (engine_client, l1, l2) = test_engine_client(cfg.clone()); - l2.expect_params( - "eth_getBlockByHash", - serde_json::json!([hash_b, true]), - Option::>::None, - ); - - // Place the safe head at N so the sanity check (`safe_head.number >= block_id.number`) passes - // and `execute()` reaches the lookup we care about. - let safe_head = L2BlockInfo { - block_info: BlockInfo { - number: N, - hash: hash_a, - parent_hash: Default::default(), - timestamp: N * 2, - }, - l1_origin: BlockNumHash::default(), - seq_num: 0, - }; - let mut state = - TestEngineStateBuilder::new().with_unsafe_head(safe_head).with_safe_head(safe_head).build(); - - let task = FinalizeTask::new( - Arc::new(engine_client), - cfg, - FinalizeBlockId::ByHash(BlockNumHash { number: N, hash: hash_b }), - ); - - let result = task.execute(&mut state).await; - - assert!( - matches!(result, Err(FinalizeTaskError::BlockNotFound(n)) if n == N), - "expected BlockNotFound({N}) — got {result:?}. The by-hash lookup must fail loudly when \ - the engine lacks the requested hash; instead, the task either succeeded (finalizing the \ - wrong block) or surfaced a different error." - ); + configure_engine(&l2); + + let result = FinalizeTask::new(Arc::new(engine_client), cfg, target).execute(state).await; + l1.assert_finished(); l2.assert_finished(); + result +} + +const fn is_behind(err: &FinalizeTaskError) -> bool { + matches!(err, FinalizeTaskError::BehindFinalizedHead { finalized: FINALIZED, target: STALE }) +} + +const fn is_conflict(err: &FinalizeTaskError) -> bool { + matches!(err, FinalizeTaskError::ConflictsWithFinalizedHead { .. }) +} + +const fn is_not_canonical(err: &FinalizeTaskError) -> bool { + matches!(err, FinalizeTaskError::NotCanonical { .. }) +} + +const fn is_number_mismatch(err: &FinalizeTaskError) -> bool { + matches!(err, FinalizeTaskError::BlockNumberMismatch { requested: ADVANCED, .. }) +} + +const fn is_block_not_found(err: &FinalizeTaskError) -> bool { + matches!(err, FinalizeTaskError::BlockNotFound(ADVANCED)) +} + +/// Targets behind the finalized head, or a pinned hash that conflicts with it, are skipped before +/// asking the execution layer for the block: the test engine expects no request at all. +#[rstest] +#[case::stale_by_hash(by_hash(&canonical(STALE)), is_behind)] +#[case::stale_by_number(by_number(&canonical(STALE)), is_behind)] +#[case::conflicting_by_hash(by_hash(&conflicting(FINALIZED)), is_conflict)] +#[tokio::test] +async fn finalize_task_skips_target_without_lookup( + #[case] target: FinalizeBlockId, + #[case] is_expected_error: fn(&FinalizeTaskError) -> bool, +) { + let mut state = engine_state(established()); + let original_state = state; + + let err = finalize(target, &mut state, |_| {}).await.unwrap_err(); + + assert!(is_expected_error(&err), "{err:?}"); + assert_eq!(err.severity(), EngineTaskErrorSeverity::Skip); + assert_eq!(state, original_state); +} + +/// A target whose block differs from the engine's canonical block is skipped. A pinned hash is +/// never replaced by the canonical block at its height, even above the finalized head, where a +/// delegate on another fork names a block the engine does not consider canonical. +#[rstest] +#[case::conflicting_by_number(by_number(&conflicting(FINALIZED)), conflicting(FINALIZED), is_conflict)] +#[case::other_fork_by_hash(by_hash(&conflicting(ADVANCED)), canonical(ADVANCED), is_not_canonical)] +#[tokio::test] +async fn finalize_task_skips_target_off_the_canonical_chain( + #[case] target: FinalizeBlockId, + #[case] served_block: RpcBlock, + #[case] is_expected_error: fn(&FinalizeTaskError) -> bool, +) { + let mut state = engine_state(established()); + let original_state = state; + + let err = finalize(target, &mut state, |l2| { + l2.expect_block_by_number(target.number(), Some(served_block)) + }) + .await + .unwrap_err(); + + assert!(is_expected_error(&err), "{err:?}"); + assert_eq!(err.severity(), EngineTaskErrorSeverity::Skip); + assert_eq!(state, original_state); +} + +/// A canonical target above the finalized head becomes the finalized head through one forkchoice +/// update. The canonical block at the finalized height is already finalized, so it needs no update. +#[rstest] +#[case::advancing_by_hash(established(), by_hash, canonical(ADVANCED), 1)] +#[case::advancing_by_number(established(), by_number, canonical(ADVANCED), 1)] +#[case::identical_by_number(established(), by_number, canonical(FINALIZED), 0)] +#[case::uninitialized_adopts_genesis(uninitialized(), by_hash, genesis_block(), 1)] +#[tokio::test] +async fn finalize_task_finalizes_canonical_target( + #[case] finalized_head: L2BlockInfo, + #[case] identify: fn(&RpcBlock) -> FinalizeBlockId, + #[case] served_block: RpcBlock, + #[case] forkchoice_updates: usize, +) { + let mut state = engine_state(finalized_head); + let target = identify(&served_block); + let expected_sync_state = with_finalized_head(&state, l2_block_info(&served_block)); + + finalize(target, &mut state, |l2| { + l2.expect_block_by_number(target.number(), Some(served_block)); + for _ in 0..forkchoice_updates { + l2.expect_forkchoice_updated( + expected_sync_state.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + } + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected_sync_state); +} + +/// A target pinned to the finalized head is already finalized, so it needs no lookup. +#[tokio::test] +async fn finalize_task_skips_lookup_for_finalized_head() { + let mut state = engine_state(established()); + let original_state = state; + + finalize(by_hash(&canonical(FINALIZED)), &mut state, |_| {}).await.unwrap(); + + assert_eq!(state, original_state); +} + +/// The engine serving no block, or a block at a height other than the requested one, stops the +/// node instead of finalizing anything. A misplaced block could otherwise bypass the guards against +/// moving finality backward. +#[rstest] +#[case::missing_by_number(FinalizeBlockId::ByNumber(ADVANCED), None, is_block_not_found)] +#[case::missing_by_hash(by_hash(&canonical(ADVANCED)), None, is_block_not_found)] +#[case::stale_block_by_number( + /* target */ FinalizeBlockId::ByNumber(ADVANCED), + /* served_block */ Some(canonical(STALE)), + /* is_expected_error */ is_number_mismatch +)] +#[case::higher_block_by_number( + /* target */ FinalizeBlockId::ByNumber(ADVANCED), + /* served_block */ Some(canonical(SAFE)), + /* is_expected_error */ is_number_mismatch +)] +#[case::stale_block_by_hash(by_hash(&canonical(ADVANCED)), Some(canonical(STALE)), is_number_mismatch)] +#[tokio::test] +async fn finalize_task_stops_on_unexpected_lookup_result( + #[case] target: FinalizeBlockId, + #[case] served_block: Option, + #[case] is_expected_error: fn(&FinalizeTaskError) -> bool, +) { + let mut state = engine_state(established()); + let original_state = state; + + let err = + finalize(target, &mut state, |l2| l2.expect_block_by_number(target.number(), served_block)) + .await + .unwrap_err(); + + assert!(is_expected_error(&err), "{err:?}"); + assert_eq!(err.severity(), EngineTaskErrorSeverity::Critical); + assert_eq!(state, original_state); } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/task.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/task.rs index ee976a02201..6b3a3d1b4cb 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/task.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/task.rs @@ -32,6 +32,10 @@ pub enum EngineTaskErrorSeverity { /// The error indicates that the engine should be flushed. #[display("flush")] Flush, + /// The task no longer applies to the engine state, so it is discarded instead of retried and + /// the engine keeps running. + #[display("skip")] + Skip, } /// The interface for an engine task error. diff --git a/rust/kona/crates/node/engine/src/test_utils/blocks.rs b/rust/kona/crates/node/engine/src/test_utils/blocks.rs new file mode 100644 index 00000000000..434d3bd01eb --- /dev/null +++ b/rust/kona/crates/node/engine/src/test_utils/blocks.rs @@ -0,0 +1,40 @@ +//! L2 blocks, and their RPC form as the execution layer serves them. + +use super::l1_info_deposit; +use alloy_consensus::transaction::Recovered; +use alloy_primitives::{Address, B256, Bytes}; +use alloy_rpc_types_eth::{BlockTransactions, Header as RpcHeader}; +use op_alloy_consensus::OpBlock; + +/// A full L2 block as the execution layer serves it over RPC. +pub type TestRpcBlock = alloy_rpc_types_eth::Block; + +/// An L2 block at `number` on `parent_hash` whose only transaction is the L1-info deposit, so it +/// converts into an `L2BlockInfo` without being the configured genesis. Blocks with different +/// `extra_data` share a height but not a hash. +pub fn test_l2_block(number: u64, parent_hash: B256, extra_data: u8) -> OpBlock { + let mut block = OpBlock::default(); + block.header.extra_data = Bytes::from(vec![extra_data]); + block.header.number = number; + block.header.parent_hash = parent_hash; + block.header.timestamp = number * 2; + block.body.transactions.push(l1_info_deposit()); + block +} + +/// `block` as the execution layer serves it over RPC. +pub fn test_rpc_block(block: &OpBlock) -> TestRpcBlock { + let transactions = block + .body + .transactions + .iter() + .cloned() + .map(|transaction| { + op_alloy_rpc_types::Transaction::from_transaction( + Recovered::new_unchecked(transaction, Address::ZERO), + Default::default(), + ) + }) + .collect(); + TestRpcBlock::new(RpcHeader::new(block.header.clone()), BlockTransactions::Full(transactions)) +} diff --git a/rust/kona/crates/node/engine/src/test_utils/mod.rs b/rust/kona/crates/node/engine/src/test_utils/mod.rs index e5949f23225..0a60403a0d6 100644 --- a/rust/kona/crates/node/engine/src/test_utils/mod.rs +++ b/rust/kona/crates/node/engine/src/test_utils/mod.rs @@ -1,6 +1,9 @@ mod attributes; pub use attributes::TestAttributesBuilder; +mod blocks; +pub use blocks::{TestRpcBlock, test_l2_block, test_rpc_block}; + mod deposits; pub use deposits::l1_info_deposit; diff --git a/rust/kona/crates/node/engine/src/test_utils/rpc.rs b/rust/kona/crates/node/engine/src/test_utils/rpc.rs index 1cfc3995356..39f23f14de2 100644 --- a/rust/kona/crates/node/engine/src/test_utils/rpc.rs +++ b/rust/kona/crates/node/engine/src/test_utils/rpc.rs @@ -1,6 +1,6 @@ //! Scripted RPC expectations over Alloy's mock transport. -use crate::EngineClient; +use crate::{EngineClient, test_utils::TestRpcBlock}; use alloy_json_rpc::{ErrorPayload, RequestPacket, ResponsePacket}; use alloy_network::Network; use alloy_provider::RootProvider; @@ -59,6 +59,16 @@ impl RpcMock { ); } + /// Expects a full-block `eth_getBlockByNumber` call for `number` and answers it with `block`, + /// or with null for `None`. + pub fn expect_block_by_number(&self, number: u64, block: Option) { + self.expect_params( + "eth_getBlockByNumber", + serde_json::json!([format!("{number:#x}"), true]), + block, + ); + } + /// Expects an `engine_forkchoiceUpdatedV3` call for `forkchoice` without payload attributes and /// answers it with `status`. pub fn expect_forkchoice_updated( diff --git a/rust/kona/crates/node/service/src/actors/derivation/actor/core.rs b/rust/kona/crates/node/service/src/actors/derivation/actor/core.rs index e3a5ea1ba4d..830b7ce4eaf 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/actor/core.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/actor/core.rs @@ -369,8 +369,9 @@ where self.attempt_derivation().await?; } - DerivationActorRequest::ProcessEngineSyncCompletionRequest(safe_head) => { + DerivationActorRequest::ProcessEngineSyncCompletionRequest(sync_state) => { info!(target: "derivation", "Engine finished syncing, starting derivation."); + let safe_head = Box::new(sync_state.local_safe_head()); self.derivation_state_machine .update(&DerivationStateUpdate::ELSyncCompleted(safe_head.clone()))?; self.truncate_safe_db(*safe_head, "EL sync completion")?; diff --git a/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs b/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs index 5bd4f9cfc00..01eb187e119 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs @@ -10,7 +10,7 @@ use kona_derive::{ ActivationSignal, Pipeline, PipelineError, PipelineErrorKind, PipelineResult, ResetError, Signal, SignalReceiver, StepResult, }; -use kona_engine::FinalizeBlockId; +use kona_engine::{EngineSyncState, EngineSyncStateUpdate, FinalizeBlockId}; use kona_genesis::{HardForkConfig, RollupConfig, SystemConfig}; use kona_protocol::{BlockInfo, L2BlockInfo, OpAttributesWithParent}; use kona_safedb::{SafeDatabase, SafeDb, SafeDbError, SafeHeadRecord}; @@ -41,12 +41,7 @@ async fn reset_adopts_safe_head_and_resumes_atomically(#[case] previous_head: u6 l1_block(0).id(), watch::channel(None).0, ); - request_tx - .send(DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(l2_block( - previous_head, - )))) - .await - .unwrap(); + request_tx.send(engine_sync_completed_at(l2_block(previous_head))).await.unwrap(); actor.step().await.unwrap(); request_tx @@ -267,10 +262,7 @@ async fn test_origin_is_published(#[case] step_result: StepResult) { actor.step().await.unwrap(); assert_eq!(derivation_origin_rx.borrow().map(|origin| origin.id()), Some(safe_head.l1_origin)); - request_tx - .send(DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(safe_head))) - .await - .unwrap(); + request_tx.send(engine_sync_completed_at(safe_head)).await.unwrap(); actor.step().await.unwrap(); assert_eq!(derivation_origin_rx.borrow().unwrap().number, 11); @@ -465,6 +457,16 @@ fn l1_block(number: u64) -> BlockInfo { BlockInfo { hash: B256::with_last_byte(number as u8), number, ..Default::default() } } +/// The engine reporting that its sync completed with `safe_head` as its safe heads. +fn engine_sync_completed_at(safe_head: L2BlockInfo) -> DerivationActorRequest { + let sync_state = EngineSyncState::default().apply_update(EngineSyncStateUpdate { + local_safe_head: Some(safe_head), + safe_head: Some(safe_head), + ..Default::default() + }); + DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(sync_state)) +} + fn l2_block(number: u64) -> L2BlockInfo { L2BlockInfo { block_info: BlockInfo { @@ -535,10 +537,7 @@ impl RecordingActor { } async fn complete_el_sync_at(&mut self, head: u64) { - self.handle(DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new( - l2_block(head), - ))) - .await; + self.handle(engine_sync_completed_at(l2_block(head))).await; } async fn confirm_safe_head(&mut self, head: u64) { diff --git a/rust/kona/crates/node/service/src/actors/derivation/actor/tests/database.rs b/rust/kona/crates/node/service/src/actors/derivation/actor/tests/database.rs index 699bc1e6933..eeee778a78a 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/actor/tests/database.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/actor/tests/database.rs @@ -135,7 +135,7 @@ fn engine_reset_request() -> DerivationActorRequest { } fn sync_completion_request() -> DerivationActorRequest { - DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(l2_block(RESUME_HEAD))) + engine_sync_completed_at(l2_block(RESUME_HEAD)) } #[rstest] diff --git a/rust/kona/crates/node/service/src/actors/derivation/actor/tests/op_node_parity.rs b/rust/kona/crates/node/service/src/actors/derivation/actor/tests/op_node_parity.rs index 104544f2522..0df6cef81c1 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/actor/tests/op_node_parity.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/actor/tests/op_node_parity.rs @@ -16,7 +16,7 @@ fn l1_head_update(number: u64) -> DerivationActorRequest { } fn el_sync_completed(head: u64) -> DerivationActorRequest { - DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(l2_block(head))) + engine_sync_completed_at(l2_block(head)) } /// A reset whose pipeline signal fails once with a temporary provider error neither stops the diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs index 44fe763d969..7001d2f71d6 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs @@ -2,10 +2,11 @@ use crate::{ DerivationActorRequest, DerivationEngineClient, NodeActor, actors::derivation::{DerivationDelegateProvider, DerivationError}, }; +use alloy_eips::BlockNumHash; use alloy_primitives::BlockHash; use async_trait::async_trait; use kona_derive::{ChainProvider, Signal}; -use kona_engine::FinalizeBlockId; +use kona_engine::{EngineState, FinalizeBlockId}; use kona_protocol::{BlockInfo, L2BlockInfo, SyncStatus}; use thiserror::Error; use tokio::{ @@ -42,6 +43,10 @@ where /// Publishes the delegate's `current_l1`: the L1 block that its derivation has reached. derivation_origin_tx: watch::Sender>, + /// The engine's state as it publishes it. Delegate statuses are checked against the + /// finalized head the engine applied, not the last one forwarded to it, which the engine may + /// have skipped. + engine_state_rx: watch::Receiver, /// The engine's L2 safe head, according to updates from the Engine. engine_l2_safe_head: L2BlockInfo, /// Whether the engine sync has completed. This will only ever go from false -> true. @@ -64,6 +69,7 @@ where derivation_delegate_provider: DelegateProvider, l1_provider: L1Provider, derivation_origin_tx: watch::Sender>, + engine_state_rx: watch::Receiver, ) -> Self { let mut delegated_derivation_ticker = time::interval(Self::DERIVATION_DELEGATE_POLL_INTERVAL); @@ -74,6 +80,7 @@ where derivation_delegate_provider, l1_provider, derivation_origin_tx, + engine_state_rx, engine_l2_safe_head: L2BlockInfo::default(), has_engine_sync_completed: false, delegated_derivation_ticker, @@ -149,9 +156,44 @@ where Ok(()) } - /// Verifies that the L1 info reported by the derivation delegate - /// are consistent with canonical L1 chain. + /// The id of an established finalized head. The default head means finality has not been + /// established yet, so any finalized block from the delegate may initialize it. + fn established_finalized_l2(finalized_head: L2BlockInfo) -> Option { + (finalized_head != L2BlockInfo::default()).then(|| finalized_head.block_info.id()) + } + + /// Rejects a finalized L2 block behind, or at the same height as but different from, the + /// engine's. Either means the delegate lags or follows another chain, so none of its status + /// applies: its safe head would move the engine back or onto that chain too. + fn validate_finalized_l2_progress( + &self, + reported: BlockNumHash, + ) -> Result<(), DerivationDelegationError> { + let engine_finalized_head = self.engine_state_rx.borrow().sync_state.finalized_head(); + let Some(engine) = Self::established_finalized_l2(engine_finalized_head) else { + return Ok(()); + }; + if reported.number < engine.number { + return Err(DerivationDelegationError::FinalizedL2Regressed { engine, reported }); + } + if reported.number == engine.number && reported.hash != engine.hash { + return Err(DerivationDelegationError::FinalizedL2Conflict { engine, reported }); + } + Ok(()) + } + + /// Verifies that the finalized L2 block reported by the derivation delegate does not move back + /// from the engine's, and that its L1 info is consistent with the canonical L1 chain. async fn validate_sync_status(&mut self, v: &SyncStatus) -> bool { + if let Err(err) = self.validate_finalized_l2_progress(v.finalized_l2.block_info.id()) { + warn!( + target: "derivation", + error = %err, + "Finalized L2 inconsistency detected at sync status from delegate" + ); + return false; + } + let checks = [ ("L1 Origin of Safe L2", v.safe_l2.l1_origin.number, v.safe_l2.l1_origin.hash), ( @@ -227,9 +269,9 @@ where debug!(target: "derivation", safe_head = ?*safe_head, "Received safe head from engine."); self.engine_l2_safe_head = *safe_head; } - DerivationActorRequest::ProcessEngineSyncCompletionRequest(safe_head) => { + DerivationActorRequest::ProcessEngineSyncCompletionRequest(sync_state) => { info!(target: "derivation", "Engine finished syncing, starting derivation."); - self.engine_l2_safe_head = *safe_head; + self.engine_l2_safe_head = sync_state.local_safe_head(); self.has_engine_sync_completed = true; } DerivationActorRequest::ProcessEngineSignalRequest(signal) => match *signal { @@ -249,6 +291,14 @@ where #[derive(Error, Debug)] enum DerivationDelegationError { + /// The delegate's finalized L2 block is a different block at the engine's finalized height. + #[error("Finalized L2 {reported:?} conflicts with the engine's finalized L2 {engine:?}")] + FinalizedL2Conflict { engine: BlockNumHash, reported: BlockNumHash }, + + /// The delegate's finalized L2 block is behind the engine's. + #[error("Finalized L2 {reported:?} is behind the engine's finalized L2 {engine:?}")] + FinalizedL2Regressed { engine: BlockNumHash, reported: BlockNumHash }, + /// The L1 provider returned an error (network, RPC, etc.) #[error("L1 provider error: {0}")] L1Provider(String), diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/mod.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/mod.rs index e5460eceb11..5431e58aa93 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/mod.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/mod.rs @@ -5,3 +5,6 @@ mod client; pub use client::{ DerivationDelegateClient, DerivationDelegateClientError, DerivationDelegateProvider, }; + +#[cfg(test)] +mod tests; diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs new file mode 100644 index 00000000000..8f976aa256c --- /dev/null +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs @@ -0,0 +1,250 @@ +use super::{DelegateDerivationActor, DerivationDelegateClientError, DerivationDelegateProvider}; +use crate::{DerivationActorRequest, EngineActorRequest, NodeActor, QueuedDerivationEngineClient}; +use alloy_eips::BlockNumHash; +use alloy_primitives::B256; +use async_trait::async_trait; +use kona_derive::test_utils::TestChainProvider; +use kona_engine::{ + ConsolidateInput, EngineState, EngineSyncState, EngineSyncStateUpdate, FinalizeBlockId, +}; +use kona_protocol::{BlockInfo, L2BlockInfo, SyncStatus}; +use rstest::rstest; +use std::{collections::VecDeque, sync::Mutex}; +use tokio::sync::{mpsc, watch}; + +const SAFE_ORIGIN: u64 = 1; +const FINALIZED_ORIGIN: u64 = 2; +const CURRENT_L1: u64 = 3; + +type SyncStatusReply = Result; + +/// Answers each sync status poll with the next scripted reply. +#[derive(Debug)] +struct ScriptedDelegate(Mutex>); + +#[async_trait] +impl DerivationDelegateProvider for ScriptedDelegate { + async fn fetch_sync_status(&self) -> SyncStatusReply { + self.0.lock().unwrap().pop_front().expect("unexpected sync status poll") + } +} + +/// A delegate actor over a canonical L1 chain, with the engine requests it sends and the +/// `current_l1` it publishes. +struct Harness { + actor: + DelegateDerivationActor, + current_l1: watch::Receiver>, + derivation_requests: mpsc::Sender, + engine_requests: mpsc::Receiver, + engine_state: watch::Sender, +} + +impl Harness { + fn new(replies: impl IntoIterator) -> Self { + let mut l1 = TestChainProvider::default(); + for number in [SAFE_ORIGIN, FINALIZED_ORIGIN, CURRENT_L1] { + l1.insert_block(number, canonical_l1_block(number)); + } + let (derivation_requests, derivation_request_rx) = mpsc::channel(8); + let (engine_request_tx, engine_requests) = mpsc::channel(8); + let (current_l1_tx, current_l1) = watch::channel(None); + let (engine_state, engine_state_rx) = watch::channel(EngineState::default()); + let actor = DelegateDerivationActor::new( + QueuedDerivationEngineClient::new(engine_request_tx), + derivation_request_rx, + ScriptedDelegate(Mutex::new(replies.into_iter().collect())), + l1, + current_l1_tx, + engine_state_rx, + ); + Self { actor, current_l1, derivation_requests, engine_requests, engine_state } + } + + async fn complete_engine_sync(&mut self) { + self.complete_engine_sync_with(EngineSyncState::default()).await; + } + + /// Completes the engine sync with `sync_state` as the engine's labels, which the engine + /// publishes before it reports the completion. + async fn complete_engine_sync_with(&mut self, sync_state: EngineSyncState) { + self.publish_engine_sync_state(sync_state); + self.derivation_requests + .send(DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(sync_state))) + .await + .unwrap(); + self.actor.step().await.unwrap(); + } + + /// Publishes the engine's labels as the engine does after each task it runs. + fn publish_engine_sync_state(&self, sync_state: EngineSyncState) { + self.engine_state.send_modify(|state| state.sync_state = sync_state); + } + + /// The engine requests sent since the last call, in order. + fn forwarded(&mut self) -> Vec { + std::iter::from_fn(|| self.engine_requests.try_recv().ok()).map(Forwarded::from).collect() + } +} + +/// The parts of an engine request that the delegate actor decides. +#[derive(Debug, PartialEq, Eq)] +enum Forwarded { + Finalized(FinalizeBlockId), + Other(String), + Safe(L2BlockInfo), +} + +impl From for Forwarded { + fn from(request: EngineActorRequest) -> Self { + match request { + EngineActorRequest::ProcessFinalizedL2Block(finalized) => Self::Finalized(*finalized), + EngineActorRequest::ProcessSafeL2Signal(ConsolidateInput::BlockInfo(safe)) => { + Self::Safe(safe) + } + other => Self::Other(format!("{other:?}")), + } + } +} + +/// What one poll of `status` forwards: the safe head, then the finalized block pinned by hash. +fn forwarded_for(status: &SyncStatus) -> Vec { + vec![ + Forwarded::Safe(status.safe_l2), + Forwarded::Finalized(FinalizeBlockId::ByHash(status.finalized_l2.block_info.id())), + ] +} + +/// What one poll forwards when the status is accepted, and nothing when it is rejected. +fn forwarded_if(accepted: bool, status: &SyncStatus) -> Vec { + if accepted { forwarded_for(status) } else { Vec::new() } +} + +fn canonical_l1_block(number: u64) -> BlockInfo { + BlockInfo { hash: B256::with_last_byte(number as u8), number, ..Default::default() } +} + +fn l2_block(number: u64, l1_origin: u64) -> L2BlockInfo { + L2BlockInfo { + block_info: BlockInfo { + hash: B256::with_last_byte(0x80 | number as u8), + number, + ..Default::default() + }, + l1_origin: canonical_l1_block(l1_origin).id(), + seq_num: 0, + } +} + +/// A sync status whose safe origin, finalized origin and `current_l1` are all canonical. +fn delegate_status() -> SyncStatus { + SyncStatus { + current_l1: canonical_l1_block(CURRENT_L1), + current_l1_finalized: BlockInfo::default(), + finalized_l1: BlockInfo::default(), + finalized_l2: l2_block(15, FINALIZED_ORIGIN), + head_l1: BlockInfo::default(), + local_safe_l2: l2_block(20, SAFE_ORIGIN), + safe_l1: BlockInfo::default(), + safe_l2: l2_block(20, SAFE_ORIGIN), + unsafe_l2: l2_block(25, SAFE_ORIGIN), + } +} + +fn non_canonical(id: BlockNumHash) -> BlockNumHash { + BlockNumHash { hash: B256::repeat_byte(0xee), ..id } +} + +/// Returns `status` with its finalized L2 block replaced by `finalized_l2`. +fn with_finalized_l2(finalized_l2: L2BlockInfo) -> SyncStatus { + SyncStatus { finalized_l2, ..delegate_status() } +} + +/// An engine sync state whose finalized head is `finalized_l2`. +fn engine_finalized_at(finalized_l2: L2BlockInfo) -> EngineSyncState { + EngineSyncState::default().apply_update(EngineSyncStateUpdate { + finalized_head: Some(finalized_l2), + ..Default::default() + }) +} + +fn conflicting(block: L2BlockInfo) -> L2BlockInfo { + L2BlockInfo { + block_info: BlockInfo { + hash: non_canonical(block.block_info.id()).hash, + ..block.block_info + }, + ..block + } +} + +/// The engine's finalized head when its sync completes is the baseline, so a delegate that starts +/// behind it, or on another chain at its height, is rejected whole from the first poll on. +#[rstest] +#[case::advancing(l2_block(16, FINALIZED_ORIGIN), true)] +#[case::unchanged(l2_block(15, FINALIZED_ORIGIN), true)] +#[case::regressed(l2_block(14, FINALIZED_ORIGIN), false)] +#[case::conflicting(conflicting(l2_block(15, FINALIZED_ORIGIN)), false)] +#[tokio::test(start_paused = true)] +async fn first_poll_rejects_status_behind_engine_finality( + #[case] finalized_l2: L2BlockInfo, + #[case] accepted: bool, +) { + let status = with_finalized_l2(finalized_l2); + let mut harness = Harness::new([Ok(status.clone())]); + harness.complete_engine_sync_with(engine_finalized_at(l2_block(15, FINALIZED_ORIGIN))).await; + + harness.actor.step().await.unwrap(); + + assert_eq!(harness.forwarded(), forwarded_if(accepted, &status)); + assert_eq!(harness.current_l1.borrow().is_some(), accepted); +} + +/// Once the engine applies a forwarded finalized block, a status whose finalized block is behind +/// it, or is a different block at its height, is rejected whole: a lagging or diverged delegate's +/// safe head must not reach the engine either, and its `current_l1` is not published. +#[rstest] +#[case::advancing(l2_block(16, FINALIZED_ORIGIN), true)] +#[case::unchanged(l2_block(15, FINALIZED_ORIGIN), true)] +#[case::regressed(l2_block(14, FINALIZED_ORIGIN), false)] +#[case::conflicting(conflicting(l2_block(15, FINALIZED_ORIGIN)), false)] +#[tokio::test(start_paused = true)] +async fn poll_rejects_status_whose_finality_moves_back( + #[case] next_finalized_l2: L2BlockInfo, + #[case] accepted: bool, +) { + let next_status = with_finalized_l2(next_finalized_l2); + let mut harness = Harness::new([Ok(delegate_status()), Ok(next_status.clone())]); + harness.complete_engine_sync().await; + harness.actor.step().await.unwrap(); + assert_eq!(harness.forwarded(), forwarded_for(&delegate_status())); + harness.publish_engine_sync_state(engine_finalized_at(delegate_status().finalized_l2)); + harness.current_l1.borrow_and_update(); + + harness.actor.step().await.unwrap(); + + assert_eq!(harness.forwarded(), forwarded_if(accepted, &next_status)); + assert_eq!(harness.current_l1.has_changed().unwrap(), accepted); +} + +/// A finalized block the engine skipped, such as one on another chain, does not become the +/// baseline: the canonical block at its height, or below it but above the engine's finalized +/// head, is still accepted, so the delegate's safe head keeps reaching the engine. +#[rstest] +#[case::canonical_at_skipped_height(l2_block(17, FINALIZED_ORIGIN))] +#[case::canonical_below_skipped_height(l2_block(16, FINALIZED_ORIGIN))] +#[tokio::test(start_paused = true)] +async fn poll_accepts_status_after_engine_skipped_forwarded_finality( + #[case] canonical_finalized_l2: L2BlockInfo, +) { + let skipped_status = with_finalized_l2(conflicting(l2_block(17, FINALIZED_ORIGIN))); + let canonical_status = with_finalized_l2(canonical_finalized_l2); + let mut harness = Harness::new([Ok(skipped_status.clone()), Ok(canonical_status.clone())]); + harness.complete_engine_sync_with(engine_finalized_at(l2_block(15, FINALIZED_ORIGIN))).await; + harness.actor.step().await.unwrap(); + assert_eq!(harness.forwarded(), forwarded_for(&skipped_status)); + + harness.actor.step().await.unwrap(); + + assert_eq!(harness.forwarded(), forwarded_for(&canonical_status)); +} diff --git a/rust/kona/crates/node/service/src/actors/derivation/request.rs b/rust/kona/crates/node/service/src/actors/derivation/request.rs index 9482eef5582..caa1e2b9ad2 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/request.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/request.rs @@ -1,4 +1,5 @@ use kona_derive::Signal; +use kona_engine::EngineSyncState; use kona_protocol::{BlockInfo, L2BlockInfo}; use thiserror::Error; @@ -21,9 +22,9 @@ pub enum DerivationClientError { /// Inbound requests that the [`crate::DerivationActor`] can process. #[derive(Debug)] pub enum DerivationActorRequest { - /// Request to process the fact that Engine sync has completed, along with the current safe - /// head. - ProcessEngineSyncCompletionRequest(Box), + /// Request to process the fact that Engine sync has completed, along with the engine's sync + /// state at that point. + ProcessEngineSyncCompletionRequest(Box), /// Request to process the provided L2 engine safe head update. ProcessEngineSafeHeadUpdateRequest(Box), /// A request containing a [`Signal`] to the derivation pipeline. diff --git a/rust/kona/crates/node/service/src/actors/engine/actor.rs b/rust/kona/crates/node/service/src/actors/engine/actor.rs index c9affd16523..87619ba2756 100644 --- a/rust/kona/crates/node/service/src/actors/engine/actor.rs +++ b/rust/kona/crates/node/service/src/actors/engine/actor.rs @@ -5,8 +5,8 @@ use async_trait::async_trait; use kona_derive::{ResetSignal, Signal}; use kona_engine::{ BuildTask, CanonicalizeTask, ConsolidateTask, Engine, EngineActorRequest, EngineClient, - EngineTask, EngineTaskError, EngineTaskErrorSeverity, FinalizeTask, ImportedBlockSink, - InsertTask, SealTask, + EngineTask, EngineTaskError, EngineTaskErrorSeverity, EngineTaskErrors, FinalizeTask, + ImportedBlockSink, InsertTask, SealTask, }; use kona_genesis::RollupConfig; use kona_protocol::L2BlockInfo; @@ -34,6 +34,9 @@ where el_sync_complete: bool, /// The last local-safe head sent to the derivation actor. last_engine_safe_head_sent: L2BlockInfo, + /// The error of the last skipped engine task, so an identical repeat logs no new warning + /// until a reset changes the state the task was judged against. + last_skipped_task_error: Option, /// A channel to use to relay the current unsafe head. /// ## Note /// This is `Some` when the node is in sequencer mode, and `None` when the node is in validator @@ -81,6 +84,7 @@ where el_sync_complete: false, engine, last_engine_safe_head_sent: L2BlockInfo::default(), + last_skipped_task_error: None, rollup: config, unsafe_head_tx, inbound_request_rx, @@ -92,6 +96,18 @@ where } } + /// Logs a skipped task at `warn!` when its error differs from the last skip, and at `debug!` + /// while the identical error repeats, so a target resubmitted on every poll warns once. + fn log_skipped_task(&mut self, err: &EngineTaskErrors) { + let message = err.to_string(); + if self.last_skipped_task_error.as_ref() == Some(&message) { + debug!(target: "engine", %err, "Skipped the same engine task again"); + return; + } + warn!(target: "engine", %err, "Skipped an engine task that no longer applies"); + self.last_skipped_task_error = Some(message); + } + fn defer_retry(&mut self) { self.retry_at = Instant::now() + self.retry_delay; self.retry_delay = (self.retry_delay * 2).min(Duration::from_secs(5)); @@ -100,6 +116,7 @@ where fn begin_reset(&mut self) { if !self.reset_pending { self.engine.clear(); + self.last_skipped_task_error = None; self.reset_pending = true; self.retry_at = Instant::now(); } @@ -210,6 +227,7 @@ where self.defer_retry(); return Ok(()); } + EngineTaskErrorSeverity::Skip => self.log_skipped_task(&err), } } } @@ -233,7 +251,7 @@ where self.el_sync_complete = true; self.derivation_client - .notify_sync_completed(self.engine.state().sync_state.local_safe_head()) + .notify_sync_completed(self.engine.state().sync_state) .await .map(|_| Ok(())) .map_err(|e| { diff --git a/rust/kona/crates/node/service/src/actors/engine/client.rs b/rust/kona/crates/node/service/src/actors/engine/client.rs index 25322a5a7f2..518e7b957d0 100644 --- a/rust/kona/crates/node/service/src/actors/engine/client.rs +++ b/rust/kona/crates/node/service/src/actors/engine/client.rs @@ -2,6 +2,7 @@ use crate::{DerivationActorRequest, DerivationClientError, DerivationClientResul use async_trait::async_trait; use derive_more::Constructor; use kona_derive::Signal; +use kona_engine::EngineSyncState; use kona_protocol::L2BlockInfo; use std::fmt::Debug; use tokio::sync::mpsc; @@ -10,9 +11,13 @@ use tokio::sync::mpsc; #[cfg_attr(test, mockall::automock)] #[async_trait] pub trait EngineDerivationClient: Debug + Send + Sync { - /// Notifies the [`crate::DerivationActor`] that engine syncing has completed. + /// Notifies the [`crate::DerivationActor`] that engine syncing has completed, with the engine's + /// sync state at that point. /// Note: Does not wait for the derivation client to process this message. - async fn notify_sync_completed(&self, safe_head: L2BlockInfo) -> DerivationClientResult<()>; + async fn notify_sync_completed( + &self, + sync_state: EngineSyncState, + ) -> DerivationClientResult<()>; /// Sends the new engine `safe_head` to the [`crate::DerivationActor`]. /// Note: Does not wait for the derivation client to process this message. @@ -33,11 +38,14 @@ pub struct QueuedEngineDerivationClient { #[async_trait] impl EngineDerivationClient for QueuedEngineDerivationClient { - async fn notify_sync_completed(&self, safe_head: L2BlockInfo) -> DerivationClientResult<()> { + async fn notify_sync_completed( + &self, + sync_state: EngineSyncState, + ) -> DerivationClientResult<()> { info!(target: "engine", "Sending sync completed to derivation actor"); self.derivation_actor_request_tx - .send(DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(safe_head))) + .send(DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(sync_state))) .await .map_err(|_| { DerivationClientError::RequestError("request channel closed.".to_string()) diff --git a/rust/kona/crates/node/service/src/actors/engine/tests.rs b/rust/kona/crates/node/service/src/actors/engine/tests.rs index b61712d95b5..d000710cbf1 100644 --- a/rust/kona/crates/node/service/src/actors/engine/tests.rs +++ b/rust/kona/crates/node/service/src/actors/engine/tests.rs @@ -312,5 +312,5 @@ async fn derivation_is_handed_the_local_safe_head() { else { panic!("expected the sync completion second"); }; - assert_eq!(*sent, local_safe_head); + assert_eq!(sent.local_safe_head(), local_safe_head); } diff --git a/rust/kona/crates/node/service/src/service/node/rollup.rs b/rust/kona/crates/node/service/src/service/node/rollup.rs index d1b5fb0a12e..ed509b22101 100644 --- a/rust/kona/crates/node/service/src/service/node/rollup.rs +++ b/rust/kona/crates/node/service/src/service/node/rollup.rs @@ -289,6 +289,7 @@ impl RollupNode { engine_actor_request_tx: mpsc::Sender, derivation_actor_request_rx: mpsc::Receiver, derivation_origin_tx: watch::Sender>, + engine_state_rx: watch::Receiver, l1_head: watch::Receiver>, ) -> Result { if let Some(provider) = self.derivation_delegate_provider.clone() { @@ -303,6 +304,7 @@ impl RollupNode { provider, l1_provider, derivation_origin_tx, + engine_state_rx, )))) } else { let pipeline = self @@ -621,6 +623,7 @@ impl RollupNode { engine_actor_request_tx.clone(), derivation_actor_request_rx, derivation_origin_tx, + engine_state_rx.clone(), l1_head_updates_rx.clone(), ) .await?; From 4f6808da8a40c3d295d58c016b260302d087db14 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Fri, 6 Nov 2026 20:36:50 -0800 Subject: [PATCH 03/14] kona-engine: keep delegated finalization pending during EL catch-up A finalization target the engine cannot apply yet no longer stops kona-node or holds the engine queue. A target above the safe head fails with `BlockNotSafe`, now of `Skip` severity, whatever its source, as op-node's promoteFinalized does: a delegate can run ahead of the local safe head, and a reset can lower the safe head below a target that was safe when requested (#23757, #23440). A hash-pinned target the execution layer has no block for yet fails with the new skipped `NotYetAvailable`, which carries the answer, whether the lookup returned null or an error response, so a retried lookup cannot hold the queue ahead of the imports that let the execution layer catch up. The delegate poll resubmits its target every 4s. A by-number target from L1 finality that a reset skipped is no longer lost until L1 finality moves again: like op-node's retry when derivation goes idle, the finalizer keeps the highest finalized L1 number and finalizes newly safe blocks against it on each L1 head update, which batches the blocks confirmed since the last update. A by-number target whose block is missing still fails with `BlockNotFound`. Tests cover the finalizer, a block derived again after a reset, a delegated poll during execution-layer sync at the actor level, and the error variant of each unavailable-block answer; `test_utils::expect_sync_start` replaces two copies of the reset's RPC script. BREAKING CHANGE: `FinalizeTaskError` has a new variant, `NotYetAvailable { engine_answer, target }`; exhaustive matches must handle it. `BlockNotSafe` keeps its name but has `Skip` severity instead of `Critical`. Co-Authored-By: Claude Opus 5.5 Co-Authored-By: Claude Fable 5.1 --- .../crates/node/engine/src/task_queue/core.rs | 12 +- .../src/task_queue/tasks/finalize/error.rs | 23 +- .../src/task_queue/tasks/finalize/id.rs | 7 +- .../src/task_queue/tasks/finalize/task.rs | 38 ++- .../task_queue/tasks/finalize/task_test.rs | 143 ++++++++-- .../crates/node/engine/src/test_utils/mod.rs | 2 +- .../crates/node/engine/src/test_utils/rpc.rs | 16 ++ .../src/actors/derivation/actor/core.rs | 32 ++- .../src/actors/derivation/actor/tests.rs | 46 ++++ .../src/actors/derivation/delegated/tests.rs | 86 +++++- .../src/actors/derivation/finalizer.rs | 112 +++++++- .../service/src/actors/engine/queue_tests.rs | 245 ++++++++++++++++-- 12 files changed, 676 insertions(+), 86 deletions(-) diff --git a/rust/kona/crates/node/engine/src/task_queue/core.rs b/rust/kona/crates/node/engine/src/task_queue/core.rs index e9d9dcb4f1c..28381216df5 100644 --- a/rust/kona/crates/node/engine/src/task_queue/core.rs +++ b/rust/kona/crates/node/engine/src/task_queue/core.rs @@ -205,7 +205,8 @@ mod recovery_tests { ConsolidateInput, ConsolidateTask, FinalizeBlockId, FinalizeTask, InsertTask, NoopBlockSink, test_utils::{ - TestAttributesBuilder, TestEngineStateBuilder, test_block_info, test_engine_client, + TestAttributesBuilder, TestEngineStateBuilder, expect_sync_start, test_block_info, + test_engine_client, }, }; use alloy_rpc_types_engine::{ @@ -369,14 +370,7 @@ mod recovery_tests { let state = TestEngineStateBuilder::new().with_unsafe_head(genesis).build(); let forkchoice = state.sync_state.create_forkchoice_state(); let (client, l1, l2) = test_engine_client(config.clone()); - for tag in ["finalized", "safe", "latest"] { - l2.expect_params("eth_getBlockByNumber", serde_json::json!([tag, true]), &block); - } - l1.expect( - "eth_getBlockByHash", - alloy_rpc_types_eth::Block::::default(), - ); - l2.expect_forkchoice_updated(forkchoice, PayloadStatusEnum::Valid); + expect_sync_start(&l1, &l2, &block, forkchoice); let (state_tx, _state_rx) = tokio::sync::watch::channel(state); let mut engine = Engine::new(state, state_tx); diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/error.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/error.rs index 51a7f2c1dda..d8f36318dec 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/error.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/error.rs @@ -22,7 +22,9 @@ pub enum FinalizeTaskError { /// The block to finalize was not found. #[error("The block to finalize was not found: Number {0}")] BlockNotFound(u64), - /// The block is not safe, and therefore cannot be finalized. + /// The block is not safe yet, and therefore cannot be finalized. A reset can lower the safe + /// head below a target that was safe when it was requested, and a delegate can run ahead of + /// the local safe head. #[error("Attempted to finalize a block that is not yet safe")] BlockNotSafe, /// The engine returned a block at a different height than the one requested. @@ -57,6 +59,14 @@ pub enum FinalizeTaskError { /// The finalization target. target: BlockNumHash, }, + /// The engine has no block at the pinned target's height yet, for example while it catches up. + #[error("The engine does not have finalization target {target:?} yet: {engine_answer}")] + NotYetAvailable { + /// What the engine answered the lookup with: null, or an error response. + engine_answer: String, + /// The finalization target. + target: BlockNumHash, + }, /// A temporary RPC failure. #[error(transparent)] TransportError(#[from] RpcError), @@ -66,12 +76,13 @@ impl EngineTaskError for FinalizeTaskError { fn severity(&self) -> EngineTaskErrorSeverity { match self { Self::BehindFinalizedHead { .. } | - Self::ConflictsWithFinalizedHead { .. } | - Self::NotCanonical { .. } => EngineTaskErrorSeverity::Skip, - Self::BlockNotFound(_) | Self::BlockNotSafe | - Self::BlockNumberMismatch { .. } | - Self::FromBlock(_) => EngineTaskErrorSeverity::Critical, + Self::ConflictsWithFinalizedHead { .. } | + Self::NotCanonical { .. } | + Self::NotYetAvailable { .. } => EngineTaskErrorSeverity::Skip, + Self::BlockNotFound(_) | Self::BlockNumberMismatch { .. } | Self::FromBlock(_) => { + EngineTaskErrorSeverity::Critical + } Self::ForkchoiceUpdateFailed(inner) => inner.severity(), Self::TransportError(_) => EngineTaskErrorSeverity::Temporary, } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/id.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/id.rs index c7c12fd6e73..13d18b4d825 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/id.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/id.rs @@ -11,8 +11,8 @@ use alloy_eips::BlockNumHash; /// `L2Finalizer`). The number is sufficient because there is only one chain to consult. /// - [`FinalizeBlockId::ByHash`] is used when an upstream supplies `(number, hash)` and the engine /// must finalize the specific block identified by hash. If the engine's canonical block at that -/// height has a different hash, the task skips the target rather than finalize the engine's block -/// in its place. +/// height has a different hash, or the engine has no block at that height yet, the task skips the +/// target rather than finalize the engine's block in its place. /// /// EL finalization is irreversible, so finalizing the wrong block is unrecoverable. Forcing every /// caller to pick a variant prevents accidental hash-loss at the boundary. @@ -21,7 +21,8 @@ pub enum FinalizeBlockId { /// Finalize whatever block the engine has at the given L2 block number. ByNumber(u64), /// Finalize the block matching the given `(number, hash)` pair, only once it is the engine's - /// canonical block at that height. + /// canonical block at that height. A skipped target leaves the state unchanged and may be + /// requested again. ByHash(BlockNumHash), } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs index a01d43c1608..841fe29fd44 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs @@ -6,6 +6,7 @@ use crate::{ task_queue::{ForkchoiceDelivery, synchronize}, }; use alloy_eips::BlockNumHash; +use alloy_transport::{RpcError, TransportErrorKind}; use async_trait::async_trait; use derive_more::Constructor; use kona_genesis::RollupConfig; @@ -16,8 +17,9 @@ use std::sync::Arc; /// [`EngineState`], and dispatches a forkchoice update to finalize the block. /// /// Finality only moves forward. A target behind the finalized head, a different block at the -/// finalized height, or a pinned hash that is not the engine's canonical block fails with an error -/// that skips the task and leaves the engine state unchanged. +/// finalized height, a pinned hash that is not the engine's canonical block, a target above the +/// safe head, or a pinned target the engine has no block for yet fails with an error that skips +/// the task and leaves the engine state unchanged. #[derive(Debug, Clone, Constructor)] pub struct FinalizeTask { /// The engine client. @@ -61,6 +63,31 @@ impl FinalizeTask { Ok(()) } + /// While the execution layer catches up, it has no block at a pinned target's height yet and + /// may answer with an error instead of null. A pinned target is skipped, carrying the answer + /// and leaving the state unchanged for the next request; failing temporarily would hold this + /// task at the head of the queue and block the imports that let the execution layer catch up. + fn lookup_failed(&self, err: RpcError) -> FinalizeTaskError { + if let FinalizeBlockId::ByHash(target) = self.block_id && + let Some(payload) = err.as_error_resp() + { + let engine_answer = format!("error {} ({})", payload.code, payload.message); + return FinalizeTaskError::NotYetAvailable { engine_answer, target }; + } + FinalizeTaskError::TransportError(err) + } + + /// A missing pinned target is not yet available, as for [`Self::lookup_failed`]. A missing + /// by-number target is below the safe head, so the engine must already have it. + fn block_missing(&self) -> FinalizeTaskError { + match self.block_id { + FinalizeBlockId::ByHash(target) => { + FinalizeTaskError::NotYetAvailable { engine_answer: "null".to_string(), target } + } + FinalizeBlockId::ByNumber(number) => FinalizeTaskError::BlockNotFound(number), + } + } + /// Whether `target` is the finalized head itself; a different block at its height is an error. fn is_already_finalized( finalized_head: &L2BlockInfo, @@ -88,8 +115,8 @@ impl FinalizeTask { .get_l2_block(requested.into()) .full() .await - .map_err(FinalizeTaskError::TransportError)? - .ok_or(FinalizeTaskError::BlockNotFound(requested))?; + .map_err(|err| self.lookup_failed(err))? + .ok_or_else(|| self.block_missing())?; let canonical = BlockNumHash { hash: block.header.hash, number: block.header.number }; debug!(target: "engine", ?canonical, "Fetched the block to finalize"); @@ -129,7 +156,8 @@ impl ExecutableEngineTask for FinalizeTask { return Ok(()); } - // Sanity check that the block that is being finalized is at least safe. + // Sanity check that the block that is being finalized is at least safe. A later request + // finalizes a target skipped here, or a newer block. if state.sync_state.safe_head().block_info.number < target_number { return Err(FinalizeTaskError::BlockNotSafe); } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs index fc2243dcb6c..a4286868b40 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs @@ -1,5 +1,6 @@ use crate::{ - EngineState, EngineSyncState, EngineSyncStateUpdate, EngineTaskError, EngineTaskErrorSeverity, + EngineState, EngineSyncState, EngineSyncStateUpdate, EngineTaskError, + EngineTaskErrorSeverity::{self, Critical, Skip, Temporary}, ExecutableEngineTask, FinalizeBlockId, FinalizeTask, FinalizeTaskError, test_utils::{ RpcMock, TestEngineStateBuilder, TestRpcBlock as RpcBlock, test_block_info, @@ -117,16 +118,56 @@ const fn is_number_mismatch(err: &FinalizeTaskError) -> bool { matches!(err, FinalizeTaskError::BlockNumberMismatch { requested: ADVANCED, .. }) } +const fn is_not_safe(err: &FinalizeTaskError) -> bool { + matches!(err, FinalizeTaskError::BlockNotSafe) +} + +const fn is_not_yet_available(err: &FinalizeTaskError) -> bool { + matches!(err, FinalizeTaskError::NotYetAvailable { .. }) +} + const fn is_block_not_found(err: &FinalizeTaskError) -> bool { matches!(err, FinalizeTaskError::BlockNotFound(ADVANCED)) } -/// Targets behind the finalized head, or a pinned hash that conflicts with it, are skipped before -/// asking the execution layer for the block: the test engine expects no request at all. +const fn is_transport_error(err: &FinalizeTaskError) -> bool { + matches!(err, FinalizeTaskError::TransportError(_)) +} + +/// A target above the finalized head, pinned to the engine's canonical block at its height. +fn pinned_target() -> FinalizeBlockId { + by_hash(&canonical(ADVANCED)) +} + +/// A target above the finalized head, identified by number only. +fn numbered_target() -> FinalizeBlockId { + by_number(&canonical(ADVANCED)) +} + +/// What the engine answers for a block it cannot serve. +#[derive(Clone, Copy, Debug)] +enum Unavailable { + ErrorResponse, + Null, +} + +fn expect_unavailable_block(l2: &RpcMock, number: u64, answer: Unavailable) { + match answer { + Unavailable::ErrorResponse => l2.expect_error_code("eth_getBlockByNumber", -32000), + Unavailable::Null => l2.expect_block_by_number(number, None), + } +} + +/// Targets behind the finalized head, a pinned hash that conflicts with it, and targets above the +/// safe head are skipped before asking the execution layer for the block: the test engine expects +/// no request at all. A reset can lower the safe head below a target that was safe when it was +/// requested, and a later request finalizes it. #[rstest] #[case::stale_by_hash(by_hash(&canonical(STALE)), is_behind)] #[case::stale_by_number(by_number(&canonical(STALE)), is_behind)] #[case::conflicting_by_hash(by_hash(&conflicting(FINALIZED)), is_conflict)] +#[case::above_safe_by_hash(by_hash(&canonical(UNSAFE)), is_not_safe)] +#[case::above_safe_by_number(by_number(&canonical(UNSAFE)), is_not_safe)] #[tokio::test] async fn finalize_task_skips_target_without_lookup( #[case] target: FinalizeBlockId, @@ -138,7 +179,7 @@ async fn finalize_task_skips_target_without_lookup( let err = finalize(target, &mut state, |_| {}).await.unwrap_err(); assert!(is_expected_error(&err), "{err:?}"); - assert_eq!(err.severity(), EngineTaskErrorSeverity::Skip); + assert_eq!(err.severity(), Skip); assert_eq!(state, original_state); } @@ -164,7 +205,7 @@ async fn finalize_task_skips_target_off_the_canonical_chain( .unwrap_err(); assert!(is_expected_error(&err), "{err:?}"); - assert_eq!(err.severity(), EngineTaskErrorSeverity::Skip); + assert_eq!(err.severity(), Skip); assert_eq!(state, original_state); } @@ -212,38 +253,94 @@ async fn finalize_task_skips_lookup_for_finalized_head() { assert_eq!(state, original_state); } -/// The engine serving no block, or a block at a height other than the requested one, stops the -/// node instead of finalizing anything. A misplaced block could otherwise bypass the guards against -/// moving finality backward. +/// The engine serving a block at a height other than the requested one stops the node instead of +/// finalizing anything. A misplaced block could otherwise bypass the guards against moving +/// finality backward. +#[rstest] +#[case::stale_block_by_number(by_number(&canonical(ADVANCED)), canonical(STALE))] +#[case::higher_block_by_number(by_number(&canonical(ADVANCED)), canonical(SAFE))] +#[case::stale_block_by_hash(by_hash(&canonical(ADVANCED)), canonical(STALE))] +#[tokio::test] +async fn finalize_task_stops_on_block_at_unrequested_height( + #[case] target: FinalizeBlockId, + #[case] served_block: RpcBlock, +) { + let mut state = engine_state(established()); + let original_state = state; + + let err = finalize(target, &mut state, |l2| { + l2.expect_block_by_number(target.number(), Some(served_block)) + }) + .await + .unwrap_err(); + + assert!(is_number_mismatch(&err), "{err:?}"); + assert_eq!(err.severity(), Critical); + assert_eq!(state, original_state); +} + +/// While the execution layer catches up, it has no block at a pinned target's height yet, and may +/// answer null or an error. Either way the target is skipped, so it neither stops the node nor +/// holds the queue until its upstream resubmits it. A by-number target is below the safe head, so +/// the engine must have it: its absence stops the node and an error response is retried. #[rstest] -#[case::missing_by_number(FinalizeBlockId::ByNumber(ADVANCED), None, is_block_not_found)] -#[case::missing_by_hash(by_hash(&canonical(ADVANCED)), None, is_block_not_found)] -#[case::stale_block_by_number( - /* target */ FinalizeBlockId::ByNumber(ADVANCED), - /* served_block */ Some(canonical(STALE)), - /* is_expected_error */ is_number_mismatch +#[case::pinned_null(pinned_target(), Unavailable::Null, is_not_yet_available, Skip)] +#[case::pinned_error_response( + pinned_target(), + Unavailable::ErrorResponse, + is_not_yet_available, + Skip )] -#[case::higher_block_by_number( - /* target */ FinalizeBlockId::ByNumber(ADVANCED), - /* served_block */ Some(canonical(SAFE)), - /* is_expected_error */ is_number_mismatch +#[case::by_number_null(numbered_target(), Unavailable::Null, is_block_not_found, Critical)] +#[case::by_number_error_response( + numbered_target(), + Unavailable::ErrorResponse, + is_transport_error, + Temporary )] -#[case::stale_block_by_hash(by_hash(&canonical(ADVANCED)), Some(canonical(STALE)), is_number_mismatch)] #[tokio::test] -async fn finalize_task_stops_on_unexpected_lookup_result( +async fn finalize_task_handles_block_the_engine_cannot_serve( #[case] target: FinalizeBlockId, - #[case] served_block: Option, + #[case] answer: Unavailable, #[case] is_expected_error: fn(&FinalizeTaskError) -> bool, + #[case] severity: EngineTaskErrorSeverity, ) { let mut state = engine_state(established()); let original_state = state; let err = - finalize(target, &mut state, |l2| l2.expect_block_by_number(target.number(), served_block)) + finalize(target, &mut state, |l2| expect_unavailable_block(l2, target.number(), answer)) .await .unwrap_err(); assert!(is_expected_error(&err), "{err:?}"); - assert_eq!(err.severity(), EngineTaskErrorSeverity::Critical); + assert_eq!(err.severity(), severity, "{err:?}"); assert_eq!(state, original_state); } + +/// A pinned target skipped while the engine catches up finalizes once its upstream resubmits it +/// and the engine has the block. +#[tokio::test] +async fn resubmitted_pinned_target_finalizes_once_engine_has_it() { + let mut state = engine_state(established()); + let served_block = canonical(ADVANCED); + let target = by_hash(&served_block); + let expected_sync_state = with_finalized_head(&state, l2_block_info(&served_block)); + + let err = finalize(target, &mut state, |l2| l2.expect_block_by_number(ADVANCED, None)) + .await + .unwrap_err(); + assert!(matches!(err, FinalizeTaskError::NotYetAvailable { .. }), "{err:?}"); + + finalize(target, &mut state, |l2| { + l2.expect_block_by_number(ADVANCED, Some(served_block)); + l2.expect_forkchoice_updated( + expected_sync_state.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected_sync_state); +} diff --git a/rust/kona/crates/node/engine/src/test_utils/mod.rs b/rust/kona/crates/node/engine/src/test_utils/mod.rs index 0a60403a0d6..89d57ac5892 100644 --- a/rust/kona/crates/node/engine/src/test_utils/mod.rs +++ b/rust/kona/crates/node/engine/src/test_utils/mod.rs @@ -8,7 +8,7 @@ mod deposits; pub use deposits::l1_info_deposit; mod rpc; -pub use rpc::{RpcMock, test_engine_client}; +pub use rpc::{RpcMock, expect_sync_start, test_engine_client}; mod engine_state; pub use engine_state::TestEngineStateBuilder; diff --git a/rust/kona/crates/node/engine/src/test_utils/rpc.rs b/rust/kona/crates/node/engine/src/test_utils/rpc.rs index 39f23f14de2..0bd0bfef0c6 100644 --- a/rust/kona/crates/node/engine/src/test_utils/rpc.rs +++ b/rust/kona/crates/node/engine/src/test_utils/rpc.rs @@ -8,6 +8,7 @@ use alloy_rpc_client::RpcClient; use alloy_rpc_types_engine::{ ForkchoiceState, ForkchoiceUpdated, PayloadStatus, PayloadStatusEnum, }; +use alloy_rpc_types_eth::{Block as EthBlock, Transaction as EthTransaction}; use alloy_transport::{ TransportError, TransportErrorKind, TransportFut, mock::{Asserter, MockResponse, MockTransport}, @@ -204,3 +205,18 @@ pub fn test_engine_client(cfg: Arc) -> (EngineClient, RpcMock, Rpc let client = EngineClient::new(l1.provider(), l2.provider(), cfg); (client, l1, l2) } + +/// Expects the sync start a reset performs: the execution layer answering `block` for its +/// finalized, safe and latest blocks, the L1 lookup of its origin, and `forkchoice` being sent. +pub fn expect_sync_start( + l1: &RpcMock, + l2: &RpcMock, + block: &TestRpcBlock, + forkchoice: ForkchoiceState, +) { + for label in ["finalized", "safe", "latest"] { + l2.expect_params("eth_getBlockByNumber", serde_json::json!([label, true]), block); + } + l1.expect("eth_getBlockByHash", EthBlock::::default()); + l2.expect_forkchoice_updated(forkchoice, PayloadStatusEnum::Valid); +} diff --git a/rust/kona/crates/node/service/src/actors/derivation/actor/core.rs b/rust/kona/crates/node/service/src/actors/derivation/actor/core.rs index 830b7ce4eaf..da702de2db3 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/actor/core.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/actor/core.rs @@ -346,17 +346,22 @@ where // Attempt to finalize the block. If successful, notify engine. if let Some(l2_block_number) = self.finalizer.try_finalize_next(*finalized_l1_block) { - // Local L1-finality: the engine's own canonical chain is the authoritative - // source at this height, so finalize by number. - self.engine_client - .send_finalized_l2_block(FinalizeBlockId::ByNumber(l2_block_number)) - .await - .map_err(|e| DerivationError::Sender(Box::new(e)))?; + self.finalize_l2_block_by_number(l2_block_number).await?; } } DerivationActorRequest::ProcessL1HeadUpdateRequest(l1_head) => { info!(target: "derivation", l1_head = ?*l1_head, "Processing l1 head update"); + // Like op-node's retry when derivation goes idle: a block that became safe after + // L1 finality last moved, or whose finalization the engine skipped because a + // reset lowered its safe head, is finalized without waiting for L1 finality to + // move again. The L1 head cadence batches the blocks confirmed since the last + // update, as op-node's finality delay does, instead of finalizing on every + // confirmation. + if let Some(l2_block_number) = self.finalizer.try_finalize_against_known_l1() { + self.finalize_l2_block_by_number(l2_block_number).await?; + } + self.derivation_state_machine.update(&DerivationStateUpdate::L1DataReceived)?; self.attempt_derivation().await?; @@ -463,6 +468,21 @@ where Ok(()) } + /// Asks the engine to finalize the L2 block at `l2_block_number`, the highest derived block + /// whose L1 inputs are finalized. + async fn finalize_l2_block_by_number( + &self, + l2_block_number: u64, + ) -> Result<(), DerivationError> { + debug!(target: "derivation", l2_block_number, "Sending L2 block to finalize to the engine"); + // Local L1-finality: the engine's own canonical chain is the authoritative source at this + // height, so finalize by number. + self.engine_client + .send_finalized_l2_block(FinalizeBlockId::ByNumber(l2_block_number)) + .await + .map_err(|e| DerivationError::Sender(Box::new(e))) + } + /// Attempts to process the next payload attributes. async fn attempt_derivation(&mut self) -> Result<(), DerivationError> { if self.derivation_state_machine.current_state() != DerivationState::Deriving { diff --git a/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs b/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs index 01eb187e119..0d1dff25286 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs @@ -664,6 +664,52 @@ async fn a_block_is_queued_for_finalization_only_once_it_is_recorded() { node.finalize_l1(DERIVED_FROM_L1).await; } +/// A block that becomes safe after its L1 inputs were finalized, as when it is derived again after +/// a reset, is finalized on the next L1 head update without waiting for L1 finality to move. A +/// later update with nothing newly safe sends nothing. +#[tokio::test] +async fn a_block_safe_after_its_l1_finalized_is_finalized_on_the_next_l1_head() { + let mut engine_client = engine_client_accepting_attributes(); + engine_client + .expect_send_finalized_l2_block() + .withf(|block| matches!(block, FinalizeBlockId::ByNumber(DERIVED_HEAD))) + .times(1) + .returning(|_| Ok(())); + let mut node = RecordingActor::new(engine_client); + node.complete_el_sync_at(RESUME_HEAD).await; + node.finalize_l1(DERIVED_FROM_L1).await; + node.confirm_safe_head(DERIVED_HEAD).await; + + for l1_head in [DERIVED_FROM_L1 + 1, DERIVED_FROM_L1 + 2] { + node.handle(DerivationActorRequest::ProcessL1HeadUpdateRequest(Box::new(l1_block( + l1_head, + )))) + .await; + } +} + +/// A reset clears the finalization queue but keeps the finalized L1 block, so a block derived +/// again after the reset is finalized on the next L1 head update. +#[tokio::test] +async fn a_block_derived_again_after_a_reset_is_finalized_on_the_next_l1_head() { + let mut engine_client = engine_client_accepting_attributes(); + engine_client + .expect_send_finalized_l2_block() + .withf(|block| matches!(block, FinalizeBlockId::ByNumber(DERIVED_HEAD))) + .times(1) + .returning(|_| Ok(())); + let mut node = RecordingActor::new(engine_client); + node.complete_el_sync_at(RESUME_HEAD).await; + node.finalize_l1(DERIVED_FROM_L1).await; + node.reset_to_resume_head().await; + node.confirm_safe_head(DERIVED_HEAD).await; + + node.handle(DerivationActorRequest::ProcessL1HeadUpdateRequest(Box::new(l1_block( + DERIVED_FROM_L1 + 1, + )))) + .await; +} + mod database; mod op_node_parity; diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs index 8f976aa256c..f9ba416517b 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs @@ -10,7 +10,12 @@ use kona_engine::{ use kona_protocol::{BlockInfo, L2BlockInfo, SyncStatus}; use rstest::rstest; use std::{collections::VecDeque, sync::Mutex}; -use tokio::sync::{mpsc, watch}; +use tokio::{ + sync::{mpsc, watch}, + time::{self, Duration, Instant}, +}; + +const POLL_INTERVAL: Duration = Duration::from_secs(4); const SAFE_ORIGIN: u64 = 1; const FINALIZED_ORIGIN: u64 = 2; @@ -248,3 +253,82 @@ async fn poll_accepts_status_after_engine_skipped_forwarded_finality( assert_eq!(harness.forwarded(), forwarded_for(&canonical_status)); } + +#[derive(Clone, Copy, Debug)] +enum DelegateReply { + CurrentL1NotCanonical, + FetchFailed, + FinalizedOriginNotCanonical, + SafeOriginNotCanonical, + Valid, +} + +impl DelegateReply { + fn sync_status(self) -> SyncStatusReply { + let mut status = delegate_status(); + match self { + Self::CurrentL1NotCanonical => { + status.current_l1.hash = non_canonical(status.current_l1.id()).hash + } + Self::FetchFailed => { + return Err(DerivationDelegateClientError::FetchFailed("unreachable".into())); + } + Self::FinalizedOriginNotCanonical => { + status.finalized_l2.l1_origin = non_canonical(status.finalized_l2.l1_origin) + } + Self::SafeOriginNotCanonical => { + status.safe_l2.l1_origin = non_canonical(status.safe_l2.l1_origin) + } + Self::Valid => {} + } + Ok(status) + } +} + +/// Polling starts only once the engine has finished syncing. +#[tokio::test(start_paused = true)] +async fn polls_wait_for_engine_sync_completion() { + let mut harness = Harness::new([]); + + assert!(time::timeout(POLL_INTERVAL * 3, harness.actor.step()).await.is_err()); + + assert_eq!(harness.forwarded(), Vec::new()); + assert_eq!(*harness.current_l1.borrow(), None); +} + +/// A valid status forwards the safe head and then the hash-pinned finalized block, and publishes +/// the delegate's `current_l1`. A failed fetch, or any L1 reference that is not canonical, skips +/// the poll without forwarding or publishing anything and without stopping the actor. +#[rstest] +#[case::valid(DelegateReply::Valid, true)] +#[case::fetch_failed(DelegateReply::FetchFailed, false)] +#[case::safe_origin_not_canonical(DelegateReply::SafeOriginNotCanonical, false)] +#[case::finalized_origin_not_canonical(DelegateReply::FinalizedOriginNotCanonical, false)] +#[case::current_l1_not_canonical(DelegateReply::CurrentL1NotCanonical, false)] +#[tokio::test(start_paused = true)] +async fn poll_forwards_only_canonical_status(#[case] reply: DelegateReply, #[case] accepted: bool) { + let mut harness = Harness::new([reply.sync_status()]); + harness.complete_engine_sync().await; + + harness.actor.step().await.unwrap(); + + assert_eq!(harness.forwarded(), forwarded_if(accepted, &delegate_status())); + assert_eq!(*harness.current_l1.borrow(), accepted.then(|| delegate_status().current_l1)); +} + +/// An unchanged status is forwarded again on every poll, one poll interval apart, so a target +/// that could not be applied on one poll is offered again on the next. +#[tokio::test(start_paused = true)] +async fn unchanged_status_is_resent_on_every_poll() { + const POLLS: u32 = 3; + let mut harness = Harness::new((0..POLLS).map(|_| DelegateReply::Valid.sync_status())); + harness.complete_engine_sync().await; + let first_poll = Instant::now(); + + for _ in 0..POLLS { + harness.actor.step().await.unwrap(); + assert_eq!(harness.forwarded(), forwarded_for(&delegate_status())); + } + + assert_eq!(Instant::now() - first_poll, POLL_INTERVAL * (POLLS - 1)); +} diff --git a/rust/kona/crates/node/service/src/actors/derivation/finalizer.rs b/rust/kona/crates/node/service/src/actors/derivation/finalizer.rs index 5fbbb46d8d7..bfffd795e16 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/finalizer.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/finalizer.rs @@ -21,6 +21,8 @@ pub(crate) struct L2Finalizer { /// block is received, the highest L2 block whose inputs are contained within the finalized /// L1 chain is finalized. awaiting_finalization: BTreeMap, + /// The number of the highest finalized L1 block received. + finalized_l1_number: Option, } impl L2Finalizer { @@ -38,7 +40,8 @@ impl L2Finalizer { .or_insert_with(|| attributes.block_number()); } - /// Clears the finalization queue. + /// Clears the finalization queue. The finalized L1 number is kept, since an L2 reset does not + /// change L1 finality. pub(crate) fn clear(&mut self) { self.awaiting_finalization.clear(); } @@ -51,19 +54,104 @@ impl L2Finalizer { &mut self, new_finalized_l1_block: BlockInfo, ) -> Option { + if self.finalized_l1_number.is_none_or(|known| known < new_finalized_l1_block.number) { + self.finalized_l1_number = Some(new_finalized_l1_block.number); + } + self.try_finalize_against_known_l1() + } + + /// Attempts to finalize L2 blocks that became safe after the highest finalized L1 block was + /// received, such as blocks derived again after a reset, without waiting for L1 finality to + /// move. + /// + /// The entries up to the finalized L1 block are drained with the returned block: a block whose + /// finalization does not go through has to be queued again to be returned again. + pub(crate) fn try_finalize_against_known_l1(&mut self) -> Option { + let finalized_l1_number = self.finalized_l1_number?; + // Find the highest safe L2 block that is contained within the finalized chain, // that the finalizer is aware of. - let highest_safe = - self.awaiting_finalization.range(..=new_finalized_l1_block.number).next_back(); - - // If the highest safe block is found, return it and drain the - // queue of all L1 blocks not contained in the finalized L1 chain. - if let Some((_, highest_safe_number)) = highest_safe { - let result = *highest_safe_number; - self.awaiting_finalization.retain(|&number, _| number > new_finalized_l1_block.number); - Some(result) - } else { - None + let (_, &highest_safe_number) = + self.awaiting_finalization.range(..=finalized_l1_number).next_back()?; + + // Drain the queue of all L1 blocks contained in the finalized L1 chain. + self.awaiting_finalization.retain(|&number, _| number > finalized_l1_number); + Some(highest_safe_number) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use kona_protocol::L2BlockInfo; + use op_alloy_rpc_types_engine::OpPayloadAttributes; + use rstest::rstest; + + const DERIVING_L1: u64 = 5; + const DERIVED_L2: u64 = 100; + + fn l1_block(number: u64) -> BlockInfo { + BlockInfo { number, ..Default::default() } + } + + /// Attributes for L2 block [`DERIVED_L2`], derived from L1 block [`DERIVING_L1`]. + fn derived_attributes() -> OpAttributesWithParent { + let mut parent = L2BlockInfo::default(); + parent.block_info.number = DERIVED_L2 - 1; + OpAttributesWithParent { + attributes: OpPayloadAttributes::default(), + derived_from: Some(l1_block(DERIVING_L1)), + is_last_in_span: true, + parent, } } + + /// A queued block is finalized once L1 finality reaches the block it was derived from. + #[rstest] + #[case::before_its_l1_block(DERIVING_L1 - 1, None)] + #[case::at_its_l1_block(DERIVING_L1, Some(DERIVED_L2))] + #[case::past_its_l1_block(DERIVING_L1 + 1, Some(DERIVED_L2))] + fn a_queued_block_finalizes_with_the_l1_block_it_was_derived_from( + #[case] finalized_l1: u64, + #[case] finalized_l2: Option, + ) { + let mut finalizer = L2Finalizer::default(); + finalizer.enqueue_for_finalization(&derived_attributes()); + + assert_eq!(finalizer.try_finalize_next(l1_block(finalized_l1)), finalized_l2); + } + + /// A finalized L1 block below the highest one received does not lower it. + #[test] + fn a_lower_finalized_l1_block_keeps_the_highest_one() { + let mut finalizer = L2Finalizer::default(); + assert_eq!(finalizer.try_finalize_next(l1_block(DERIVING_L1)), None); + finalizer.enqueue_for_finalization(&derived_attributes()); + + assert_eq!(finalizer.try_finalize_next(l1_block(DERIVING_L1 - 1)), Some(DERIVED_L2)); + } + + /// Clearing the queue keeps the finalized L1 block, so a block queued again afterwards is + /// finalized without new L1 finality. + #[test] + fn a_block_queued_after_a_clear_finalizes_against_the_kept_l1_block() { + let mut finalizer = L2Finalizer::default(); + finalizer.enqueue_for_finalization(&derived_attributes()); + assert_eq!(finalizer.try_finalize_next(l1_block(DERIVING_L1)), Some(DERIVED_L2)); + finalizer.clear(); + finalizer.enqueue_for_finalization(&derived_attributes()); + + assert_eq!(finalizer.try_finalize_against_known_l1(), Some(DERIVED_L2)); + } + + /// Nothing is finalized before any L1 finality is known, or once the queue is drained. + #[test] + fn nothing_finalizes_without_l1_finality_or_a_queued_block() { + let mut finalizer = L2Finalizer::default(); + finalizer.enqueue_for_finalization(&derived_attributes()); + assert_eq!(finalizer.try_finalize_against_known_l1(), None); + assert_eq!(finalizer.try_finalize_next(l1_block(DERIVING_L1)), Some(DERIVED_L2)); + + assert_eq!(finalizer.try_finalize_against_known_l1(), None); + } } diff --git a/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs b/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs index 3dc21d249e1..9da3810a2e7 100644 --- a/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs +++ b/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs @@ -1,13 +1,18 @@ use super::*; use crate::QueuedEngineDerivationClient; +use alloy_primitives::B256; use alloy_rpc_types_engine::{ - ExecutionPayloadV1, ForkchoiceUpdated, PayloadStatus, PayloadStatusEnum, + ExecutionPayloadV1, ForkchoiceState, PayloadStatus, PayloadStatusEnum, }; use kona_engine::{ - ConsolidateInput, EngineSyncStateUpdate, NoopBlockSink, - test_utils::{TestEngineStateBuilder, l1_info_deposit, test_block_info, test_engine_client}, + ConsolidateInput, EngineState, EngineSyncStateUpdate, FinalizeBlockId, NoopBlockSink, + ResetRequest, + test_utils::{ + RpcMock, TestEngineStateBuilder, TestRpcBlock, expect_sync_start, test_block_info, + test_engine_client, test_l2_block, test_rpc_block, + }, }; -use op_alloy_consensus::OpBlock; +use op_alloy_consensus::{OpBlock, OpTxEnvelope}; use op_alloy_rpc_types_engine::OpExecutionPayloadEnvelope; /// Exercise the real actor request/drain loop with forty consecutive unsafe blocks, without @@ -33,11 +38,7 @@ async fn healthy_unsafe_imports_keep_engine_queue_bounded() { ); let mut expected_heads = vec![initial_head]; for number in 1..=40 { - let mut block = OpBlock::default(); - block.header.number = number; - block.header.timestamp = number * 2; - block.header.parent_hash = expected_heads.last().unwrap().hash(); - block.body.transactions.push(l1_info_deposit()); + let block = test_l2_block(number, expected_heads.last().unwrap().hash(), 0); let payload = ExecutionPayloadV1::from_block_slow(&block); let imported: OpBlock = OpExecutionPayloadEnvelope::V1(payload.clone()).try_into_block().unwrap(); @@ -50,17 +51,9 @@ async fn healthy_unsafe_imports_keep_engine_queue_bounded() { serde_json::json!([payload]), PayloadStatus::from_status(PayloadStatusEnum::Valid), ); - l2.expect_params( - "engine_forkchoiceUpdatedV3", - serde_json::json!([ - alloy_rpc_types_engine::ForkchoiceState { - head_block_hash: expected_heads.last().unwrap().hash(), - safe_block_hash: initial_head.hash(), - finalized_block_hash: initial_head.hash(), - }, - null - ]), - ForkchoiceUpdated::new(PayloadStatus::from_status(PayloadStatusEnum::Valid)), + l2.expect_forkchoice_updated( + forkchoice(*expected_heads.last().unwrap(), initial_head, initial_head), + PayloadStatusEnum::Valid, ); requests .send(EngineActorRequest::ProcessUnsafeL2Block(Box::new( @@ -140,3 +133,215 @@ async fn delegated_safe_head_is_sent_on_every_poll() { l1.assert_finished(); l2.assert_finished(); } + +/// A delegate poll while the execution layer syncs toward the unsafe head: the delegate's safe +/// head is below the unsafe head but not in the execution layer yet, and neither is its finalized +/// block. The consolidate moves the safe labels and the finalize is skipped, so the actor goes on. +#[tokio::test(start_paused = true)] +async fn delegated_poll_during_el_sync_moves_safe_labels_and_skips_finalization() { + let config = Arc::new(RollupConfig::default()); + let (client, l1, l2) = test_engine_client(config.clone()); + let state = TestEngineStateBuilder::new() + .with_unsafe_head(test_block_info(20)) + .with_safe_head(test_block_info(5)) + .with_finalized_head(test_block_info(3)) + .build(); + let delegated_safe_head = test_block_info(10); + let delegated_finalized_head = test_block_info(8); + let expected_sync_state = state.sync_state.apply_update(EngineSyncStateUpdate { + local_safe_head: Some(delegated_safe_head), + safe_head: Some(delegated_safe_head), + ..Default::default() + }); + l2.expect_block_by_number(delegated_safe_head.block_info.number, None); + l2.expect_forkchoice_updated( + expected_sync_state.create_forkchoice_state(), + PayloadStatusEnum::Syncing, + ); + l2.expect_block_by_number(delegated_finalized_head.block_info.number, None); + let (state_tx, state_rx) = watch::channel(state); + let (requests, request_rx) = mpsc::channel(8); + let (derivation_tx, _derivation_rx) = mpsc::channel(8); + let mut actor = EngineActor::new( + Arc::new(client), + config, + QueuedEngineDerivationClient { derivation_actor_request_tx: derivation_tx }, + Engine::new(state, state_tx), + None, + request_rx, + Arc::new(NoopBlockSink), + ); + + // One poll: the safe head, then the finalized block pinned by hash. Each step drains the + // previous request's task and then accepts the next request. + requests + .send(EngineActorRequest::ProcessSafeL2Signal(ConsolidateInput::BlockInfo( + delegated_safe_head, + ))) + .await + .unwrap(); + requests + .send(EngineActorRequest::ProcessFinalizedL2Block(Box::new(FinalizeBlockId::ByHash( + delegated_finalized_head.block_info.id(), + )))) + .await + .unwrap(); + for _ in 0..2 { + time::timeout(Duration::from_secs(1), actor.step()).await.unwrap().unwrap(); + } + drop(requests); + assert!(matches!(actor.step().await, Err(EngineError::ChannelClosed))); + + assert!(actor.engine.is_empty()); + assert!(!actor.reset_pending); + assert_eq!(state_rx.borrow().sync_state, expected_sync_state); + l1.assert_finished(); + l2.assert_finished(); +} + +fn forkchoice(head: L2BlockInfo, safe: L2BlockInfo, finalized: L2BlockInfo) -> ForkchoiceState { + ForkchoiceState { + finalized_block_hash: finalized.hash(), + head_block_hash: head.hash(), + safe_block_hash: safe.hash(), + } +} + +/// The reset finds the EL's finalized, safe and unsafe heads all at genesis, then synchronizes +/// the EL to them. +fn expect_reset_to_genesis( + l1: &RpcMock, + l2: &RpcMock, + genesis_block: &TestRpcBlock, + genesis: L2BlockInfo, +) { + expect_sync_start(l1, l2, genesis_block, forkchoice(genesis, genesis, genesis)); +} + +fn reset_request() -> (EngineActorRequest, mpsc::Receiver>) { + let (result_tx, result_rx) = mpsc::channel(1); + (EngineActorRequest::Reset(Box::new(ResetRequest { result_tx })), result_rx) +} + +/// How the finalization target is identified: delegated derivation pins the hash, and +/// finalization from L1 finality uses the block number. +#[derive(Clone, Copy, Debug)] +enum TargetKind { + ByHash, + ByNumber, +} + +impl TargetKind { + fn finalize_request(self, target: L2BlockInfo) -> EngineActorRequest { + let block_id = match self { + Self::ByHash => FinalizeBlockId::ByHash(target.block_info.id()), + Self::ByNumber => FinalizeBlockId::ByNumber(target.block_info.number), + }; + EngineActorRequest::ProcessFinalizedL2Block(Box::new(block_id)) + } +} + +/// A reset can restore a safe head below a finalization target that was safe when it was +/// requested. The target must wait across repeated resets without stopping the node, holding the +/// queue, or triggering another reset, and must finalize once the safe head reaches it. +#[rstest::rstest] +#[case::by_hash(TargetKind::ByHash)] +#[case::by_number(TargetKind::ByNumber)] +#[tokio::test(start_paused = true)] +async fn finalization_target_survives_resets_below_it(#[case] target_kind: TargetKind) { + let genesis_block = TestRpcBlock::default(); + let mut config = RollupConfig::default(); + config.genesis.l2.hash = genesis_block.header.inner.hash_slow(); + let config = Arc::new(config); + let genesis = L2BlockInfo::from_block_and_genesis( + &genesis_block.clone().into_consensus(), + &config.genesis, + ) + .unwrap(); + let target_block = test_l2_block(15, B256::with_last_byte(14), 0); + let target = L2BlockInfo::from_block_and_genesis(&target_block, &config.genesis).unwrap(); + let restored_safe = test_block_info(20); + let unsafe_payload = OpExecutionPayloadEnvelope::V1(ExecutionPayloadV1::from_block_slow( + &test_l2_block(1, genesis.hash(), 0), + )); + let unsafe_head = L2BlockInfo::from_block_and_genesis( + &unsafe_payload.clone().try_into_block::().unwrap(), + &config.genesis, + ) + .unwrap(); + + let (client, l1, l2) = test_engine_client(config.clone()); + let (requests, request_rx) = mpsc::channel(8); + let (derivation_tx, _derivation_rx) = mpsc::channel(32); + let (state_tx, state_rx) = watch::channel(EngineState::default()); + let mut actor = EngineActor::new( + Arc::new(client), + config.clone(), + QueuedEngineDerivationClient { derivation_actor_request_tx: derivation_tx }, + Engine::new(EngineState::default(), state_tx), + None, + request_rx, + Arc::new(NoopBlockSink), + ); + + // Startup reset, with the finalization target arriving right behind it. + let (reset, mut first_reset_result) = reset_request(); + requests.send(reset).await.unwrap(); + actor.step().await.unwrap(); + expect_reset_to_genesis(&l1, &l2, &genesis_block, genesis); + requests.send(target_kind.finalize_request(target)).await.unwrap(); + actor.step().await.unwrap(); + assert!(first_reset_result.try_recv().unwrap().is_ok()); + assert_eq!(state_rx.borrow().sync_state.safe_head(), genesis); + + // The target is above the reset safe head: skipped without a lookup or forkchoice update, + // and dropped from the queue rather than retried, so the unsafe import received in the same + // step is the only task left. + requests + .send(EngineActorRequest::ProcessUnsafeL2Block(Box::new(unsafe_payload))) + .await + .unwrap(); + actor.step().await.unwrap(); + assert_eq!(actor.engine.len(), 1, "only the unsafe import may remain queued"); + assert!(!actor.reset_pending); + l2.expect("engine_newPayloadV1", PayloadStatus::from_status(PayloadStatusEnum::Valid)); + l2.expect_forkchoice_updated( + forkchoice(unsafe_head, genesis, genesis), + PayloadStatusEnum::Valid, + ); + let (reset, mut second_reset_result) = reset_request(); + requests.send(reset).await.unwrap(); + actor.step().await.unwrap(); + assert_eq!(state_rx.borrow().sync_state.unsafe_head(), unsafe_head); + + // A second reset lowers the heads again, and the same target is requested again. + expect_reset_to_genesis(&l1, &l2, &genesis_block, genesis); + requests.send(target_kind.finalize_request(target)).await.unwrap(); + actor.step().await.unwrap(); + assert!(second_reset_result.try_recv().unwrap().is_ok()); + requests.send(EngineActorRequest::ProcessSafeL2Signal(restored_safe.into())).await.unwrap(); + actor.step().await.unwrap(); + assert_eq!(actor.engine.len(), 1, "only the safe head update may remain queued"); + assert!(!actor.reset_pending); + assert_eq!(state_rx.borrow().sync_state.finalized_head(), genesis); + + // Once the safe head reaches the target, the repeated request finalizes it. + l2.expect_forkchoice_updated( + forkchoice(restored_safe, restored_safe, genesis), + PayloadStatusEnum::Valid, + ); + requests.send(target_kind.finalize_request(target)).await.unwrap(); + actor.step().await.unwrap(); + assert_eq!(state_rx.borrow().sync_state.safe_head(), restored_safe); + l2.expect_block_by_number(15, Some(test_rpc_block(&target_block))); + l2.expect_forkchoice_updated( + forkchoice(restored_safe, restored_safe, target), + PayloadStatusEnum::Valid, + ); + // With nothing left to do, the actor waits for requests instead of retrying. + assert!(time::timeout(Duration::from_secs(60), actor.step()).await.is_err()); + assert_eq!(state_rx.borrow().sync_state.finalized_head(), target); + assert!(actor.engine.is_empty()); + l1.assert_finished(); + l2.assert_finished(); +} From b73f56aa627ec55b7d4a3d84ebe09f34bddd208a Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 12 Nov 2026 14:30:53 +0100 Subject: [PATCH 04/14] kona-engine: replace ForkchoiceDelivery with two synchronize functions `synchronize` skips a forkchoice update that leaves the labels unchanged; `synchronize_unconditionally` always sends it. Both share a private `send_forkchoice`. The reset and the delegated paths of the consolidate task call `synchronize_unconditionally`; the insert, finalize and derived consolidate paths call `synchronize`. The consolidate task's thin wrappers `reconcile_to_safe_head`, `follow_safe_head_during_el_sync` and `apply_delegated_safe_head` are inlined at their call sites, keeping their logs and comments. Behavior is unchanged. Co-Authored-By: Claude Opus 5.5 --- .../crates/node/engine/src/task_queue/core.rs | 5 +- .../src/task_queue/tasks/consolidate/task.rs | 198 ++++++++---------- .../src/task_queue/tasks/finalize/task.rs | 4 +- .../src/task_queue/tasks/insert/task.rs | 4 +- .../node/engine/src/task_queue/tasks/mod.rs | 2 +- .../task_queue/tasks/synchronize/helper.rs | 70 ++++--- .../src/task_queue/tasks/synchronize/mod.rs | 2 +- .../src/task_queue/tasks/synchronize/tests.rs | 39 +++- 8 files changed, 159 insertions(+), 165 deletions(-) diff --git a/rust/kona/crates/node/engine/src/task_queue/core.rs b/rust/kona/crates/node/engine/src/task_queue/core.rs index acc3fa27d40..2167623aaa2 100644 --- a/rust/kona/crates/node/engine/src/task_queue/core.rs +++ b/rust/kona/crates/node/engine/src/task_queue/core.rs @@ -5,7 +5,7 @@ use crate::{ EngineClient, EngineState, EngineSyncStateUpdate, EngineTask, EngineTaskError, EngineTaskErrorSeverity, Metrics, SyncStartError, SynchronizeTaskError, find_starting_forkchoice, - task_queue::{EngineTaskErrors, ForkchoiceDelivery, synchronize}, + task_queue::{EngineTaskErrors, synchronize_unconditionally}, }; use kona_genesis::RollupConfig; use kona_protocol::L2BlockInfo; @@ -118,7 +118,7 @@ impl Engine { // One attempt. The actor retains the reset request and retries transient failures. The // execution layer may have lost or changed its forkchoice, so the reset sends one even when // the recovered labels equal the current ones. - synchronize( + synchronize_unconditionally( client.as_ref(), &mut self.state, EngineSyncStateUpdate { @@ -127,7 +127,6 @@ impl Engine { safe_head: Some(start.safe), finalized_head: Some(start.finalized), }, - ForkchoiceDelivery::Always, ) .await?; diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs index 9531eac4aa3..d3183cdafa5 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs @@ -3,7 +3,7 @@ use crate::{ ConsolidateTaskError, EngineClient, EngineState, ExecutableEngineTask, ImportedBlockSink, state::EngineSyncStateUpdate, - task_queue::{ForkchoiceDelivery, build_and_import, synchronize}, + task_queue::{build_and_import, synchronize, synchronize_unconditionally}, }; use alloy_rpc_types_eth::Block; use async_trait::async_trait; @@ -56,16 +56,6 @@ impl ConsolidateInput { const fn is_intermediate_span_block(&self) -> bool { matches!(self, Self::Attributes(attributes) if !attributes.is_last_in_span) } - - /// Derived attributes move the safe head on every consolidation, so an unchanged forkchoice was - /// already sent. A delegated safe head can repeat a forkchoice the execution layer answered - /// `SYNCING` or lost, so it is always sent. - const fn forkchoice_delivery(&self) -> ForkchoiceDelivery { - match self { - Self::Attributes(_) => ForkchoiceDelivery::WhenChanged, - Self::BlockInfo(_) => ForkchoiceDelivery::Always, - } - } } /// The [`ConsolidateTask`] attempts to consolidate the engine state @@ -110,96 +100,6 @@ impl ConsolidateTask { Ok(()) } - /// This provides symmetric fallback behavior with `build_and_import`. - async fn reconcile_to_safe_head( - &self, - state: &mut EngineState, - safe_l2: &L2BlockInfo, - ) -> Result<(), ConsolidateTaskError> { - warn!( - target: "engine", - safe_l2 = %safe_l2, - "Apply safe head" - ); - - // We intentionally set the unsafe head to safe_l2 to ensure the engine observes a - // self-consistent head state. This is required to correctly handle reorgs (where unsafe - // may be ahead on a non-canonical fork) and to trigger EL sync when the local unsafe head - // lags behind the safe head. - self.apply_delegated_safe_head( - state, - safe_l2, - EngineSyncStateUpdate { - local_safe_head: Some(*safe_l2), - safe_head: Some(*safe_l2), - unsafe_head: Some(*safe_l2), - ..Default::default() - }, - "Updated safe head via follow safe", - ) - .await - } - - /// Moves the safe labels to a delegated safe head that the execution layer does not have yet, - /// leaving the unsafe head in place. - /// - /// A block missing below the unsafe head means the execution layer is still syncing toward the - /// unsafe head. Resetting would restart that sync from the execution layer's old labels, and - /// moving the unsafe head back would retarget it. Like op-node's follow-source mode, only the - /// safe labels follow the delegate, and the forkchoice update carries them to the execution - /// layer. - async fn follow_safe_head_during_el_sync( - &self, - state: &mut EngineState, - safe_l2: &L2BlockInfo, - ) -> Result<(), ConsolidateTaskError> { - warn!( - target: "engine", - safe_l2 = %safe_l2, - unsafe_number = state.sync_state.unsafe_head().block_info.number, - "Execution layer does not have the delegated safe head yet; keeping its sync target" - ); - - self.apply_delegated_safe_head( - state, - safe_l2, - EngineSyncStateUpdate { - local_safe_head: Some(*safe_l2), - safe_head: Some(*safe_l2), - ..Default::default() - }, - "Updated safe head while the execution layer syncs", - ) - .await - } - - /// Applies `update`, whose labels follow the delegated `safe_l2`, and always sends its - /// forkchoice update: the execution layer may have answered `SYNCING` to the same labels - /// before, or lost them in a restart. - async fn apply_delegated_safe_head( - &self, - state: &mut EngineState, - safe_l2: &L2BlockInfo, - update: EngineSyncStateUpdate, - outcome: &'static str, - ) -> Result<(), ConsolidateTaskError> { - synchronize(self.client.as_ref(), state, update, ForkchoiceDelivery::Always) - .await - .map_err(|e| { - warn!(target: "engine", ?e, safe_l2 = %safe_l2, "Applying the delegated safe head failed"); - e - })?; - - info!( - target: "engine", - hash = %safe_l2.block_info.hash, - number = safe_l2.block_info.number, - "{outcome}" - ); - - Ok(()) - } - /// Handles the fallback case when the block doesn't match the input or does not exist. async fn reconcile_unsafe_to_safe( &self, @@ -210,7 +110,37 @@ impl ConsolidateTask { self.build_and_import_attributes(state, attributes).await } ConsolidateInput::BlockInfo(safe_l2) => { - self.reconcile_to_safe_head(state, safe_l2).await + warn!(target: "engine", safe_l2 = %safe_l2, "Apply safe head"); + + // We intentionally set the unsafe head to safe_l2 to ensure the engine observes a + // self-consistent head state. This is required to correctly handle reorgs (where + // unsafe may be ahead on a non-canonical fork) and to trigger EL sync when the + // local unsafe head lags behind the safe head. The forkchoice update is always + // sent: the execution layer may have answered `SYNCING` to the same labels before, + // or lost them in a restart. + synchronize_unconditionally( + self.client.as_ref(), + state, + EngineSyncStateUpdate { + local_safe_head: Some(*safe_l2), + safe_head: Some(*safe_l2), + unsafe_head: Some(*safe_l2), + ..Default::default() + }, + ) + .await + .map_err(|e| { + warn!(target: "engine", ?e, safe_l2 = %safe_l2, "Applying the delegated safe head failed"); + e + })?; + + info!( + target: "engine", + hash = %safe_l2.block_info.hash, + number = safe_l2.block_info.number, + "Updated safe head via follow safe" + ); + Ok(()) } } } @@ -230,7 +160,39 @@ impl ConsolidateTask { Ok(Some(block)) => block, Ok(None) => { if let ConsolidateInput::BlockInfo(safe_l2) = &self.input { - return self.follow_safe_head_during_el_sync(state, safe_l2).await; + // A block missing below the unsafe head means the execution layer is still + // syncing toward the unsafe head. Resetting would restart that sync from the + // execution layer's old labels, and moving the unsafe head back would retarget + // it. Like op-node's follow-source mode, only the safe labels follow the + // delegate, and the forkchoice update, sent even when unchanged, carries them + // to the execution layer. + warn!( + target: "engine", + safe_l2 = %safe_l2, + unsafe_number = state.sync_state.unsafe_head().block_info.number, + "Execution layer does not have the delegated safe head yet; keeping its sync target" + ); + synchronize_unconditionally( + self.client.as_ref(), + state, + EngineSyncStateUpdate { + local_safe_head: Some(*safe_l2), + safe_head: Some(*safe_l2), + ..Default::default() + }, + ) + .await + .map_err(|e| { + warn!(target: "engine", ?e, safe_l2 = %safe_l2, "Applying the delegated safe head failed"); + e + })?; + info!( + target: "engine", + hash = %safe_l2.block_info.hash, + number = safe_l2.block_info.number, + "Updated safe head while the execution layer syncs" + ); + return Ok(()); } warn!(target: "engine", "Received `None` block for {}", block_num); return Err(ConsolidateTaskError::MissingUnsafeL2Block(block_num)); @@ -278,18 +240,24 @@ impl ConsolidateTask { // The next attributes built are this block's child, and ask for its config. self.block_sink.block_imported(consensus_block, block_info); - synchronize( - self.client.as_ref(), - state, - EngineSyncStateUpdate { - safe_head: Some(block_info), - local_safe_head: Some(block_info), - ..Default::default() - }, - self.input.forkchoice_delivery(), - ) - .await - .map_err(|e| { + let update = EngineSyncStateUpdate { + safe_head: Some(block_info), + local_safe_head: Some(block_info), + ..Default::default() + }; + let synchronized = match &self.input { + // Derived attributes move the safe head on every consolidation, so an + // unchanged forkchoice was already sent. + ConsolidateInput::Attributes(_) => { + synchronize(self.client.as_ref(), state, update).await + } + // A delegated safe head can repeat a forkchoice the execution layer + // answered `SYNCING` or lost, so it is always sent. + ConsolidateInput::BlockInfo(_) => { + synchronize_unconditionally(self.client.as_ref(), state, update).await + } + }; + synchronized.map_err(|e| { warn!(target: "engine", ?e, "Consolidation failed"); e })?; diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs index d00bb74b624..3461d7d860f 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs @@ -2,8 +2,7 @@ use crate::{ EngineClient, EngineState, ExecutableEngineTask, FinalizeBlockId, FinalizeTaskError, - state::EngineSyncStateUpdate, - task_queue::{ForkchoiceDelivery, synchronize}, + state::EngineSyncStateUpdate, task_queue::synchronize, }; use alloy_eips::BlockId; use async_trait::async_trait; @@ -72,7 +71,6 @@ impl ExecutableEngineTask for FinalizeTask { self.client.as_ref(), state, EngineSyncStateUpdate { finalized_head: Some(block_info), ..Default::default() }, - ForkchoiceDelivery::WhenChanged, ) .await?; diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/insert/task.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/insert/task.rs index 10110e3b622..54d2d380709 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/insert/task.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/insert/task.rs @@ -2,8 +2,7 @@ use crate::{ EngineClient, EngineState, ExecutableEngineTask, ImportedBlockSink, InsertTaskError, - state::EngineSyncStateUpdate, - task_queue::{ForkchoiceDelivery, synchronize}, + state::EngineSyncStateUpdate, task_queue::synchronize, }; use alloy_rpc_types_engine::{ExecutionPayloadInputV2, PayloadStatusEnum}; use async_trait::async_trait; @@ -126,7 +125,6 @@ pub(in crate::task_queue) async fn insert_payload( safe_head: is_payload_safe.then_some(new_unsafe_ref), ..Default::default() }, - ForkchoiceDelivery::WhenChanged, ) .await?; diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/mod.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/mod.rs index c5aa7a8ad47..9eddaa76d00 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/mod.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/mod.rs @@ -7,7 +7,7 @@ pub use task::{ mod synchronize; pub use synchronize::SynchronizeTaskError; -pub(super) use synchronize::{ForkchoiceDelivery, synchronize}; +pub(super) use synchronize::{synchronize, synchronize_unconditionally}; mod insert; pub(super) use insert::insert_payload; diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs index 219416bd6f1..a9bc15146ff 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs @@ -1,61 +1,73 @@ //! The `engine_forkchoiceUpdated` call, with no attributes, that the engine tasks share. -use crate::{EngineClient, EngineState, SynchronizeTaskError, state::EngineSyncStateUpdate}; +use crate::{ + EngineClient, EngineState, EngineSyncState, SynchronizeTaskError, state::EngineSyncStateUpdate, +}; use alloy_rpc_types_engine::{INVALID_FORK_CHOICE_STATE_ERROR, PayloadStatusEnum}; use op_alloy_provider::ext::engine::OpEngineApi; -/// Whether [`synchronize`] may skip a forkchoice update that leaves the consensus labels unchanged. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(in crate::task_queue) enum ForkchoiceDelivery { - /// Send the update even when it leaves the consensus labels unchanged. - /// - /// Unchanged labels do not prove that the execution layer holds their forkchoice: a `SYNCING` - /// answer moves the labels too, and an execution layer that restarts can lose its forkchoice. - Always, - /// Skip the update when it leaves the consensus labels unchanged. - WhenChanged, -} - /// Applies `state_update` to the engine's sync state and synchronizes the execution layer's -/// forkchoice with it, using an `engine_forkchoiceUpdated` call without payload attributes. +/// forkchoice with it, using an `engine_forkchoiceUpdated` call without payload attributes. An +/// update that leaves the labels unchanged is skipped. /// -/// Engine tasks call this whenever they move the unsafe, safe, or finalized head, and the engine -/// calls it during a reset to establish the initial forkchoice state. Forkchoice updates that -/// start a block build are made by `start_build` instead. +/// Engine tasks call this whenever they move the unsafe, safe, or finalized head. Forkchoice +/// updates that start a block build are made by `start_build` instead. #[tracing::instrument( name = "synchronize", target = "engine", level = "info", skip_all, - fields(?delivery, ?state_update, forkchoice = tracing::field::Empty) + fields(?state_update, forkchoice = tracing::field::Empty) )] pub(in crate::task_queue) async fn synchronize( client: &EngineClient, state: &mut EngineState, state_update: EngineSyncStateUpdate, - delivery: ForkchoiceDelivery, ) -> Result<(), SynchronizeTaskError> { - // Apply the sync state update to the engine state. let new_sync_state = state.sync_state.apply_update(state_update); - // Check if a forkchoice update is not needed, return early. - // A forkchoice update is not needed if... - // 1. The caller allows skipping an unchanged forkchoice, - // 2. The engine state is not default (initial forkchoice state has been emitted), and - // 3. The new sync state is the same as the current sync state (no changes to the sync state). + // A forkchoice update is not needed once the initial forkchoice state has been emitted and the + // update leaves the labels unchanged. // // NOTE: // We shouldn't retry the synchronization here. Since the `sync_state` is only updated inside // `synchronize` (except inside the ConsolidateTask, when the block is not the last in the // batch) - the engine would get stuck retrying the synchronization. - if delivery == ForkchoiceDelivery::WhenChanged && - state.sync_state != Default::default() && - state.sync_state == new_sync_state - { + if state.sync_state != Default::default() && state.sync_state == new_sync_state { debug!(target: "engine", ?new_sync_state, "No forkchoice update needed"); return Ok(()); } + send_forkchoice(client, state, new_sync_state).await +} + +/// Like [`synchronize`], but sends the forkchoice update even when it leaves the labels unchanged. +/// +/// Unchanged labels do not prove that the execution layer holds their forkchoice: a `SYNCING` +/// answer moves the labels too, and an execution layer that restarts can lose its forkchoice. +#[tracing::instrument( + name = "synchronize_unconditionally", + target = "engine", + level = "info", + skip_all, + fields(?state_update, forkchoice = tracing::field::Empty) +)] +pub(in crate::task_queue) async fn synchronize_unconditionally( + client: &EngineClient, + state: &mut EngineState, + state_update: EngineSyncStateUpdate, +) -> Result<(), SynchronizeTaskError> { + let new_sync_state = state.sync_state.apply_update(state_update); + send_forkchoice(client, state, new_sync_state).await +} + +/// Sends the forkchoice of `new_sync_state` and, once the execution layer accepts it, makes it the +/// engine's sync state. +async fn send_forkchoice( + client: &EngineClient, + state: &mut EngineState, + new_sync_state: EngineSyncState, +) -> Result<(), SynchronizeTaskError> { // Check if the head is behind the finalized head. if new_sync_state.unsafe_head().block_info.number < new_sync_state.finalized_head().block_info.number diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/mod.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/mod.rs index e73923b3228..9de83da3a5c 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/mod.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/mod.rs @@ -1,7 +1,7 @@ //! The forkchoice engine update shared by the engine tasks, and its error type. mod helper; -pub(in crate::task_queue) use helper::{ForkchoiceDelivery, synchronize}; +pub(in crate::task_queue) use helper::{synchronize, synchronize_unconditionally}; mod error; pub use error::SynchronizeTaskError; diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/tests.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/tests.rs index 1ebc3a56248..64909cb394f 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/tests.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/tests.rs @@ -1,4 +1,4 @@ -use super::{ForkchoiceDelivery, synchronize}; +use super::{synchronize, synchronize_unconditionally}; use crate::{ EngineState, EngineSyncState, EngineSyncStateUpdate, EngineTaskError, EngineTaskErrorSeverity::{self, Reset, Temporary}, @@ -72,16 +72,13 @@ fn expect_reply(l2: &RpcMock, forkchoice: ForkchoiceState, reply: Reply) { } } -/// An update that leaves the labels unchanged is skipped only when the caller allows it. +/// `synchronize` skips an update that leaves the labels unchanged. #[rstest] -#[case::changed_labels_are_sent(true, ForkchoiceDelivery::WhenChanged, 1)] -#[case::unchanged_labels_are_skipped(false, ForkchoiceDelivery::WhenChanged, 0)] -#[case::changed_labels_are_sent_always(true, ForkchoiceDelivery::Always, 1)] -#[case::unchanged_labels_are_sent_always(false, ForkchoiceDelivery::Always, 1)] +#[case::changed_labels_are_sent(true, 1)] +#[case::unchanged_labels_are_skipped(false, 0)] #[tokio::test] -async fn synchronize_skips_unchanged_labels_only_when_allowed( +async fn synchronize_skips_unchanged_labels( #[case] moves_safe_head: bool, - #[case] delivery: ForkchoiceDelivery, #[case] forkchoice_updates: usize, ) { let mut state = engine_state(); @@ -95,7 +92,29 @@ async fn synchronize_skips_unchanged_labels_only_when_allowed( ); } - synchronize(&client, &mut state, update, delivery).await.unwrap(); + synchronize(&client, &mut state, update).await.unwrap(); + + assert_eq!(state.sync_state, expected_sync_state); + l1.assert_finished(); + l2.assert_finished(); +} + +/// `synchronize_unconditionally` sends the update even when it leaves the labels unchanged. +#[rstest] +#[case::changed_labels_are_sent(true)] +#[case::unchanged_labels_are_sent(false)] +#[tokio::test] +async fn synchronize_unconditionally_sends_unchanged_labels(#[case] moves_safe_head: bool) { + let mut state = engine_state(); + let update = state_update(moves_safe_head); + let expected_sync_state = state.sync_state.apply_update(update); + let (client, l1, l2) = test_engine_client(Arc::new(RollupConfig::default())); + l2.expect_forkchoice_updated( + expected_sync_state.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + + synchronize_unconditionally(&client, &mut state, update).await.unwrap(); assert_eq!(state.sync_state, expected_sync_state); l1.assert_finished(); @@ -124,7 +143,7 @@ async fn synchronize_applies_labels_only_when_accepted( let (client, l1, l2) = test_engine_client(Arc::new(RollupConfig::default())); expect_reply(&l2, moved_sync_state.create_forkchoice_state(), reply); - let result = synchronize(&client, &mut state, update, ForkchoiceDelivery::WhenChanged).await; + let result = synchronize(&client, &mut state, update).await; assert_eq!(result.as_ref().err().map(EngineTaskError::severity), error_severity, "{result:?}"); assert_eq!(state.sync_state, expected_sync_state); From e9305c437df6547fbe172cb14d7acafb0082affd Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 12 Nov 2026 14:38:20 +0100 Subject: [PATCH 05/14] kona-engine: inline the finalize task's finality checks `ensure_not_behind_finalized_head` and `ensure_no_conflict_with_finalized_head` each had one caller, so their checks move inline. A new test pins a target above the safe head: it fails with `BlockNotSafe` before any lookup. Co-Authored-By: Claude Opus 5.5 --- .../src/task_queue/tasks/finalize/task.rs | 48 +++++-------------- .../task_queue/tasks/finalize/task_test.rs | 14 ++++++ 2 files changed, 27 insertions(+), 35 deletions(-) diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs index 2099ccd7403..dac3404750a 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs @@ -28,48 +28,21 @@ pub struct FinalizeTask { } impl FinalizeTask { - /// Rejects a target that would move finality backward. - const fn ensure_not_behind_finalized_head( - finalized_head: &L2BlockInfo, - target_number: u64, - ) -> Result<(), FinalizeTaskError> { - if target_number < finalized_head.block_info.number { - return Err(FinalizeTaskError::BehindFinalizedHead { - finalized: finalized_head.block_info.number, - target: target_number, - }); - } - Ok(()) - } - - /// Rejects a different block at the finalized head's height. - /// - /// The default head means finality has not been established yet, so any target, including the - /// configured genesis, may initialize it. - fn ensure_no_conflict_with_finalized_head( - finalized_head: &L2BlockInfo, - target: BlockNumHash, - ) -> Result<(), FinalizeTaskError> { - let finalized = finalized_head.block_info.id(); - if *finalized_head != L2BlockInfo::default() && - target.number == finalized.number && - target.hash != finalized.hash - { - return Err(FinalizeTaskError::ConflictsWithFinalizedHead { finalized, target }); - } - Ok(()) - } - /// Whether `target` is the finalized head itself; a different block at its height is an error. fn is_already_finalized( finalized_head: &L2BlockInfo, target: BlockNumHash, ) -> Result { - if target == finalized_head.block_info.id() { + let finalized = finalized_head.block_info.id(); + if target == finalized { debug!(target: "engine", ?target, "Target is already finalized"); return Ok(true); } - Self::ensure_no_conflict_with_finalized_head(finalized_head, target)?; + // The default head means finality has not been established yet, so any target, including + // the configured genesis, may initialize it. + if *finalized_head != L2BlockInfo::default() && target.number == finalized.number { + return Err(FinalizeTaskError::ConflictsWithFinalizedHead { finalized, target }); + } Ok(false) } @@ -121,7 +94,12 @@ impl ExecutableEngineTask for FinalizeTask { // A lagging source can report an older finalized block, or one from another chain. A // pinned hash is enough to detect either without asking the execution layer. - Self::ensure_not_behind_finalized_head(&finalized_head, target_number)?; + if target_number < finalized_head.block_info.number { + return Err(FinalizeTaskError::BehindFinalizedHead { + finalized: finalized_head.block_info.number, + target: target_number, + }); + } if let FinalizeBlockId::ByHash(target) = self.block_id && Self::is_already_finalized(&finalized_head, target)? { diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs index fc2243dcb6c..fb273f99831 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs @@ -142,6 +142,20 @@ async fn finalize_task_skips_target_without_lookup( assert_eq!(state, original_state); } +/// A target above the safe head is refused before asking the execution layer for the block: the +/// test engine expects no request at all. +#[tokio::test] +async fn finalize_task_refuses_target_above_safe_head_without_lookup() { + let mut state = engine_state(established()); + let original_state = state; + + let err = finalize(by_hash(&canonical(SAFE + 5)), &mut state, |_| {}).await.unwrap_err(); + + assert!(matches!(err, FinalizeTaskError::BlockNotSafe), "{err:?}"); + assert_eq!(err.severity(), EngineTaskErrorSeverity::Critical); + assert_eq!(state, original_state); +} + /// A target whose block differs from the engine's canonical block is skipped. A pinned hash is /// never replaced by the canonical block at its height, even above the finalized head, where a /// delegate on another fork names a block the engine does not consider canonical. From d92d1fcb5e390b99f4b4456541af403af0bb6a5b Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 12 Nov 2026 14:38:21 +0100 Subject: [PATCH 06/14] kona-node-service: report sync completion with the safe head only `ProcessEngineSyncCompletionRequest` and `notify_sync_completed` carry the local-safe head again instead of the whole `EngineSyncState`. Both consumers only read the local-safe head; the delegate actor takes its finalized baseline from the engine's published state. Co-Authored-By: Claude Opus 5.5 --- .../src/actors/derivation/actor/core.rs | 3 +- .../src/actors/derivation/actor/tests.rs | 29 ++++++++++--------- .../actors/derivation/actor/tests/database.rs | 2 +- .../derivation/actor/tests/op_node_parity.rs | 2 +- .../src/actors/derivation/delegated/actor.rs | 4 +-- .../src/actors/derivation/delegated/tests.rs | 4 ++- .../service/src/actors/derivation/request.rs | 7 ++--- .../node/service/src/actors/engine/actor.rs | 2 +- .../node/service/src/actors/engine/client.rs | 16 +++------- .../node/service/src/actors/engine/tests.rs | 2 +- 10 files changed, 32 insertions(+), 39 deletions(-) diff --git a/rust/kona/crates/node/service/src/actors/derivation/actor/core.rs b/rust/kona/crates/node/service/src/actors/derivation/actor/core.rs index 830b7ce4eaf..e3a5ea1ba4d 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/actor/core.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/actor/core.rs @@ -369,9 +369,8 @@ where self.attempt_derivation().await?; } - DerivationActorRequest::ProcessEngineSyncCompletionRequest(sync_state) => { + DerivationActorRequest::ProcessEngineSyncCompletionRequest(safe_head) => { info!(target: "derivation", "Engine finished syncing, starting derivation."); - let safe_head = Box::new(sync_state.local_safe_head()); self.derivation_state_machine .update(&DerivationStateUpdate::ELSyncCompleted(safe_head.clone()))?; self.truncate_safe_db(*safe_head, "EL sync completion")?; diff --git a/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs b/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs index 01eb187e119..5bd4f9cfc00 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs @@ -10,7 +10,7 @@ use kona_derive::{ ActivationSignal, Pipeline, PipelineError, PipelineErrorKind, PipelineResult, ResetError, Signal, SignalReceiver, StepResult, }; -use kona_engine::{EngineSyncState, EngineSyncStateUpdate, FinalizeBlockId}; +use kona_engine::FinalizeBlockId; use kona_genesis::{HardForkConfig, RollupConfig, SystemConfig}; use kona_protocol::{BlockInfo, L2BlockInfo, OpAttributesWithParent}; use kona_safedb::{SafeDatabase, SafeDb, SafeDbError, SafeHeadRecord}; @@ -41,7 +41,12 @@ async fn reset_adopts_safe_head_and_resumes_atomically(#[case] previous_head: u6 l1_block(0).id(), watch::channel(None).0, ); - request_tx.send(engine_sync_completed_at(l2_block(previous_head))).await.unwrap(); + request_tx + .send(DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(l2_block( + previous_head, + )))) + .await + .unwrap(); actor.step().await.unwrap(); request_tx @@ -262,7 +267,10 @@ async fn test_origin_is_published(#[case] step_result: StepResult) { actor.step().await.unwrap(); assert_eq!(derivation_origin_rx.borrow().map(|origin| origin.id()), Some(safe_head.l1_origin)); - request_tx.send(engine_sync_completed_at(safe_head)).await.unwrap(); + request_tx + .send(DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(safe_head))) + .await + .unwrap(); actor.step().await.unwrap(); assert_eq!(derivation_origin_rx.borrow().unwrap().number, 11); @@ -457,16 +465,6 @@ fn l1_block(number: u64) -> BlockInfo { BlockInfo { hash: B256::with_last_byte(number as u8), number, ..Default::default() } } -/// The engine reporting that its sync completed with `safe_head` as its safe heads. -fn engine_sync_completed_at(safe_head: L2BlockInfo) -> DerivationActorRequest { - let sync_state = EngineSyncState::default().apply_update(EngineSyncStateUpdate { - local_safe_head: Some(safe_head), - safe_head: Some(safe_head), - ..Default::default() - }); - DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(sync_state)) -} - fn l2_block(number: u64) -> L2BlockInfo { L2BlockInfo { block_info: BlockInfo { @@ -537,7 +535,10 @@ impl RecordingActor { } async fn complete_el_sync_at(&mut self, head: u64) { - self.handle(engine_sync_completed_at(l2_block(head))).await; + self.handle(DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new( + l2_block(head), + ))) + .await; } async fn confirm_safe_head(&mut self, head: u64) { diff --git a/rust/kona/crates/node/service/src/actors/derivation/actor/tests/database.rs b/rust/kona/crates/node/service/src/actors/derivation/actor/tests/database.rs index eeee778a78a..699bc1e6933 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/actor/tests/database.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/actor/tests/database.rs @@ -135,7 +135,7 @@ fn engine_reset_request() -> DerivationActorRequest { } fn sync_completion_request() -> DerivationActorRequest { - engine_sync_completed_at(l2_block(RESUME_HEAD)) + DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(l2_block(RESUME_HEAD))) } #[rstest] diff --git a/rust/kona/crates/node/service/src/actors/derivation/actor/tests/op_node_parity.rs b/rust/kona/crates/node/service/src/actors/derivation/actor/tests/op_node_parity.rs index 0df6cef81c1..104544f2522 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/actor/tests/op_node_parity.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/actor/tests/op_node_parity.rs @@ -16,7 +16,7 @@ fn l1_head_update(number: u64) -> DerivationActorRequest { } fn el_sync_completed(head: u64) -> DerivationActorRequest { - engine_sync_completed_at(l2_block(head)) + DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(l2_block(head))) } /// A reset whose pipeline signal fails once with a temporary provider error neither stops the diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs index 7001d2f71d6..c7ec925ecd2 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs @@ -269,9 +269,9 @@ where debug!(target: "derivation", safe_head = ?*safe_head, "Received safe head from engine."); self.engine_l2_safe_head = *safe_head; } - DerivationActorRequest::ProcessEngineSyncCompletionRequest(sync_state) => { + DerivationActorRequest::ProcessEngineSyncCompletionRequest(safe_head) => { info!(target: "derivation", "Engine finished syncing, starting derivation."); - self.engine_l2_safe_head = sync_state.local_safe_head(); + self.engine_l2_safe_head = *safe_head; self.has_engine_sync_completed = true; } DerivationActorRequest::ProcessEngineSignalRequest(signal) => match *signal { diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs index 8f976aa256c..0eda07f0beb 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs @@ -70,7 +70,9 @@ impl Harness { async fn complete_engine_sync_with(&mut self, sync_state: EngineSyncState) { self.publish_engine_sync_state(sync_state); self.derivation_requests - .send(DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(sync_state))) + .send(DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new( + sync_state.local_safe_head(), + ))) .await .unwrap(); self.actor.step().await.unwrap(); diff --git a/rust/kona/crates/node/service/src/actors/derivation/request.rs b/rust/kona/crates/node/service/src/actors/derivation/request.rs index caa1e2b9ad2..9482eef5582 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/request.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/request.rs @@ -1,5 +1,4 @@ use kona_derive::Signal; -use kona_engine::EngineSyncState; use kona_protocol::{BlockInfo, L2BlockInfo}; use thiserror::Error; @@ -22,9 +21,9 @@ pub enum DerivationClientError { /// Inbound requests that the [`crate::DerivationActor`] can process. #[derive(Debug)] pub enum DerivationActorRequest { - /// Request to process the fact that Engine sync has completed, along with the engine's sync - /// state at that point. - ProcessEngineSyncCompletionRequest(Box), + /// Request to process the fact that Engine sync has completed, along with the current safe + /// head. + ProcessEngineSyncCompletionRequest(Box), /// Request to process the provided L2 engine safe head update. ProcessEngineSafeHeadUpdateRequest(Box), /// A request containing a [`Signal`] to the derivation pipeline. diff --git a/rust/kona/crates/node/service/src/actors/engine/actor.rs b/rust/kona/crates/node/service/src/actors/engine/actor.rs index 87619ba2756..f09047db8a2 100644 --- a/rust/kona/crates/node/service/src/actors/engine/actor.rs +++ b/rust/kona/crates/node/service/src/actors/engine/actor.rs @@ -251,7 +251,7 @@ where self.el_sync_complete = true; self.derivation_client - .notify_sync_completed(self.engine.state().sync_state) + .notify_sync_completed(self.engine.state().sync_state.local_safe_head()) .await .map(|_| Ok(())) .map_err(|e| { diff --git a/rust/kona/crates/node/service/src/actors/engine/client.rs b/rust/kona/crates/node/service/src/actors/engine/client.rs index 518e7b957d0..25322a5a7f2 100644 --- a/rust/kona/crates/node/service/src/actors/engine/client.rs +++ b/rust/kona/crates/node/service/src/actors/engine/client.rs @@ -2,7 +2,6 @@ use crate::{DerivationActorRequest, DerivationClientError, DerivationClientResul use async_trait::async_trait; use derive_more::Constructor; use kona_derive::Signal; -use kona_engine::EngineSyncState; use kona_protocol::L2BlockInfo; use std::fmt::Debug; use tokio::sync::mpsc; @@ -11,13 +10,9 @@ use tokio::sync::mpsc; #[cfg_attr(test, mockall::automock)] #[async_trait] pub trait EngineDerivationClient: Debug + Send + Sync { - /// Notifies the [`crate::DerivationActor`] that engine syncing has completed, with the engine's - /// sync state at that point. + /// Notifies the [`crate::DerivationActor`] that engine syncing has completed. /// Note: Does not wait for the derivation client to process this message. - async fn notify_sync_completed( - &self, - sync_state: EngineSyncState, - ) -> DerivationClientResult<()>; + async fn notify_sync_completed(&self, safe_head: L2BlockInfo) -> DerivationClientResult<()>; /// Sends the new engine `safe_head` to the [`crate::DerivationActor`]. /// Note: Does not wait for the derivation client to process this message. @@ -38,14 +33,11 @@ pub struct QueuedEngineDerivationClient { #[async_trait] impl EngineDerivationClient for QueuedEngineDerivationClient { - async fn notify_sync_completed( - &self, - sync_state: EngineSyncState, - ) -> DerivationClientResult<()> { + async fn notify_sync_completed(&self, safe_head: L2BlockInfo) -> DerivationClientResult<()> { info!(target: "engine", "Sending sync completed to derivation actor"); self.derivation_actor_request_tx - .send(DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(sync_state))) + .send(DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(safe_head))) .await .map_err(|_| { DerivationClientError::RequestError("request channel closed.".to_string()) diff --git a/rust/kona/crates/node/service/src/actors/engine/tests.rs b/rust/kona/crates/node/service/src/actors/engine/tests.rs index d000710cbf1..b61712d95b5 100644 --- a/rust/kona/crates/node/service/src/actors/engine/tests.rs +++ b/rust/kona/crates/node/service/src/actors/engine/tests.rs @@ -312,5 +312,5 @@ async fn derivation_is_handed_the_local_safe_head() { else { panic!("expected the sync completion second"); }; - assert_eq!(sent.local_safe_head(), local_safe_head); + assert_eq!(*sent, local_safe_head); } From e2bcf65ec2a0e8794c37936808398f103b90f775 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 12 Nov 2026 14:38:22 +0100 Subject: [PATCH 07/14] kona-node-service: reject a delegate status whose finalized is above its safe Such a snapshot is inconsistent: the engine would refuse to finalize it. op-node's follow mode rejects it the same way. A test also pins that any finalized block, genesis included, is accepted before the engine establishes finality, and the doc on the finality check now states that it only catches a fork at the engine's finalized height. Co-Authored-By: Claude Opus 5.5 --- .../src/actors/derivation/delegated/actor.rs | 23 ++++++++++++-- .../src/actors/derivation/delegated/tests.rs | 31 +++++++++++++++++++ 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs index c7ec925ecd2..9962dd9e388 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs @@ -165,6 +165,10 @@ where /// Rejects a finalized L2 block behind, or at the same height as but different from, the /// engine's. Either means the delegate lags or follows another chain, so none of its status /// applies: its safe head would move the engine back or onto that chain too. + /// + /// Only a fork at the engine's finalized height is caught here. A chain that forked below it + /// and reports a higher finalized block passes, and the execution layer rejects the forkchoice + /// update that would move onto it. fn validate_finalized_l2_progress( &self, reported: BlockNumHash, @@ -182,9 +186,20 @@ where Ok(()) } - /// Verifies that the finalized L2 block reported by the derivation delegate does not move back - /// from the engine's, and that its L1 info is consistent with the canonical L1 chain. + /// Verifies that the derivation delegate's finalized L2 block is not above its safe L2 block + /// and does not move back from the engine's, and that its L1 info is consistent with the + /// canonical L1 chain. async fn validate_sync_status(&mut self, v: &SyncStatus) -> bool { + // A finalized block above the safe head is an inconsistent snapshot; the engine would only + // refuse to finalize it. + if v.finalized_l2.block_info.number > v.safe_l2.block_info.number { + let err = DerivationDelegationError::FinalizedAheadOfSafe { + finalized: v.finalized_l2.block_info.number, + safe: v.safe_l2.block_info.number, + }; + warn!(target: "derivation", error = %err, "Inconsistent sync status from delegate"); + return false; + } if let Err(err) = self.validate_finalized_l2_progress(v.finalized_l2.block_info.id()) { warn!( target: "derivation", @@ -291,6 +306,10 @@ where #[derive(Error, Debug)] enum DerivationDelegationError { + /// The delegate's finalized L2 block is above its safe L2 block. + #[error("Finalized L2 {finalized} is ahead of safe L2 {safe}")] + FinalizedAheadOfSafe { finalized: u64, safe: u64 }, + /// The delegate's finalized L2 block is a different block at the engine's finalized height. #[error("Finalized L2 {reported:?} conflicts with the engine's finalized L2 {engine:?}")] FinalizedL2Conflict { engine: BlockNumHash, reported: BlockNumHash }, diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs index 0eda07f0beb..2468e203727 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs @@ -250,3 +250,34 @@ async fn poll_accepts_status_after_engine_skipped_forwarded_finality( assert_eq!(harness.forwarded(), forwarded_for(&canonical_status)); } + +/// Before the engine establishes finality, any finalized block from the delegate may initialize +/// it, including one at genesis height. +#[tokio::test(start_paused = true)] +async fn first_poll_accepts_finality_before_the_engine_establishes_it() { + let status = with_finalized_l2(l2_block(0, FINALIZED_ORIGIN)); + let mut harness = Harness::new([Ok(status.clone())]); + harness.complete_engine_sync().await; + + harness.actor.step().await.unwrap(); + + assert_eq!(harness.forwarded(), forwarded_for(&status)); +} + +/// A status whose finalized block is above its safe block is inconsistent and rejected whole. +#[rstest] +#[case::at_safe_head(l2_block(20, FINALIZED_ORIGIN), true)] +#[case::ahead_of_safe_head(l2_block(21, FINALIZED_ORIGIN), false)] +#[tokio::test(start_paused = true)] +async fn poll_rejects_status_whose_finalized_is_ahead_of_its_safe( + #[case] finalized_l2: L2BlockInfo, + #[case] accepted: bool, +) { + let status = with_finalized_l2(finalized_l2); + let mut harness = Harness::new([Ok(status.clone())]); + harness.complete_engine_sync().await; + + harness.actor.step().await.unwrap(); + + assert_eq!(harness.forwarded(), forwarded_if(accepted, &status)); +} From 5c7d4925978a6aaadb62fd8d98f8bdd6dcdf6e2c Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 12 Nov 2026 14:38:22 +0100 Subject: [PATCH 08/14] kona-node-service: drive a skipped finalization through the engine actor The test sends stale finalizations through the actor: it keeps running, an identical repeat keeps the same skip in memory, and a reset forgets it. Co-Authored-By: Claude Opus 5.5 --- .../service/src/actors/engine/queue_tests.rs | 46 ++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs b/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs index 3dc21d249e1..0b4bc8f58ff 100644 --- a/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs +++ b/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs @@ -4,7 +4,7 @@ use alloy_rpc_types_engine::{ ExecutionPayloadV1, ForkchoiceUpdated, PayloadStatus, PayloadStatusEnum, }; use kona_engine::{ - ConsolidateInput, EngineSyncStateUpdate, NoopBlockSink, + ConsolidateInput, EngineSyncStateUpdate, FinalizeBlockId, NoopBlockSink, test_utils::{TestEngineStateBuilder, l1_info_deposit, test_block_info, test_engine_client}, }; use op_alloy_consensus::OpBlock; @@ -140,3 +140,47 @@ async fn delegated_safe_head_is_sent_on_every_poll() { l1.assert_finished(); l2.assert_finished(); } + +/// A finalization the engine skips leaves the actor running, an identical repeat keeps the same +/// skip in memory, and a reset forgets it. +#[tokio::test(start_paused = true)] +async fn skipped_finalization_keeps_the_actor_running_until_a_reset_forgets_it() { + let config = Arc::new(RollupConfig::default()); + let (client, l1, l2) = test_engine_client(config.clone()); + let state = TestEngineStateBuilder::new() + .with_unsafe_head(test_block_info(30)) + .with_safe_head(test_block_info(20)) + .with_finalized_head(test_block_info(10)) + .build(); + let (state_tx, _state_rx) = watch::channel(state); + let (requests, request_rx) = mpsc::channel(8); + let (derivation_tx, _derivation_rx) = mpsc::channel(8); + let mut actor = EngineActor::new( + Arc::new(client), + config, + QueuedEngineDerivationClient { derivation_actor_request_tx: derivation_tx }, + Engine::new(state, state_tx), + None, + request_rx, + Arc::new(NoopBlockSink), + ); + let stale_finalization = + || EngineActorRequest::ProcessFinalizedL2Block(Box::new(FinalizeBlockId::ByNumber(5))); + + // Each step drains the previous request's task, then accepts the next request. + requests.send(stale_finalization()).await.unwrap(); + time::timeout(Duration::from_secs(1), actor.step()).await.unwrap().unwrap(); + requests.send(stale_finalization()).await.unwrap(); + time::timeout(Duration::from_secs(1), actor.step()).await.unwrap().unwrap(); + let first_skip = actor.last_skipped_task_error.clone(); + assert!(first_skip.is_some(), "the skipped finalization is remembered"); + + requests.send(stale_finalization()).await.unwrap(); + time::timeout(Duration::from_secs(1), actor.step()).await.unwrap().unwrap(); + assert_eq!(actor.last_skipped_task_error, first_skip); + + actor.begin_reset(); + assert_eq!(actor.last_skipped_task_error, None); + l1.assert_finished(); + l2.assert_finished(); +} From 55610da28228ae75ea37bab55bd778d44b53ab40 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 12 Nov 2026 23:24:03 +0100 Subject: [PATCH 09/14] kona-engine: skip only a null finalization lookup and name the safe head in BlockNotSafe A pinned finalization target is skipped as `NotYetAvailable` only when the execution layer answers its lookup with null. An error response is a failed lookup again and is retried as a Temporary `TransportError`: during catch-up the consolidate task's lookup of the delegated safe head fails first and holds the queue, so the error branch only turned real RPC failures into a skip that repeated on every poll. `BlockNotSafe` carries the target and the safe head, so the warning shown for a skipped target names both, and two skipped targets are not deduplicated as one. BREAKING CHANGE: `FinalizeTaskError::NotYetAvailable` no longer carries `engine_answer`; `FinalizeTaskError::BlockNotSafe` is now `BlockNotSafe { safe, target }`. Co-Authored-By: Claude Opus 5.5 --- .../src/task_queue/tasks/finalize/error.rs | 15 ++++++---- .../src/task_queue/tasks/finalize/task.rs | 30 ++++++------------- .../task_queue/tasks/finalize/task_test.rs | 21 +++++++------ 3 files changed, 30 insertions(+), 36 deletions(-) diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/error.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/error.rs index d8f36318dec..92eb39d2e96 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/error.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/error.rs @@ -25,8 +25,13 @@ pub enum FinalizeTaskError { /// The block is not safe yet, and therefore cannot be finalized. A reset can lower the safe /// head below a target that was safe when it was requested, and a delegate can run ahead of /// the local safe head. - #[error("Attempted to finalize a block that is not yet safe")] - BlockNotSafe, + #[error("Finalization target {target} is above the safe head {safe}")] + BlockNotSafe { + /// The number of the safe head. + safe: u64, + /// The number of the finalization target. + target: u64, + }, /// The engine returned a block at a different height than the one requested. #[error("Requested block {requested} to finalize, but the engine returned block {returned}")] BlockNumberMismatch { @@ -60,10 +65,8 @@ pub enum FinalizeTaskError { target: BlockNumHash, }, /// The engine has no block at the pinned target's height yet, for example while it catches up. - #[error("The engine does not have finalization target {target:?} yet: {engine_answer}")] + #[error("The engine does not have finalization target {target:?} yet")] NotYetAvailable { - /// What the engine answered the lookup with: null, or an error response. - engine_answer: String, /// The finalization target. target: BlockNumHash, }, @@ -76,7 +79,7 @@ impl EngineTaskError for FinalizeTaskError { fn severity(&self) -> EngineTaskErrorSeverity { match self { Self::BehindFinalizedHead { .. } | - Self::BlockNotSafe | + Self::BlockNotSafe { .. } | Self::ConflictsWithFinalizedHead { .. } | Self::NotCanonical { .. } | Self::NotYetAvailable { .. } => EngineTaskErrorSeverity::Skip, diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs index f810aae585a..bc0d2552292 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs @@ -5,7 +5,6 @@ use crate::{ state::EngineSyncStateUpdate, task_queue::synchronize, }; use alloy_eips::BlockNumHash; -use alloy_transport::{RpcError, TransportErrorKind}; use async_trait::async_trait; use derive_more::Constructor; use kona_genesis::RollupConfig; @@ -30,26 +29,14 @@ pub struct FinalizeTask { } impl FinalizeTask { - /// While the execution layer catches up, it has no block at a pinned target's height yet and - /// may answer with an error instead of null. A pinned target is skipped, carrying the answer - /// and leaving the state unchanged for the next request; failing temporarily would hold this - /// task at the head of the queue and block the imports that let the execution layer catch up. - fn lookup_failed(&self, err: RpcError) -> FinalizeTaskError { - if let FinalizeBlockId::ByHash(target) = self.block_id && - let Some(payload) = err.as_error_resp() - { - let engine_answer = format!("error {} ({})", payload.code, payload.message); - return FinalizeTaskError::NotYetAvailable { engine_answer, target }; - } - FinalizeTaskError::TransportError(err) - } - - /// A missing pinned target is not yet available, as for [`Self::lookup_failed`]. A missing - /// by-number target is below the safe head, so the engine must already have it. + /// A pinned target the execution layer answers with null is not available yet, for example + /// while it catches up: the task is skipped rather than held at the head of the queue ahead + /// of the imports that let it catch up. A missing by-number target is below the safe head, so + /// the engine must already have it. fn block_missing(&self) -> FinalizeTaskError { match self.block_id { FinalizeBlockId::ByHash(target) => { - FinalizeTaskError::NotYetAvailable { engine_answer: "null".to_string(), target } + FinalizeTaskError::NotYetAvailable { target } } FinalizeBlockId::ByNumber(number) => FinalizeTaskError::BlockNotFound(number), } @@ -87,7 +74,7 @@ impl FinalizeTask { .get_l2_block(requested.into()) .full() .await - .map_err(|err| self.lookup_failed(err))? + .map_err(FinalizeTaskError::TransportError)? .ok_or_else(|| self.block_missing())?; let canonical = BlockNumHash { hash: block.header.hash, number: block.header.number }; debug!(target: "engine", ?canonical, "Fetched the block to finalize"); @@ -135,8 +122,9 @@ impl ExecutableEngineTask for FinalizeTask { // Sanity check that the block that is being finalized is at least safe. A later request // finalizes a target skipped here, or a newer block. - if state.sync_state.safe_head().block_info.number < target_number { - return Err(FinalizeTaskError::BlockNotSafe); + let safe_number = state.sync_state.safe_head().block_info.number; + if safe_number < target_number { + return Err(FinalizeTaskError::BlockNotSafe { safe: safe_number, target: target_number }); } let block_info = self.fetch_canonical_block_info().await?; diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs index 0e832d155c2..786bbb19b98 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs @@ -119,7 +119,7 @@ const fn is_number_mismatch(err: &FinalizeTaskError) -> bool { } const fn is_not_safe(err: &FinalizeTaskError) -> bool { - matches!(err, FinalizeTaskError::BlockNotSafe) + matches!(err, FinalizeTaskError::BlockNotSafe { .. }) } const fn is_not_yet_available(err: &FinalizeTaskError) -> bool { @@ -192,8 +192,11 @@ async fn finalize_task_refuses_target_above_safe_head_without_lookup() { let err = finalize(by_hash(&canonical(SAFE + 5)), &mut state, |_| {}).await.unwrap_err(); - assert!(matches!(err, FinalizeTaskError::BlockNotSafe), "{err:?}"); - assert_eq!(err.severity(), EngineTaskErrorSeverity::Critical); + assert!( + matches!(err, FinalizeTaskError::BlockNotSafe { safe: SAFE, target } if target == SAFE + 5), + "{err:?}" + ); + assert_eq!(err.severity(), EngineTaskErrorSeverity::Skip); assert_eq!(state, original_state); } @@ -293,17 +296,17 @@ async fn finalize_task_stops_on_block_at_unrequested_height( assert_eq!(state, original_state); } -/// While the execution layer catches up, it has no block at a pinned target's height yet, and may -/// answer null or an error. Either way the target is skipped, so it neither stops the node nor -/// holds the queue until its upstream resubmits it. A by-number target is below the safe head, so -/// the engine must have it: its absence stops the node and an error response is retried. +/// While the execution layer catches up, it answers null for a pinned target's height, and the +/// target is skipped until its upstream resubmits it. An error response is a failed lookup and is +/// retried for either kind of target. A by-number target is below the safe head, so the engine +/// must have it: its absence stops the node. #[rstest] #[case::pinned_null(pinned_target(), Unavailable::Null, is_not_yet_available, Skip)] #[case::pinned_error_response( pinned_target(), Unavailable::ErrorResponse, - is_not_yet_available, - Skip + is_transport_error, + Temporary )] #[case::by_number_null(numbered_target(), Unavailable::Null, is_block_not_found, Critical)] #[case::by_number_error_response( From d732306ac5445b67e54ab9b209f23ede1f8300c0 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 12 Nov 2026 23:32:42 +0100 Subject: [PATCH 10/14] kona-engine: make the finalize task's missing-block classification const Co-Authored-By: Claude Opus 5.5 --- .../node/engine/src/task_queue/tasks/finalize/task.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs index bc0d2552292..30a08a102bd 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs @@ -33,11 +33,9 @@ impl FinalizeTask { /// while it catches up: the task is skipped rather than held at the head of the queue ahead /// of the imports that let it catch up. A missing by-number target is below the safe head, so /// the engine must already have it. - fn block_missing(&self) -> FinalizeTaskError { + const fn block_missing(&self) -> FinalizeTaskError { match self.block_id { - FinalizeBlockId::ByHash(target) => { - FinalizeTaskError::NotYetAvailable { target } - } + FinalizeBlockId::ByHash(target) => FinalizeTaskError::NotYetAvailable { target }, FinalizeBlockId::ByNumber(number) => FinalizeTaskError::BlockNotFound(number), } } @@ -124,7 +122,10 @@ impl ExecutableEngineTask for FinalizeTask { // finalizes a target skipped here, or a newer block. let safe_number = state.sync_state.safe_head().block_info.number; if safe_number < target_number { - return Err(FinalizeTaskError::BlockNotSafe { safe: safe_number, target: target_number }); + return Err(FinalizeTaskError::BlockNotSafe { + safe: safe_number, + target: target_number, + }); } let block_info = self.fetch_canonical_block_info().await?; From 548e139f52a65851aae17ccbf18ba59fd115a272 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 12 Nov 2026 23:32:42 +0100 Subject: [PATCH 11/14] kona-node-service: retry finalization only while derivation is idle The derivation actor finalizes against the known finalized L1 block on an L1 head update only once derivation is idle, after the update moves it to `Deriving`. While attributes await the engine's confirmation, or a signal is pending, the finalization queue can still hold blocks from before an engine reset that this actor has not been signaled yet, so they are not sent. The comment now describes the cadence as it is: every L1 head while idle, not op-node's at most once per 64 L1 blocks. The finalizer warns when a finalized L1 block below the highest known one is received, as op-node does, and its docs describe the L1-head retry. Co-Authored-By: Claude Opus 5.5 --- .../src/actors/derivation/actor/core.rs | 19 ++++++------ .../src/actors/derivation/actor/tests.rs | 30 +++++++++++++++++++ .../src/actors/derivation/finalizer.rs | 18 +++++++---- 3 files changed, 53 insertions(+), 14 deletions(-) diff --git a/rust/kona/crates/node/service/src/actors/derivation/actor/core.rs b/rust/kona/crates/node/service/src/actors/derivation/actor/core.rs index cc084969b46..2b602e11ff9 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/actor/core.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/actor/core.rs @@ -352,18 +352,19 @@ where DerivationActorRequest::ProcessL1HeadUpdateRequest(l1_head) => { info!(target: "derivation", l1_head = ?*l1_head, "Processing l1 head update"); - // Like op-node's retry when derivation goes idle: a block that became safe after - // L1 finality last moved, or whose finalization the engine skipped because a - // reset lowered its safe head, is finalized without waiting for L1 finality to - // move again. The L1 head cadence batches the blocks confirmed since the last - // update, as op-node's finality delay does, instead of finalizing on every - // confirmation. - if let Some(l2_block_number) = self.finalizer.try_finalize_against_known_l1() { + self.derivation_state_machine.update(&DerivationStateUpdate::L1DataReceived)?; + + // A block that became safe after L1 finality last moved is finalized on the next + // L1 head update instead of waiting for L1 finality to move again. This runs on + // every L1 head while derivation is idle, unlike op-node, which retries at most + // once per 64 L1 blocks of progress. Outside `Deriving` the queue can still hold + // blocks from before an engine reset that this actor has not been signaled yet. + if self.derivation_state_machine.current_state() == DerivationState::Deriving && + let Some(l2_block_number) = self.finalizer.try_finalize_against_known_l1() + { self.finalize_l2_block_by_number(l2_block_number).await?; } - self.derivation_state_machine.update(&DerivationStateUpdate::L1DataReceived)?; - self.attempt_derivation().await?; } DerivationActorRequest::ProcessEngineSafeHeadUpdateRequest(safe_head) => { diff --git a/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs b/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs index 2f9eb318eeb..a7a7174673c 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs @@ -711,6 +711,36 @@ async fn a_block_derived_again_after_a_reset_is_finalized_on_the_next_l1_head() .await; } +/// An L1 head update finalizes against the known L1 block only once derivation is idle: while +/// attributes await the engine's confirmation, the queued block is left for a later update. +#[tokio::test] +async fn an_l1_head_update_does_not_finalize_while_attributes_await_confirmation() { + let mut engine_client = engine_client_accepting_attributes(); + engine_client.expect_send_finalized_l2_block().times(0); + let mut node = RecordingActor::new(engine_client); + node.actor.pipeline.attributes.push_back(OpAttributesWithParent { + parent: l2_block(DERIVED_HEAD), + ..derived_attributes() + }); + node.complete_el_sync_at(RESUME_HEAD).await; + node.finalize_l1(DERIVED_FROM_L1).await; + node.confirm_safe_head(DERIVED_HEAD).await; + assert_eq!( + node.actor.derivation_state_machine.current_state(), + DerivationState::AwaitingSafeHeadConfirmation + ); + + node.handle(DerivationActorRequest::ProcessL1HeadUpdateRequest(Box::new(l1_block( + DERIVED_FROM_L1 + 1, + )))) + .await; + + assert_eq!( + node.actor.derivation_state_machine.current_state(), + DerivationState::AwaitingSafeHeadConfirmation + ); +} + mod database; mod op_node_parity; diff --git a/rust/kona/crates/node/service/src/actors/derivation/finalizer.rs b/rust/kona/crates/node/service/src/actors/derivation/finalizer.rs index bfffd795e16..8af4e2645cc 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/finalizer.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/finalizer.rs @@ -12,8 +12,9 @@ type L2BlockNumber = u64; /// The [`L2Finalizer`] is responsible for tracking L2 blocks derived from L1 blocks and /// determining which L2 blocks can be finalized when L1 blocks are finalized. /// -/// It maintains a queue of derived L2 blocks that are awaiting finalization, and returns -/// the L2 block numbers that can be finalized as new finalized L1 blocks are received. +/// It maintains a queue of derived L2 blocks that are awaiting finalization, and returns the +/// L2 block numbers that can be finalized when a new finalized L1 block is received, or when it is +/// asked again against the highest finalized L1 block it already knows. #[derive(Debug, Default)] pub(crate) struct L2Finalizer { /// A map of `L1 block number -> highest derived L2 block number` within the L1 epoch, used to @@ -46,7 +47,8 @@ impl L2Finalizer { self.awaiting_finalization.clear(); } - /// Attempts to find L2 blocks that can be finalized based on the new finalized L1 block. + /// Records `new_finalized_l1_block` as the highest finalized L1 block, unless a higher one is + /// already known, and attempts to finalize against the highest one. /// /// Returns `Some(l2_block_number)` if there is an L2 block that can be finalized, /// or `None` if no blocks are ready for finalization. @@ -54,8 +56,14 @@ impl L2Finalizer { &mut self, new_finalized_l1_block: BlockInfo, ) -> Option { - if self.finalized_l1_number.is_none_or(|known| known < new_finalized_l1_block.number) { - self.finalized_l1_number = Some(new_finalized_l1_block.number); + match self.finalized_l1_number { + Some(known) if new_finalized_l1_block.number < known => warn!( + target: "derivation", + known, + received = new_finalized_l1_block.number, + "Finalized L1 block moved backwards, ignoring it. Is the L1 provider corrupted?" + ), + _ => self.finalized_l1_number = Some(new_finalized_l1_block.number), } self.try_finalize_against_known_l1() } From 539115a7eef872491719688709a0be0a861785c6 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Fri, 13 Nov 2026 01:34:36 +0100 Subject: [PATCH 12/14] kona-node: pin finalize lookup failures and the reset resend in tests - The finalize task table covers a pinned target whose lookup fails at the transport, and asserts the target that NotYetAvailable carries. - The actor test sends the target before the reset and again after it is derived again. - The delegate poll interval is a module constant shared with the tests. Co-Authored-By: Claude Opus 5.5 --- .../task_queue/tasks/finalize/task_test.rs | 19 ++++++++++++++----- .../src/actors/derivation/actor/tests.rs | 8 +++++--- .../src/actors/derivation/delegated/actor.rs | 10 ++++------ .../src/actors/derivation/delegated/tests.rs | 13 ++++++++----- 4 files changed, 31 insertions(+), 19 deletions(-) diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs index 786bbb19b98..45420631095 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs @@ -122,8 +122,9 @@ const fn is_not_safe(err: &FinalizeTaskError) -> bool { matches!(err, FinalizeTaskError::BlockNotSafe { .. }) } -const fn is_not_yet_available(err: &FinalizeTaskError) -> bool { - matches!(err, FinalizeTaskError::NotYetAvailable { .. }) +fn is_not_yet_available(err: &FinalizeTaskError) -> bool { + let pinned_id = l2_block_info(&canonical(ADVANCED)).block_info.id(); + matches!(err, FinalizeTaskError::NotYetAvailable { target } if *target == pinned_id) } const fn is_block_not_found(err: &FinalizeTaskError) -> bool { @@ -149,12 +150,14 @@ fn numbered_target() -> FinalizeBlockId { enum Unavailable { ErrorResponse, Null, + TransportFailure, } fn expect_unavailable_block(l2: &RpcMock, number: u64, answer: Unavailable) { match answer { Unavailable::ErrorResponse => l2.expect_error_code("eth_getBlockByNumber", -32000), Unavailable::Null => l2.expect_block_by_number(number, None), + Unavailable::TransportFailure => l2.expect_transport_error("eth_getBlockByNumber"), } } @@ -297,9 +300,9 @@ async fn finalize_task_stops_on_block_at_unrequested_height( } /// While the execution layer catches up, it answers null for a pinned target's height, and the -/// target is skipped until its upstream resubmits it. An error response is a failed lookup and is -/// retried for either kind of target. A by-number target is below the safe head, so the engine -/// must have it: its absence stops the node. +/// target is skipped until its upstream resubmits it. An error response or a transport failure is a +/// failed lookup and is retried for either kind of target. A by-number target is below the safe +/// head, so the engine must have it: its absence stops the node. #[rstest] #[case::pinned_null(pinned_target(), Unavailable::Null, is_not_yet_available, Skip)] #[case::pinned_error_response( @@ -308,6 +311,12 @@ async fn finalize_task_stops_on_block_at_unrequested_height( is_transport_error, Temporary )] +#[case::pinned_transport_failure( + pinned_target(), + Unavailable::TransportFailure, + is_transport_error, + Temporary +)] #[case::by_number_null(numbered_target(), Unavailable::Null, is_block_not_found, Critical)] #[case::by_number_error_response( numbered_target(), diff --git a/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs b/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs index a7a7174673c..818294a343b 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs @@ -689,18 +689,20 @@ async fn a_block_safe_after_its_l1_finalized_is_finalized_on_the_next_l1_head() } } -/// A reset clears the finalization queue but keeps the finalized L1 block, so a block derived -/// again after the reset is finalized on the next L1 head update. +/// A reset clears the finalization queue but keeps the finalized L1 block. A block sent for +/// finalization before the reset, which the engine may skip once the reset lowers its safe head, is +/// sent again on the next L1 head update after it is derived again. #[tokio::test] async fn a_block_derived_again_after_a_reset_is_finalized_on_the_next_l1_head() { let mut engine_client = engine_client_accepting_attributes(); engine_client .expect_send_finalized_l2_block() .withf(|block| matches!(block, FinalizeBlockId::ByNumber(DERIVED_HEAD))) - .times(1) + .times(2) .returning(|_| Ok(())); let mut node = RecordingActor::new(engine_client); node.complete_el_sync_at(RESUME_HEAD).await; + node.confirm_safe_head(DERIVED_HEAD).await; node.finalize_l1(DERIVED_FROM_L1).await; node.reset_to_resume_head().await; node.confirm_safe_head(DERIVED_HEAD).await; diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs index 9962dd9e388..fb860835003 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs @@ -15,6 +15,9 @@ use tokio::{ time, }; +/// How often the derivation delegate is polled for its sync status. +pub(super) const DERIVATION_DELEGATE_POLL_INTERVAL: time::Duration = time::Duration::from_secs(4); + /// The [`NodeActor`] for the delegate derivation sub-routine. /// /// This actor is responsible for receiving messages from [`NodeActor`]s and polls @@ -71,8 +74,7 @@ where derivation_origin_tx: watch::Sender>, engine_state_rx: watch::Receiver, ) -> Self { - let mut delegated_derivation_ticker = - time::interval(Self::DERIVATION_DELEGATE_POLL_INTERVAL); + let mut delegated_derivation_ticker = time::interval(DERIVATION_DELEGATE_POLL_INTERVAL); delegated_derivation_ticker.set_missed_tick_behavior(time::MissedTickBehavior::Skip); Self { inbound_request_rx, @@ -127,10 +129,6 @@ where DelegateProvider: DerivationDelegateProvider + 'static, L1Provider: ChainProvider + 'static, { - /// Hardcoded poll interval for Derivation Delegation - const DERIVATION_DELEGATE_POLL_INTERVAL: std::time::Duration = - std::time::Duration::from_secs(4); - /// Validates a single L1 block height and hash against the canonical L1 chain. async fn validate_l1_block( &mut self, diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs index 02ac1e3836d..28c24fbfb23 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs @@ -1,4 +1,7 @@ -use super::{DelegateDerivationActor, DerivationDelegateClientError, DerivationDelegateProvider}; +use super::{ + DelegateDerivationActor, DerivationDelegateClientError, DerivationDelegateProvider, + actor::DERIVATION_DELEGATE_POLL_INTERVAL, +}; use crate::{DerivationActorRequest, EngineActorRequest, NodeActor, QueuedDerivationEngineClient}; use alloy_eips::BlockNumHash; use alloy_primitives::B256; @@ -15,8 +18,6 @@ use tokio::{ time::{self, Duration, Instant}, }; -const POLL_INTERVAL: Duration = Duration::from_secs(4); - const SAFE_ORIGIN: u64 = 1; const FINALIZED_ORIGIN: u64 = 2; const CURRENT_L1: u64 = 3; @@ -292,7 +293,9 @@ impl DelegateReply { async fn polls_wait_for_engine_sync_completion() { let mut harness = Harness::new([]); - assert!(time::timeout(POLL_INTERVAL * 3, harness.actor.step()).await.is_err()); + assert!( + time::timeout(DERIVATION_DELEGATE_POLL_INTERVAL * 3, harness.actor.step()).await.is_err() + ); assert_eq!(harness.forwarded(), Vec::new()); assert_eq!(*harness.current_l1.borrow(), None); @@ -332,7 +335,7 @@ async fn unchanged_status_is_resent_on_every_poll() { assert_eq!(harness.forwarded(), forwarded_for(&delegate_status())); } - assert_eq!(Instant::now() - first_poll, POLL_INTERVAL * (POLLS - 1)); + assert_eq!(Instant::now() - first_poll, DERIVATION_DELEGATE_POLL_INTERVAL * (POLLS - 1)); } /// Before the engine establishes finality, any finalized block from the delegate may initialize From fb19fabb49cc81d13f5b3fcb1db0516402d55b2c Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Fri, 13 Nov 2026 01:49:29 +0100 Subject: [PATCH 13/14] kona-node-service: drop the unused Duration import in the delegate tests Co-Authored-By: Claude Opus 5.5 --- .../node/service/src/actors/derivation/delegated/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs index 28c24fbfb23..90189735385 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs @@ -15,7 +15,7 @@ use rstest::rstest; use std::{collections::VecDeque, sync::Mutex}; use tokio::{ sync::{mpsc, watch}, - time::{self, Duration, Instant}, + time::{self, Instant}, }; const SAFE_ORIGIN: u64 = 1; From 21c75569645a8e26122015a610925634e71b68ca Mon Sep 17 00:00:00 2001 From: Josh Klopfenstein Date: Tue, 17 Nov 2026 08:00:14 -0800 Subject: [PATCH 14/14] kona-node-service: expect newPayloadV2 in the finalization reset test develop now inserts pre-Canyon payloads with engine_newPayloadV2 (#24092), so finalization_target_survives_resets_below_it must expect that method. Co-Authored-By: Claude Opus 5.5 --- rust/kona/crates/node/service/src/actors/engine/queue_tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs b/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs index f3fb516bba6..cd3c695f383 100644 --- a/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs +++ b/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs @@ -304,7 +304,7 @@ async fn finalization_target_survives_resets_below_it(#[case] target_kind: Targe actor.step().await.unwrap(); assert_eq!(actor.engine.len(), 1, "only the unsafe import may remain queued"); assert!(!actor.reset_pending); - l2.expect("engine_newPayloadV1", PayloadStatus::from_status(PayloadStatusEnum::Valid)); + l2.expect("engine_newPayloadV2", PayloadStatus::from_status(PayloadStatusEnum::Valid)); l2.expect_forkchoice_updated( forkchoice(unsafe_head, genesis, genesis), PayloadStatusEnum::Valid,