From c1c8522c0dd6632d52e55e0df9e3eacf1416a7c6 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Fri, 6 Nov 2026 20:36:49 -0800 Subject: [PATCH 1/7] kona-engine: deliver and retry delegated forkchoice updates A delegated safe head below the unsafe head was treated like an intermediate span-batch block: it moved the consensus labels without a forkchoice update. An update that left the labels unchanged was also skipped, but a SYNCING answer moves the labels, so an identical retry never reached the execution layer. And while the execution layer was still syncing and lacked the delegated safe block, consolidation reset the engine, which restarted the sync (the delegated case of #23931). Only derived attributes inside a span batch now defer the update. The delegated consolidation paths and the reset always send their forkchoice, so each delegate poll is the retry after a SYNCING answer or an execution-layer restart. A delegated safe head whose block the syncing execution layer does not have yet moves only the safe labels, as in op-node's follow-source mode, instead of resetting; the missing block is logged at warn. One helper applies a delegated safe head with or without the unsafe head. The finalize that follows such a poll still fails on the same syncing execution layer until #23908 skips it, so this lands together with #23899 and #23908. `test_utils::l1_info_deposit` replaces three inline copies of the L1-info deposit, and the delegated consolidation table asserts what reaches the block sink. Co-Authored-By: Claude Opus 5.5 Co-Authored-By: Claude Fable 5.1 --- .../crates/node/engine/src/task_queue/core.rs | 40 ++- .../src/task_queue/tasks/canonicalize_test.rs | 14 +- .../src/task_queue/tasks/consolidate/task.rs | 317 ++++++++++++++++-- .../src/task_queue/tasks/finalize/task.rs | 4 +- .../src/task_queue/tasks/insert/task.rs | 4 +- .../node/engine/src/task_queue/tasks/mod.rs | 2 +- .../task_queue/tasks/synchronize/helper.rs | 25 +- .../src/task_queue/tasks/synchronize/mod.rs | 5 +- .../src/task_queue/tasks/synchronize/tests.rs | 134 ++++++++ .../node/engine/src/test_utils/deposits.rs | 14 + .../crates/node/engine/src/test_utils/mod.rs | 3 + .../crates/node/engine/src/test_utils/rpc.rs | 17 + .../service/src/actors/engine/queue_tests.rs | 71 +++- 13 files changed, 588 insertions(+), 62 deletions(-) create mode 100644 rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/tests.rs create mode 100644 rust/kona/crates/node/engine/src/test_utils/deposits.rs diff --git a/rust/kona/crates/node/engine/src/task_queue/core.rs b/rust/kona/crates/node/engine/src/task_queue/core.rs index 816c8d0519e..acc3fa27d40 100644 --- a/rust/kona/crates/node/engine/src/task_queue/core.rs +++ b/rust/kona/crates/node/engine/src/task_queue/core.rs @@ -5,7 +5,7 @@ use crate::{ EngineClient, EngineState, EngineSyncStateUpdate, EngineTask, EngineTaskError, EngineTaskErrorSeverity, Metrics, SyncStartError, SynchronizeTaskError, find_starting_forkchoice, - task_queue::{EngineTaskErrors, synchronize}, + task_queue::{EngineTaskErrors, ForkchoiceDelivery, synchronize}, }; use kona_genesis::RollupConfig; use kona_protocol::L2BlockInfo; @@ -115,7 +115,9 @@ impl Engine { ) -> Result { let start = find_starting_forkchoice(&config, client.as_ref()).await?; - // One attempt. The actor retains the reset request and retries transient failures. + // One attempt. The actor retains the reset request and retries transient failures. The + // execution layer may have lost or changed its forkchoice, so the reset sends one even when + // the recovered labels equal the current ones. synchronize( client.as_ref(), &mut self.state, @@ -125,6 +127,7 @@ impl Engine { safe_head: Some(start.safe), finalized_head: Some(start.finalized), }, + ForkchoiceDelivery::Always, ) .await?; @@ -345,4 +348,37 @@ mod recovery_tests { imported.header.hash_slow() ); } + + // An execution layer that restarted can have lost its forkchoice, so a reset sends the + // recovered forkchoice even when the labels already equal it. + #[tokio::test] + async fn reset_resends_an_unchanged_forkchoice() { + let block: ::BlockResponse = + Default::default(); + let mut config = RollupConfig::default(); + config.genesis.l2.hash = block.header.inner.hash_slow(); + let genesis = + L2BlockInfo::from_block_and_genesis(&block.clone().into_consensus(), &config.genesis) + .unwrap(); + let config = Arc::new(config); + let state = TestEngineStateBuilder::new().with_unsafe_head(genesis).build(); + let forkchoice = state.sync_state.create_forkchoice_state(); + let (client, l1, l2) = test_engine_client(config.clone()); + for tag in ["finalized", "safe", "latest"] { + l2.expect_params("eth_getBlockByNumber", serde_json::json!([tag, true]), &block); + } + l1.expect( + "eth_getBlockByHash", + alloy_rpc_types_eth::Block::::default(), + ); + l2.expect_forkchoice_updated(forkchoice, PayloadStatusEnum::Valid); + let (state_tx, _state_rx) = tokio::sync::watch::channel(state); + let mut engine = Engine::new(state, state_tx); + + engine.reset(Arc::new(client), config).await.unwrap(); + + assert_eq!(engine.state().sync_state, state.sync_state); + l1.assert_finished(); + l2.assert_finished(); + } } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/canonicalize_test.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/canonicalize_test.rs index 9da42da2fe8..f8d73461107 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/canonicalize_test.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/canonicalize_test.rs @@ -1,11 +1,14 @@ use super::*; -use crate::{InsertTaskError, NoopBlockSink, SynchronizeTaskError, test_utils::test_engine_client}; +use crate::{ + InsertTaskError, NoopBlockSink, SynchronizeTaskError, + test_utils::{l1_info_deposit, test_engine_client}, +}; use alloy_consensus::{Block, SignableTransaction, TxEip1559}; -use alloy_primitives::{Sealed, Signature}; +use alloy_primitives::Signature; use alloy_rpc_types_engine::{ ExecutionPayloadV1, ForkchoiceUpdated, PayloadStatus, PayloadStatusEnum, }; -use op_alloy_consensus::{OpTxEnvelope, TxDeposit}; +use op_alloy_consensus::OpTxEnvelope; #[derive(Debug, Default)] pub(crate) struct RecordingSink(pub(crate) std::sync::Mutex>); @@ -106,10 +109,7 @@ async fn invalid_committed_payload_does_not_build_an_uncommitted_replacement( ) { let PayloadFixture { cfg, .. } = payload_fixture(); let mut block = Block::::default(); - block.body.transactions.push(OpTxEnvelope::Deposit(Sealed::new(TxDeposit { - input: kona_protocol::L1BlockInfoBedrock::default().encode_calldata(), - ..Default::default() - }))); + block.body.transactions.push(l1_info_deposit()); if !deposits_only { block.body.transactions.push(OpTxEnvelope::Eip1559( TxEip1559::default().into_signed(Signature::test_signature()), diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs index a9ada74d3c0..9531eac4aa3 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs @@ -3,7 +3,7 @@ use crate::{ ConsolidateTaskError, EngineClient, EngineState, ExecutableEngineTask, ImportedBlockSink, state::EngineSyncStateUpdate, - task_queue::{build_and_import, synchronize}, + task_queue::{ForkchoiceDelivery, build_and_import, synchronize}, }; use alloy_rpc_types_eth::Block; use async_trait::async_trait; @@ -52,13 +52,19 @@ impl ConsolidateInput { } } - /// Returns true if this is `Attributes` and `attributes.is_last_in_span` is true. - const fn is_attributes_last_in_span(&self) -> bool { - matches!( - self, - Self::Attributes(attributes) - if attributes.is_last_in_span - ) + /// Whether these are derived attributes for a block before the last block of its span batch. + const fn is_intermediate_span_block(&self) -> bool { + matches!(self, Self::Attributes(attributes) if !attributes.is_last_in_span) + } + + /// Derived attributes move the safe head on every consolidation, so an unchanged forkchoice was + /// already sent. A delegated safe head can repeat a forkchoice the execution layer answered + /// `SYNCING` or lost, so it is always sent. + const fn forkchoice_delivery(&self) -> ForkchoiceDelivery { + match self { + Self::Attributes(_) => ForkchoiceDelivery::WhenChanged, + Self::BlockInfo(_) => ForkchoiceDelivery::Always, + } } } @@ -120,27 +126,75 @@ impl ConsolidateTask { // self-consistent head state. This is required to correctly handle reorgs (where unsafe // may be ahead on a non-canonical fork) and to trigger EL sync when the local unsafe head // lags behind the safe head. - synchronize( - self.client.as_ref(), + self.apply_delegated_safe_head( state, + safe_l2, EngineSyncStateUpdate { - unsafe_head: Some(*safe_l2), + local_safe_head: Some(*safe_l2), safe_head: Some(*safe_l2), + unsafe_head: Some(*safe_l2), + ..Default::default() + }, + "Updated safe head via follow safe", + ) + .await + } + + /// Moves the safe labels to a delegated safe head that the execution layer does not have yet, + /// leaving the unsafe head in place. + /// + /// A block missing below the unsafe head means the execution layer is still syncing toward the + /// unsafe head. Resetting would restart that sync from the execution layer's old labels, and + /// moving the unsafe head back would retarget it. Like op-node's follow-source mode, only the + /// safe labels follow the delegate, and the forkchoice update carries them to the execution + /// layer. + async fn follow_safe_head_during_el_sync( + &self, + state: &mut EngineState, + safe_l2: &L2BlockInfo, + ) -> Result<(), ConsolidateTaskError> { + warn!( + target: "engine", + safe_l2 = %safe_l2, + unsafe_number = state.sync_state.unsafe_head().block_info.number, + "Execution layer does not have the delegated safe head yet; keeping its sync target" + ); + + self.apply_delegated_safe_head( + state, + safe_l2, + EngineSyncStateUpdate { local_safe_head: Some(*safe_l2), + safe_head: Some(*safe_l2), ..Default::default() }, + "Updated safe head while the execution layer syncs", ) .await - .map_err(|e| { - warn!(target: "engine", ?e, "Apply safe head failed"); - e - })?; + } + + /// Applies `update`, whose labels follow the delegated `safe_l2`, and always sends its + /// forkchoice update: the execution layer may have answered `SYNCING` to the same labels + /// before, or lost them in a restart. + async fn apply_delegated_safe_head( + &self, + state: &mut EngineState, + safe_l2: &L2BlockInfo, + update: EngineSyncStateUpdate, + outcome: &'static str, + ) -> Result<(), ConsolidateTaskError> { + synchronize(self.client.as_ref(), state, update, ForkchoiceDelivery::Always) + .await + .map_err(|e| { + warn!(target: "engine", ?e, safe_l2 = %safe_l2, "Applying the delegated safe head failed"); + e + })?; info!( target: "engine", hash = %safe_l2.block_info.hash, number = safe_l2.block_info.number, - "Updated safe head via follow safe" + "{outcome}" ); Ok(()) @@ -175,6 +229,9 @@ impl ConsolidateTask { let block = match self.client.l2_block_by_label(block_num.into()).await { Ok(Some(block)) => block, Ok(None) => { + if let ConsolidateInput::BlockInfo(safe_l2) = &self.input { + return self.follow_safe_head_during_el_sync(state, safe_l2).await; + } warn!(target: "engine", "Received `None` block for {}", block_num); return Err(ConsolidateTaskError::MissingUnsafeL2Block(block_num)); } @@ -194,10 +251,10 @@ impl ConsolidateTask { let consensus_block = block.into_consensus().map_transactions(|t| t.inner.inner.into_inner()); match L2BlockInfo::from_block_and_genesis(&consensus_block, &self.cfg.genesis) { - // Only issue a forkchoice update if the attributes are the last in the span - // batch. This is an optimization to avoid sending a FCU - // call for every block in the span batch. - Ok(block_info) if !self.input.is_attributes_last_in_span() => { + // A span batch's blocks before its last one only move the safe label; the last + // block's forkchoice update delivers them all. This avoids sending a FCU call for + // every block in the span batch. + Ok(block_info) if self.input.is_intermediate_span_block() => { // The next attributes built are this block's child, and ask for its config. self.block_sink.block_imported(consensus_block, block_info); @@ -229,6 +286,7 @@ impl ConsolidateTask { local_safe_head: Some(block_info), ..Default::default() }, + self.input.forkchoice_delivery(), ) .await .map_err(|e| { @@ -297,10 +355,16 @@ impl ExecutableEngineTask for ConsolidateTask { mod tests { use super::*; use crate::{ - EngineTaskError, ImportedBlockSink, SealTaskError, test_utils::test_engine_client, + EngineSyncState, EngineTaskError, ImportedBlockSink, NoopBlockSink, SealTaskError, + test_utils::{ + TestAttributesBuilder, TestEngineStateBuilder, l1_info_deposit, test_block_info, + test_engine_client, + }, }; - use alloy_eips::BlockNumHash; - use alloy_primitives::B256; + use alloy_consensus::transaction::Recovered; + use alloy_eips::{BlockNumHash, Encodable2718}; + use alloy_primitives::{Address, B256}; + use alloy_rpc_types_engine::{ForkchoiceUpdated, PayloadStatus, PayloadStatusEnum}; use alloy_rpc_types_eth::{Block as RpcBlock, BlockTransactions, Header as RpcHeader}; use kona_genesis::ChainGenesis; use kona_protocol::BlockInfo; @@ -320,21 +384,14 @@ mod tests { // Consolidation reads the unsafe block in full to compare it against the derived // attributes; the next attributes built ask for this same block's system config, so it // must reach the sink rather than being dropped. - let header = RpcHeader::new(alloy_consensus::Header::default()); - let block_hash = header.hash; - let block: RpcBlock = - RpcBlock::new(header, BlockTransactions::Full(vec![])); - - // Pin genesis to this block so its L2BlockInfo needs no L1-info deposit. - let cfg = Arc::new(RollupConfig { - genesis: ChainGenesis { - l2: BlockNumHash { hash: block_hash, number: 0 }, - ..Default::default() - }, - ..Default::default() - }); + let (block, cfg) = genesis_pinned_block(0); + let block_hash = block.header.hash; let (client, l1, l2) = test_engine_client(cfg.clone()); l2.expect_params("eth_getBlockByNumber", serde_json::json!(["0x0", true]), block); + l2.expect( + "engine_forkchoiceUpdatedV3", + ForkchoiceUpdated::new(PayloadStatus::from_status(PayloadStatusEnum::Valid)), + ); let client = Arc::new(client); let safe_l2 = L2BlockInfo { @@ -479,4 +536,192 @@ mod tests { "got {err:?}" ); } + + /// An engine state with the unsafe head at 20, the finalized head at 5 and `safe_head` between. + fn engine_state_with_safe_head(safe_head: L2BlockInfo) -> EngineState { + TestEngineStateBuilder::new() + .with_unsafe_head(test_block_info(20)) + .with_safe_head(safe_head) + .with_finalized_head(test_block_info(5)) + .build() + } + + fn with_safe_head(state: &EngineState, safe_head: L2BlockInfo) -> EngineSyncState { + state.sync_state.apply_update(EngineSyncStateUpdate { + local_safe_head: Some(safe_head), + safe_head: Some(safe_head), + ..Default::default() + }) + } + + fn block_info_of(cfg: &RollupConfig, block: &RpcBlock) -> L2BlockInfo { + let consensus_block = + block.clone().into_consensus().map_transactions(|t| t.inner.inner.into_inner()); + L2BlockInfo::from_block_and_genesis(&consensus_block, &cfg.genesis).unwrap() + } + + /// An empty block at `number`, with genesis pinned to it so its `L2BlockInfo` needs no L1-info + /// deposit. + fn genesis_pinned_block(number: u64) -> (RpcBlock, Arc) { + let header = RpcHeader::new(alloy_consensus::Header { number, ..Default::default() }); + let cfg = Arc::new(RollupConfig { + genesis: ChainGenesis { + l2: BlockNumHash { hash: header.hash, number }, + ..Default::default() + }, + ..Default::default() + }); + (RpcBlock::new(header, BlockTransactions::Full(vec![])), cfg) + } + + /// What the execution layer answers for the block at the delegated safe head's height. + fn el_block(el_has_block: bool, block: RpcBlock) -> Option> { + el_has_block.then_some(block) + } + + /// The safe head before consolidation: either already the delegated target, as a `SYNCING` + /// answer or an earlier poll leaves it, or an older block. + fn initial_safe_head(starts_at_target: bool, target: L2BlockInfo) -> L2BlockInfo { + if starts_at_target { target } else { test_block_info(5) } + } + + /// Below the unsafe head, a delegated safe head always reaches the execution layer: when the + /// labels already hold it, and when the execution layer is still syncing and does not have its + /// block yet. A missing block then moves only the safe labels instead of resetting, and only a + /// block the execution layer has reaches the block sink. + #[rstest::rstest] + #[case::new_target(true, false, PayloadStatusEnum::Valid)] + #[case::repeated_target(true, true, PayloadStatusEnum::Valid)] + #[case::new_target_missing_from_syncing_el(false, false, PayloadStatusEnum::Syncing)] + #[case::repeated_target_missing_from_syncing_el(false, true, PayloadStatusEnum::Syncing)] + #[tokio::test] + async fn delegated_safe_head_below_unsafe_head_is_always_sent( + #[case] el_has_block: bool, + #[case] starts_at_target: bool, + #[case] reply: PayloadStatusEnum, + ) { + let (block, cfg) = genesis_pinned_block(10); + let delegated_safe_head = block_info_of(&cfg, &block); + let mut state = + engine_state_with_safe_head(initial_safe_head(starts_at_target, delegated_safe_head)); + let expected_sync_state = with_safe_head(&state, delegated_safe_head); + let (client, l1, l2) = test_engine_client(cfg.clone()); + l2.expect_params( + "eth_getBlockByNumber", + serde_json::json!(["0xa", true]), + el_block(el_has_block, block), + ); + l2.expect_forkchoice_updated(expected_sync_state.create_forkchoice_state(), reply); + let sink = Arc::new(RecordingSink::default()); + let task = ConsolidateTask::new( + Arc::new(client), + cfg, + ConsolidateInput::BlockInfo(delegated_safe_head), + sink.clone(), + ); + + task.execute(&mut state).await.unwrap(); + + assert_eq!(state.sync_state, expected_sync_state); + assert_eq!(sink.0.lock().unwrap().len(), usize::from(el_has_block)); + l1.assert_finished(); + l2.assert_finished(); + } + + /// Unlike a delegated safe head, derived attributes whose block is missing below the unsafe + /// head still reset the engine. + #[tokio::test] + async fn derived_attributes_missing_from_el_reset_the_engine() { + let cfg = Arc::new(RollupConfig::default()); + let attributes = derived_attributes(true); + let mut state = engine_state_with_safe_head(attributes.parent); + let state_before = state; + let (client, l1, l2) = test_engine_client(cfg.clone()); + l2.expect_params( + "eth_getBlockByNumber", + serde_json::json!(["0xa", true]), + None::>, + ); + let task = ConsolidateTask::new( + Arc::new(client), + cfg, + ConsolidateInput::Attributes(Box::new(attributes)), + Arc::new(NoopBlockSink), + ); + + let err = task.execute(&mut state).await.unwrap_err(); + + assert!(matches!(err, ConsolidateTaskError::MissingUnsafeL2Block(10)), "got {err:?}"); + assert_eq!(err.severity(), crate::EngineTaskErrorSeverity::Reset); + assert_eq!(state, state_before); + l1.assert_finished(); + l2.assert_finished(); + } + + fn derived_attributes(is_last_in_span: bool) -> OpAttributesWithParent { + let mut attributes = TestAttributesBuilder::new() + .with_parent(test_block_info(9)) + .with_transactions(vec![l1_info_deposit().encoded_2718().into()]) + .build(); + attributes.is_last_in_span = is_last_in_span; + attributes + } + + /// The unsafe block that `attributes` derive, so consolidation finds it consistent. + fn unsafe_block_matching(attributes: &OpAttributesWithParent) -> RpcBlock { + let payload_attributes = &attributes.attributes.payload_attributes; + let header = alloy_consensus::Header { + beneficiary: payload_attributes.suggested_fee_recipient, + gas_limit: attributes.attributes.gas_limit.unwrap(), + mix_hash: payload_attributes.prev_randao, + number: attributes.block_number(), + parent_beacon_block_root: payload_attributes.parent_beacon_block_root, + parent_hash: attributes.parent.block_info.hash, + timestamp: payload_attributes.timestamp, + ..Default::default() + }; + let deposit = Transaction::from_transaction( + Recovered::new_unchecked(l1_info_deposit(), Address::ZERO), + Default::default(), + ); + RpcBlock::new(RpcHeader::new(header), BlockTransactions::Full(vec![deposit])) + } + + /// Derived blocks inside a span batch only move the safe label; the span's last block sends + /// the forkchoice update that delivers it. + #[rstest::rstest] + #[case::intermediate_block_defers_forkchoice(false, 0)] + #[case::last_block_sends_forkchoice(true, 1)] + #[tokio::test] + async fn span_batch_sends_forkchoice_only_at_its_last_block( + #[case] is_last_in_span: bool, + #[case] forkchoice_updates: usize, + ) { + let cfg = Arc::new(RollupConfig::default()); + let attributes = derived_attributes(is_last_in_span); + let block = unsafe_block_matching(&attributes); + let derived_safe_head = block_info_of(&cfg, &block); + let mut state = engine_state_with_safe_head(attributes.parent); + let expected_sync_state = with_safe_head(&state, derived_safe_head); + let (client, l1, l2) = test_engine_client(cfg.clone()); + l2.expect_params("eth_getBlockByNumber", serde_json::json!(["0xa", true]), block); + for _ in 0..forkchoice_updates { + l2.expect_forkchoice_updated( + expected_sync_state.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + } + let task = ConsolidateTask::new( + Arc::new(client), + cfg, + ConsolidateInput::Attributes(Box::new(attributes)), + Arc::new(NoopBlockSink), + ); + + task.execute(&mut state).await.unwrap(); + + assert_eq!(state.sync_state, expected_sync_state); + l1.assert_finished(); + l2.assert_finished(); + } } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs index 3461d7d860f..d00bb74b624 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs @@ -2,7 +2,8 @@ use crate::{ EngineClient, EngineState, ExecutableEngineTask, FinalizeBlockId, FinalizeTaskError, - state::EngineSyncStateUpdate, task_queue::synchronize, + state::EngineSyncStateUpdate, + task_queue::{ForkchoiceDelivery, synchronize}, }; use alloy_eips::BlockId; use async_trait::async_trait; @@ -71,6 +72,7 @@ impl ExecutableEngineTask for FinalizeTask { self.client.as_ref(), state, EngineSyncStateUpdate { finalized_head: Some(block_info), ..Default::default() }, + ForkchoiceDelivery::WhenChanged, ) .await?; diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/insert/task.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/insert/task.rs index 54d2d380709..10110e3b622 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/insert/task.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/insert/task.rs @@ -2,7 +2,8 @@ use crate::{ EngineClient, EngineState, ExecutableEngineTask, ImportedBlockSink, InsertTaskError, - state::EngineSyncStateUpdate, task_queue::synchronize, + state::EngineSyncStateUpdate, + task_queue::{ForkchoiceDelivery, synchronize}, }; use alloy_rpc_types_engine::{ExecutionPayloadInputV2, PayloadStatusEnum}; use async_trait::async_trait; @@ -125,6 +126,7 @@ pub(in crate::task_queue) async fn insert_payload( safe_head: is_payload_safe.then_some(new_unsafe_ref), ..Default::default() }, + ForkchoiceDelivery::WhenChanged, ) .await?; diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/mod.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/mod.rs index 224c7a9e34f..c5aa7a8ad47 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/mod.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/mod.rs @@ -7,7 +7,7 @@ pub use task::{ mod synchronize; pub use synchronize::SynchronizeTaskError; -pub(super) use synchronize::synchronize; +pub(super) use synchronize::{ForkchoiceDelivery, synchronize}; mod insert; pub(super) use insert::insert_payload; diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs index abae6d99464..219416bd6f1 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs @@ -4,6 +4,18 @@ use crate::{EngineClient, EngineState, SynchronizeTaskError, state::EngineSyncSt use alloy_rpc_types_engine::{INVALID_FORK_CHOICE_STATE_ERROR, PayloadStatusEnum}; use op_alloy_provider::ext::engine::OpEngineApi; +/// Whether [`synchronize`] may skip a forkchoice update that leaves the consensus labels unchanged. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(in crate::task_queue) enum ForkchoiceDelivery { + /// Send the update even when it leaves the consensus labels unchanged. + /// + /// Unchanged labels do not prove that the execution layer holds their forkchoice: a `SYNCING` + /// answer moves the labels too, and an execution layer that restarts can lose its forkchoice. + Always, + /// Skip the update when it leaves the consensus labels unchanged. + WhenChanged, +} + /// Applies `state_update` to the engine's sync state and synchronizes the execution layer's /// forkchoice with it, using an `engine_forkchoiceUpdated` call without payload attributes. /// @@ -15,26 +27,31 @@ use op_alloy_provider::ext::engine::OpEngineApi; target = "engine", level = "info", skip_all, - fields(?state_update, forkchoice = tracing::field::Empty) + fields(?delivery, ?state_update, forkchoice = tracing::field::Empty) )] pub(in crate::task_queue) async fn synchronize( client: &EngineClient, state: &mut EngineState, state_update: EngineSyncStateUpdate, + delivery: ForkchoiceDelivery, ) -> Result<(), SynchronizeTaskError> { // Apply the sync state update to the engine state. let new_sync_state = state.sync_state.apply_update(state_update); // Check if a forkchoice update is not needed, return early. // A forkchoice update is not needed if... - // 1. The engine state is not default (initial forkchoice state has been emitted), and - // 2. The new sync state is the same as the current sync state (no changes to the sync state). + // 1. The caller allows skipping an unchanged forkchoice, + // 2. The engine state is not default (initial forkchoice state has been emitted), and + // 3. The new sync state is the same as the current sync state (no changes to the sync state). // // NOTE: // We shouldn't retry the synchronization here. Since the `sync_state` is only updated inside // `synchronize` (except inside the ConsolidateTask, when the block is not the last in the // batch) - the engine would get stuck retrying the synchronization. - if state.sync_state != Default::default() && state.sync_state == new_sync_state { + if delivery == ForkchoiceDelivery::WhenChanged && + state.sync_state != Default::default() && + state.sync_state == new_sync_state + { debug!(target: "engine", ?new_sync_state, "No forkchoice update needed"); return Ok(()); } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/mod.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/mod.rs index 67c378103a7..e73923b3228 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/mod.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/mod.rs @@ -1,7 +1,10 @@ //! The forkchoice engine update shared by the engine tasks, and its error type. mod helper; -pub(in crate::task_queue) use helper::synchronize; +pub(in crate::task_queue) use helper::{ForkchoiceDelivery, synchronize}; mod error; pub use error::SynchronizeTaskError; + +#[cfg(test)] +mod tests; diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/tests.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/tests.rs new file mode 100644 index 00000000000..1ebc3a56248 --- /dev/null +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/tests.rs @@ -0,0 +1,134 @@ +use super::{ForkchoiceDelivery, synchronize}; +use crate::{ + EngineState, EngineSyncState, EngineSyncStateUpdate, EngineTaskError, + EngineTaskErrorSeverity::{self, Reset, Temporary}, + test_utils::{RpcMock, TestEngineStateBuilder, test_block_info, test_engine_client}, +}; +use alloy_rpc_types_engine::{ForkchoiceState, INVALID_FORK_CHOICE_STATE_ERROR, PayloadStatusEnum}; +use kona_genesis::RollupConfig; +use rstest::rstest; +use std::sync::Arc; + +const FORKCHOICE_UPDATED: &str = "engine_forkchoiceUpdatedV3"; + +/// How the execution layer answers the forkchoice update. +#[derive(Clone, Copy, Debug)] +enum Reply { + Invalid, + InvalidForkchoiceState, + Syncing, + TransportError, + Valid, +} + +/// Which labels the engine holds after the forkchoice update. +#[derive(Clone, Copy, Debug)] +enum Labels { + Kept, + Moved, +} + +/// An engine state that has not seen a `VALID` answer yet. +fn engine_state() -> EngineState { + TestEngineStateBuilder::new() + .with_unsafe_head(test_block_info(20)) + .with_safe_head(test_block_info(10)) + .with_finalized_head(test_block_info(5)) + .with_el_sync_finished(false) + .build() +} + +/// An update that moves the safe head, or one that leaves every label unchanged. +fn state_update(moves_safe_head: bool) -> EngineSyncStateUpdate { + EngineSyncStateUpdate { + safe_head: moves_safe_head.then(|| test_block_info(15)), + ..Default::default() + } +} + +const fn labels_after( + labels: Labels, + before: EngineSyncState, + moved: EngineSyncState, +) -> EngineSyncState { + match labels { + Labels::Kept => before, + Labels::Moved => moved, + } +} + +fn expect_reply(l2: &RpcMock, forkchoice: ForkchoiceState, reply: Reply) { + match reply { + Reply::Invalid => l2.expect_forkchoice_updated( + forkchoice, + PayloadStatusEnum::Invalid { validation_error: "invalid".into() }, + ), + Reply::InvalidForkchoiceState => { + l2.expect_error_code(FORKCHOICE_UPDATED, INVALID_FORK_CHOICE_STATE_ERROR as i64) + } + Reply::Syncing => l2.expect_forkchoice_updated(forkchoice, PayloadStatusEnum::Syncing), + Reply::TransportError => l2.expect_transport_error(FORKCHOICE_UPDATED), + Reply::Valid => l2.expect_forkchoice_updated(forkchoice, PayloadStatusEnum::Valid), + } +} + +/// An update that leaves the labels unchanged is skipped only when the caller allows it. +#[rstest] +#[case::changed_labels_are_sent(true, ForkchoiceDelivery::WhenChanged, 1)] +#[case::unchanged_labels_are_skipped(false, ForkchoiceDelivery::WhenChanged, 0)] +#[case::changed_labels_are_sent_always(true, ForkchoiceDelivery::Always, 1)] +#[case::unchanged_labels_are_sent_always(false, ForkchoiceDelivery::Always, 1)] +#[tokio::test] +async fn synchronize_skips_unchanged_labels_only_when_allowed( + #[case] moves_safe_head: bool, + #[case] delivery: ForkchoiceDelivery, + #[case] forkchoice_updates: usize, +) { + let mut state = engine_state(); + let update = state_update(moves_safe_head); + let expected_sync_state = state.sync_state.apply_update(update); + let (client, l1, l2) = test_engine_client(Arc::new(RollupConfig::default())); + for _ in 0..forkchoice_updates { + l2.expect_forkchoice_updated( + expected_sync_state.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + } + + synchronize(&client, &mut state, update, delivery).await.unwrap(); + + assert_eq!(state.sync_state, expected_sync_state); + l1.assert_finished(); + l2.assert_finished(); +} + +/// Only a `VALID` or `SYNCING` answer moves the labels, and only `VALID` marks the execution layer +/// as synced. Failures leave the engine state untouched. +#[rstest] +#[case::valid(Reply::Valid, Labels::Moved, true, None)] +#[case::syncing(Reply::Syncing, Labels::Moved, false, None)] +#[case::invalid_payload(Reply::Invalid, Labels::Kept, false, Some(Reset))] +#[case::invalid_forkchoice_state(Reply::InvalidForkchoiceState, Labels::Kept, false, Some(Reset))] +#[case::transport_error(Reply::TransportError, Labels::Kept, false, Some(Temporary))] +#[tokio::test] +async fn synchronize_applies_labels_only_when_accepted( + #[case] reply: Reply, + #[case] labels: Labels, + #[case] el_sync_finished: bool, + #[case] error_severity: Option, +) { + let mut state = engine_state(); + let update = state_update(true); + let moved_sync_state = state.sync_state.apply_update(update); + let expected_sync_state = labels_after(labels, state.sync_state, moved_sync_state); + let (client, l1, l2) = test_engine_client(Arc::new(RollupConfig::default())); + expect_reply(&l2, moved_sync_state.create_forkchoice_state(), reply); + + let result = synchronize(&client, &mut state, update, ForkchoiceDelivery::WhenChanged).await; + + assert_eq!(result.as_ref().err().map(EngineTaskError::severity), error_severity, "{result:?}"); + assert_eq!(state.sync_state, expected_sync_state); + assert_eq!(state.el_sync_finished, el_sync_finished); + l1.assert_finished(); + l2.assert_finished(); +} diff --git a/rust/kona/crates/node/engine/src/test_utils/deposits.rs b/rust/kona/crates/node/engine/src/test_utils/deposits.rs new file mode 100644 index 00000000000..78d60d4bf01 --- /dev/null +++ b/rust/kona/crates/node/engine/src/test_utils/deposits.rs @@ -0,0 +1,14 @@ +//! Deposit transaction fixtures. + +use alloy_primitives::Sealed; +use kona_protocol::L1BlockInfoBedrock; +use op_alloy_consensus::{OpTxEnvelope, TxDeposit}; + +/// The L1-info deposit a block needs for `L2BlockInfo::from_block_and_genesis` to read its L1 +/// origin. +pub fn l1_info_deposit() -> OpTxEnvelope { + OpTxEnvelope::Deposit(Sealed::new(TxDeposit { + input: L1BlockInfoBedrock::default().encode_calldata(), + ..Default::default() + })) +} diff --git a/rust/kona/crates/node/engine/src/test_utils/mod.rs b/rust/kona/crates/node/engine/src/test_utils/mod.rs index a5575963a67..e5949f23225 100644 --- a/rust/kona/crates/node/engine/src/test_utils/mod.rs +++ b/rust/kona/crates/node/engine/src/test_utils/mod.rs @@ -1,6 +1,9 @@ mod attributes; pub use attributes::TestAttributesBuilder; +mod deposits; +pub use deposits::l1_info_deposit; + mod rpc; pub use rpc::{RpcMock, test_engine_client}; diff --git a/rust/kona/crates/node/engine/src/test_utils/rpc.rs b/rust/kona/crates/node/engine/src/test_utils/rpc.rs index be93a9816fe..1cfc3995356 100644 --- a/rust/kona/crates/node/engine/src/test_utils/rpc.rs +++ b/rust/kona/crates/node/engine/src/test_utils/rpc.rs @@ -5,6 +5,9 @@ use alloy_json_rpc::{ErrorPayload, RequestPacket, ResponsePacket}; use alloy_network::Network; use alloy_provider::RootProvider; use alloy_rpc_client::RpcClient; +use alloy_rpc_types_engine::{ + ForkchoiceState, ForkchoiceUpdated, PayloadStatus, PayloadStatusEnum, +}; use alloy_transport::{ TransportError, TransportErrorKind, TransportFut, mock::{Asserter, MockResponse, MockTransport}, @@ -56,6 +59,20 @@ impl RpcMock { ); } + /// Expects an `engine_forkchoiceUpdatedV3` call for `forkchoice` without payload attributes and + /// answers it with `status`. + pub fn expect_forkchoice_updated( + &self, + forkchoice: ForkchoiceState, + status: PayloadStatusEnum, + ) { + self.expect_params( + "engine_forkchoiceUpdatedV3", + serde_json::json!([forkchoice, null]), + ForkchoiceUpdated::new(PayloadStatus::from_status(status)), + ); + } + /// Expects a call and delays its response, for recovery tests using Tokio's paused clock. pub fn expect_with_delay( &self, diff --git a/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs b/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs index 49e30d84f94..3dc21d249e1 100644 --- a/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs +++ b/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs @@ -1,15 +1,13 @@ use super::*; use crate::QueuedEngineDerivationClient; -use alloy_primitives::Sealed; use alloy_rpc_types_engine::{ ExecutionPayloadV1, ForkchoiceUpdated, PayloadStatus, PayloadStatusEnum, }; use kona_engine::{ - NoopBlockSink, - test_utils::{TestEngineStateBuilder, test_block_info, test_engine_client}, + ConsolidateInput, EngineSyncStateUpdate, NoopBlockSink, + test_utils::{TestEngineStateBuilder, l1_info_deposit, test_block_info, test_engine_client}, }; -use kona_protocol::L1BlockInfoBedrock; -use op_alloy_consensus::{OpBlock, OpTxEnvelope, TxDeposit}; +use op_alloy_consensus::OpBlock; use op_alloy_rpc_types_engine::OpExecutionPayloadEnvelope; /// Exercise the real actor request/drain loop with forty consecutive unsafe blocks, without @@ -39,10 +37,7 @@ async fn healthy_unsafe_imports_keep_engine_queue_bounded() { block.header.number = number; block.header.timestamp = number * 2; block.header.parent_hash = expected_heads.last().unwrap().hash(); - block.body.transactions.push(OpTxEnvelope::Deposit(Sealed::new(TxDeposit { - input: L1BlockInfoBedrock::default().encode_calldata(), - ..Default::default() - }))); + block.body.transactions.push(l1_info_deposit()); let payload = ExecutionPayloadV1::from_block_slow(&block); let imported: OpBlock = OpExecutionPayloadEnvelope::V1(payload.clone()).try_into_block().unwrap(); @@ -87,3 +82,61 @@ async fn healthy_unsafe_imports_keep_engine_queue_bounded() { l1.assert_finished(); l2.assert_finished(); } + +/// While the execution layer catches up to a delegated safe head ahead of the local unsafe head, +/// each delegate poll repeats the same target. Every poll sends it, after a `SYNCING` answer and +/// after a `VALID` one, since an execution layer that restarts can lose an applied forkchoice. +#[tokio::test(start_paused = true)] +async fn delegated_safe_head_is_sent_on_every_poll() { + let config = Arc::new(RollupConfig::default()); + let (client, l1, l2) = test_engine_client(config.clone()); + let state = TestEngineStateBuilder::new() + .with_unsafe_head(test_block_info(20)) + .with_safe_head(test_block_info(10)) + .with_finalized_head(test_block_info(5)) + .build(); + let delegated_safe_head = test_block_info(30); + let target = state + .sync_state + .apply_update(EngineSyncStateUpdate { + local_safe_head: Some(delegated_safe_head), + safe_head: Some(delegated_safe_head), + unsafe_head: Some(delegated_safe_head), + ..Default::default() + }) + .create_forkchoice_state(); + for status in [PayloadStatusEnum::Syncing, PayloadStatusEnum::Valid, PayloadStatusEnum::Valid] { + l2.expect_forkchoice_updated(target, status); + } + let (state_tx, state_rx) = watch::channel(state); + let (requests, request_rx) = mpsc::channel(8); + let (derivation_tx, _derivation_rx) = mpsc::channel(8); + let mut actor = EngineActor::new( + Arc::new(client), + config, + QueuedEngineDerivationClient { derivation_actor_request_tx: derivation_tx }, + Engine::new(state, state_tx), + None, + request_rx, + Arc::new(NoopBlockSink), + ); + + // Three polls report the same target. Each step drains the previous poll's task and then + // accepts the next poll. + for _ in 0..3 { + requests + .send(EngineActorRequest::ProcessSafeL2Signal(ConsolidateInput::BlockInfo( + delegated_safe_head, + ))) + .await + .unwrap(); + time::timeout(Duration::from_secs(1), actor.step()).await.unwrap().unwrap(); + } + drop(requests); + assert!(matches!(actor.step().await, Err(EngineError::ChannelClosed))); + + assert!(actor.engine.is_empty()); + assert_eq!(state_rx.borrow().sync_state.safe_head(), delegated_safe_head); + l1.assert_finished(); + l2.assert_finished(); +} From 3ca85833f58d2b22adcea5be86104d3f386c6f07 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Fri, 6 Nov 2026 20:36:49 -0800 Subject: [PATCH 2/7] kona-engine: prevent delegated finalized-head regression A lagging delegate could move finality backward, and a delegate on another fork could name a finalized block that is not the execution layer's canonical block. The first is irreversible. The second either stopped the node or failed every forkchoice update and reset the engine on each poll. The finalize task now looks the target up by number and finalizes a pinned hash only when it is the canonical block at that height. A target behind the finalized head, a different block at the finalized height, or a pinned hash that is not canonical fails with an error of the new `Skip` severity: the engine drops the task, counts it as a failure, and the engine actor warns once per distinct error until a reset. A target equal to the finalized head needs no lookup or forkchoice update, and each finality guard runs once per kind of target. A block served at another height is a distinct `BlockNumberMismatch`. The delegate actor also rejects a whole sync status whose finalized L2 block is behind, or conflicts with, the engine's, so a lagging or diverged delegate's safe head does not reach the engine either. The baseline is the finalized head the engine applied, read from the state it publishes, not the last target forwarded to it: a target the engine skipped must not make the canonical block at its height look like a conflict, which would stop the delegate's safe head too. BREAKING CHANGE: `EngineTaskErrorSeverity` has a new `Skip` variant, and `FinalizeTaskError` has new variants `BehindFinalizedHead`, `BlockNumberMismatch`, `ConflictsWithFinalizedHead` and `NotCanonical`; exhaustive matches must handle them. A block served at another height is now `BlockNumberMismatch` instead of `BlockNotFound`. `DerivationActorRequest::ProcessEngineSyncCompletionRequest` carries an `EngineSyncState`, and `EngineDerivationClient::notify_sync_completed` takes one. `DelegateDerivationActor::new` takes the engine's state receiver. Co-Authored-By: Claude Opus 5.5 Co-Authored-By: Claude Fable 5.1 --- .../crates/node/engine/src/task_queue/core.rs | 39 ++- .../src/task_queue/tasks/finalize/error.rs | 57 +++- .../src/task_queue/tasks/finalize/id.rs | 10 +- .../src/task_queue/tasks/finalize/task.rs | 134 ++++++-- .../task_queue/tasks/finalize/task_test.rs | 289 ++++++++++++++---- .../node/engine/src/task_queue/tasks/task.rs | 4 + .../node/engine/src/test_utils/blocks.rs | 40 +++ .../crates/node/engine/src/test_utils/mod.rs | 3 + .../crates/node/engine/src/test_utils/rpc.rs | 12 +- .../src/actors/derivation/actor/core.rs | 3 +- .../src/actors/derivation/actor/tests.rs | 29 +- .../actors/derivation/actor/tests/database.rs | 2 +- .../derivation/actor/tests/op_node_parity.rs | 2 +- .../src/actors/derivation/delegated/actor.rs | 60 +++- .../src/actors/derivation/delegated/mod.rs | 3 + .../src/actors/derivation/delegated/tests.rs | 250 +++++++++++++++ .../service/src/actors/derivation/request.rs | 7 +- .../node/service/src/actors/engine/actor.rs | 24 +- .../node/service/src/actors/engine/client.rs | 16 +- .../node/service/src/actors/engine/tests.rs | 2 +- .../node/service/src/service/node/rollup.rs | 3 + 21 files changed, 855 insertions(+), 134 deletions(-) create mode 100644 rust/kona/crates/node/engine/src/test_utils/blocks.rs create mode 100644 rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs diff --git a/rust/kona/crates/node/engine/src/task_queue/core.rs b/rust/kona/crates/node/engine/src/task_queue/core.rs index acc3fa27d40..e9d9dcb4f1c 100644 --- a/rust/kona/crates/node/engine/src/task_queue/core.rs +++ b/rust/kona/crates/node/engine/src/task_queue/core.rs @@ -145,7 +145,7 @@ impl Engine { /// Attempts to drain the queue by executing all [`EngineTask`]s in-order. If any task returns /// an error along the way, the error is returned and the task is not popped from the queue (in - /// case it must be retried), unless it is a flush error. + /// case it must be retried), unless it is a flush or skip error. pub async fn drain(&mut self) -> Result<(), EngineTaskErrors> { // Drain tasks in order of priority, halting on errors for a retry to be attempted. loop { @@ -156,9 +156,13 @@ impl Engine { let result = task.execute(&mut self.state).await; // A flush reports that the task replaced its block with a deposits-only block, so the - // task is done; retrying it would replace that block again. + // task is done; retrying it would replace that block again. A skip reports a task that + // no longer applies, so retrying it cannot succeed either. if let Err(err) = &result && - err.severity() != EngineTaskErrorSeverity::Flush + !matches!( + err.severity(), + EngineTaskErrorSeverity::Flush | EngineTaskErrorSeverity::Skip + ) { return result; } @@ -198,7 +202,8 @@ impl EngineResetError { mod recovery_tests { use super::*; use crate::{ - ConsolidateInput, ConsolidateTask, InsertTask, NoopBlockSink, + ConsolidateInput, ConsolidateTask, FinalizeBlockId, FinalizeTask, InsertTask, + NoopBlockSink, test_utils::{ TestAttributesBuilder, TestEngineStateBuilder, test_block_info, test_engine_client, }, @@ -381,4 +386,30 @@ mod recovery_tests { l1.assert_finished(); l2.assert_finished(); } + + // A skipped task is dropped instead of being kept for a retry, so it cannot hold the queue. + #[tokio::test] + async fn skipped_task_is_dropped_from_the_queue() { + let config = Arc::new(RollupConfig::default()); + let (client, l1, l2) = test_engine_client(config.clone()); + let state = TestEngineStateBuilder::new() + .with_unsafe_head(test_block_info(20)) + .with_finalized_head(test_block_info(10)) + .build(); + let (state_tx, _state_rx) = tokio::sync::watch::channel(state); + let mut engine = Engine::new(state, state_tx); + engine.enqueue(EngineTask::Finalize(Box::new(FinalizeTask::new( + Arc::new(client), + config, + FinalizeBlockId::ByNumber(5), + )))); + + let error = engine.drain().await.unwrap_err(); + + assert_eq!(error.severity(), EngineTaskErrorSeverity::Skip, "got {error:?}"); + assert!(engine.is_empty()); + assert_eq!(engine.state().sync_state, state.sync_state); + l1.assert_finished(); + l2.assert_finished(); + } } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/error.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/error.rs index c8444c2595e..51a7f2c1dda 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/error.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/error.rs @@ -3,6 +3,7 @@ use crate::{ EngineTaskError, SynchronizeTaskError, task_queue::tasks::task::EngineTaskErrorSeverity, }; +use alloy_eips::BlockNumHash; use alloy_transport::{RpcError, TransportErrorKind}; use kona_protocol::FromBlockError; use thiserror::Error; @@ -10,33 +11,69 @@ use thiserror::Error; /// An error that occurs when running the [`crate::FinalizeTask`]. #[derive(Debug, Error)] pub enum FinalizeTaskError { - /// The block is not safe, and therefore cannot be finalized. - #[error("Attempted to finalize a block that is not yet safe")] - BlockNotSafe, + /// The target is below the finalized head, and finality never moves backward. + #[error("Finalization target {target} is behind the finalized head {finalized}")] + BehindFinalizedHead { + /// The number of the finalized head. + finalized: u64, + /// The number of the finalization target. + target: u64, + }, /// The block to finalize was not found. #[error("The block to finalize was not found: Number {0}")] BlockNotFound(u64), + /// The block is not safe, and therefore cannot be finalized. + #[error("Attempted to finalize a block that is not yet safe")] + BlockNotSafe, + /// The engine returned a block at a different height than the one requested. + #[error("Requested block {requested} to finalize, but the engine returned block {returned}")] + BlockNumberMismatch { + /// The requested block number. + requested: u64, + /// The number of the block the engine returned. + returned: u64, + }, + /// The target is a different block at the finalized head's height. + #[error("Finalization target {target:?} conflicts with the finalized head {finalized:?}")] + ConflictsWithFinalizedHead { + /// The finalized head. + finalized: BlockNumHash, + /// The finalization target. + target: BlockNumHash, + }, + /// The forkchoice update call to finalize the block failed. + #[error(transparent)] + ForkchoiceUpdateFailed(#[from] SynchronizeTaskError), /// An error occurred while transforming the RPC block into [`L2BlockInfo`]. /// /// [`L2BlockInfo`]: kona_protocol::L2BlockInfo #[error(transparent)] FromBlock(#[from] FromBlockError), + /// The pinned target is not the engine's canonical block at its height. + #[error("Finalization target {target:?} is not canonical; the engine has {canonical:?}")] + NotCanonical { + /// The engine's canonical block at the target's height. + canonical: BlockNumHash, + /// The finalization target. + target: BlockNumHash, + }, /// A temporary RPC failure. #[error(transparent)] TransportError(#[from] RpcError), - /// The forkchoice update call to finalize the block failed. - #[error(transparent)] - ForkchoiceUpdateFailed(#[from] SynchronizeTaskError), } impl EngineTaskError for FinalizeTaskError { fn severity(&self) -> EngineTaskErrorSeverity { match self { - Self::BlockNotSafe | Self::BlockNotFound(_) | Self::FromBlock(_) => { - EngineTaskErrorSeverity::Critical - } - Self::TransportError(_) => EngineTaskErrorSeverity::Temporary, + Self::BehindFinalizedHead { .. } | + Self::ConflictsWithFinalizedHead { .. } | + Self::NotCanonical { .. } => EngineTaskErrorSeverity::Skip, + Self::BlockNotFound(_) | + Self::BlockNotSafe | + Self::BlockNumberMismatch { .. } | + Self::FromBlock(_) => EngineTaskErrorSeverity::Critical, Self::ForkchoiceUpdateFailed(inner) => inner.severity(), + Self::TransportError(_) => EngineTaskErrorSeverity::Temporary, } } } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/id.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/id.rs index ab5a92e4ddb..c7c12fd6e73 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/id.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/id.rs @@ -10,9 +10,9 @@ use alloy_eips::BlockNumHash; /// authoritative source for the block at a given height (e.g. local L1-finality driven by /// `L2Finalizer`). The number is sufficient because there is only one chain to consult. /// - [`FinalizeBlockId::ByHash`] is used when an upstream supplies `(number, hash)` and the engine -/// must finalize the specific block identified by hash. If the engine does not have that hash, -/// the task must error rather than silently finalize the engine's canonical block at the same -/// height. +/// must finalize the specific block identified by hash. If the engine's canonical block at that +/// height has a different hash, the task skips the target rather than finalize the engine's block +/// in its place. /// /// EL finalization is irreversible, so finalizing the wrong block is unrecoverable. Forcing every /// caller to pick a variant prevents accidental hash-loss at the boundary. @@ -20,8 +20,8 @@ use alloy_eips::BlockNumHash; pub enum FinalizeBlockId { /// Finalize whatever block the engine has at the given L2 block number. ByNumber(u64), - /// Finalize the block matching the given `(number, hash)` pair. The task fails if the engine - /// does not have that hash. + /// Finalize the block matching the given `(number, hash)` pair, only once it is the engine's + /// canonical block at that height. ByHash(BlockNumHash), } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs index d00bb74b624..a01d43c1608 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs @@ -5,7 +5,7 @@ use crate::{ state::EngineSyncStateUpdate, task_queue::{ForkchoiceDelivery, synchronize}, }; -use alloy_eips::BlockId; +use alloy_eips::BlockNumHash; use async_trait::async_trait; use derive_more::Constructor; use kona_genesis::RollupConfig; @@ -14,6 +14,10 @@ use std::sync::Arc; /// The [`FinalizeTask`] fetches the [`L2BlockInfo`] identified by `block_id`, updates the /// [`EngineState`], and dispatches a forkchoice update to finalize the block. +/// +/// Finality only moves forward. A target behind the finalized head, a different block at the +/// finalized height, or a pinned hash that is not the engine's canonical block fails with an error +/// that skips the task and leaves the engine state unchanged. #[derive(Debug, Clone, Constructor)] pub struct FinalizeTask { /// The engine client. @@ -24,6 +28,88 @@ pub struct FinalizeTask { pub block_id: FinalizeBlockId, } +impl FinalizeTask { + /// Rejects a target that would move finality backward. + const fn ensure_not_behind_finalized_head( + finalized_head: &L2BlockInfo, + target_number: u64, + ) -> Result<(), FinalizeTaskError> { + if target_number < finalized_head.block_info.number { + return Err(FinalizeTaskError::BehindFinalizedHead { + finalized: finalized_head.block_info.number, + target: target_number, + }); + } + Ok(()) + } + + /// Rejects a different block at the finalized head's height. + /// + /// The default head means finality has not been established yet, so any target, including the + /// configured genesis, may initialize it. + fn ensure_no_conflict_with_finalized_head( + finalized_head: &L2BlockInfo, + target: BlockNumHash, + ) -> Result<(), FinalizeTaskError> { + let finalized = finalized_head.block_info.id(); + if *finalized_head != L2BlockInfo::default() && + target.number == finalized.number && + target.hash != finalized.hash + { + return Err(FinalizeTaskError::ConflictsWithFinalizedHead { finalized, target }); + } + Ok(()) + } + + /// Whether `target` is the finalized head itself; a different block at its height is an error. + fn is_already_finalized( + finalized_head: &L2BlockInfo, + target: BlockNumHash, + ) -> Result { + if target == finalized_head.block_info.id() { + debug!(target: "engine", ?target, "Target is already finalized"); + return Ok(true); + } + Self::ensure_no_conflict_with_finalized_head(finalized_head, target)?; + Ok(false) + } + + /// Fetches the engine's canonical block at the target's height. + /// + /// A pinned hash must be that block: a forkchoice update cannot finalize a block off the + /// canonical chain, and finalizing the canonical block instead would finalize a block the + /// caller never named. The block's identity is checked before it is decoded, so a mismatch is + /// reported as such rather than as a decoding failure. + async fn fetch_canonical_block_info(&self) -> Result { + let requested = self.block_id.number(); + debug!(target: "engine", block_id = ?self.block_id, "Fetching the block to finalize"); + let block = self + .client + .get_l2_block(requested.into()) + .full() + .await + .map_err(FinalizeTaskError::TransportError)? + .ok_or(FinalizeTaskError::BlockNotFound(requested))?; + let canonical = BlockNumHash { hash: block.header.hash, number: block.header.number }; + debug!(target: "engine", ?canonical, "Fetched the block to finalize"); + + if canonical.number != requested { + return Err(FinalizeTaskError::BlockNumberMismatch { + requested, + returned: canonical.number, + }); + } + if let FinalizeBlockId::ByHash(target) = self.block_id && + target.hash != canonical.hash + { + return Err(FinalizeTaskError::NotCanonical { canonical, target }); + } + + L2BlockInfo::from_block_and_genesis(&block.into_consensus(), &self.cfg.genesis) + .map_err(FinalizeTaskError::FromBlock) + } +} + #[async_trait] impl ExecutableEngineTask for FinalizeTask { type Output = (); @@ -31,40 +117,30 @@ impl ExecutableEngineTask for FinalizeTask { type Error = FinalizeTaskError; async fn execute(&self, state: &mut EngineState) -> Result<(), FinalizeTaskError> { - let block_number = self.block_id.number(); + let finalized_head = state.sync_state.finalized_head(); + let target_number = self.block_id.number(); + + // A lagging source can report an older finalized block, or one from another chain. A + // pinned hash is enough to detect either without asking the execution layer. + Self::ensure_not_behind_finalized_head(&finalized_head, target_number)?; + if let FinalizeBlockId::ByHash(target) = self.block_id && + Self::is_already_finalized(&finalized_head, target)? + { + return Ok(()); + } // Sanity check that the block that is being finalized is at least safe. - if state.sync_state.safe_head().block_info.number < block_number { + if state.sync_state.safe_head().block_info.number < target_number { return Err(FinalizeTaskError::BlockNotSafe); } - // Look up by hash when the caller pinned a specific hash (delegated polling supplies - // `(number, hash)`); otherwise look up by number. Finalization is irreversible, so a stale - // hash must produce a hard error, not silently finalize whatever the engine has at the - // same height. - let lookup: BlockId = match self.block_id { - FinalizeBlockId::ByHash(id) => id.hash.into(), - FinalizeBlockId::ByNumber(n) => n.into(), - }; - let block = self - .client - .get_l2_block(lookup) - .full() - .await - .map_err(FinalizeTaskError::TransportError)? - .ok_or(FinalizeTaskError::BlockNotFound(block_number))? - .into_consensus(); - let block_info = L2BlockInfo::from_block_and_genesis(&block, &self.client.cfg().genesis) - .map_err(FinalizeTaskError::FromBlock)?; - - // For ByHash, also assert the returned block's height matches the caller's claim. The - // engine should never serve a block at a different height for a given hash, but a height - // mismatch indicates either an upstream protocol bug or a misuse of the API and must - // never lead to silent finalization. - if let FinalizeBlockId::ByHash(id) = self.block_id && - block_info.block_info.number != id.number + let block_info = self.fetch_canonical_block_info().await?; + + // A by-number target only learns its hash from the lookup. + if matches!(self.block_id, FinalizeBlockId::ByNumber(_)) && + Self::is_already_finalized(&finalized_head, block_info.block_info.id())? { - return Err(FinalizeTaskError::BlockNotFound(id.number)); + return Ok(()); } // Dispatch a forkchoice update. diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs index 08c4f153e37..fc2243dcb6c 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs @@ -1,62 +1,249 @@ use crate::{ + EngineState, EngineSyncState, EngineSyncStateUpdate, EngineTaskError, EngineTaskErrorSeverity, ExecutableEngineTask, FinalizeBlockId, FinalizeTask, FinalizeTaskError, - test_utils::{TestEngineStateBuilder, test_engine_client}, + test_utils::{ + RpcMock, TestEngineStateBuilder, TestRpcBlock as RpcBlock, test_block_info, + test_engine_client, test_l2_block, test_rpc_block, + }, }; +use alloy_consensus::Header; use alloy_eips::BlockNumHash; -use alloy_primitives::b256; -use kona_genesis::RollupConfig; -use kona_protocol::{BlockInfo, L2BlockInfo}; +use alloy_primitives::B256; +use alloy_rpc_types_engine::PayloadStatusEnum; +use alloy_rpc_types_eth::{BlockTransactions, Header as RpcHeader}; +use kona_genesis::{ChainGenesis, RollupConfig}; +use kona_protocol::L2BlockInfo; +use rstest::rstest; use std::sync::Arc; -/// When the engine receives a `ByHash` finalize request for a block hash it doesn't have, -/// [`FinalizeTask`] must fail with [`FinalizeTaskError::BlockNotFound`] rather than silently -/// finalize whatever it happens to have at the same height. -/// -/// Assert the exact by-hash RPC lookup, so a by-number lookup cannot silently finalize a -/// different canonical block at the same height. -#[tokio::test] -async fn finalize_task_by_hash_errors_when_engine_lacks_hash() { - const N: u64 = 10; - let hash_a = b256!("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); - let hash_b = b256!("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"); +const STALE: u64 = 5; +const FINALIZED: u64 = 10; +const ADVANCED: u64 = 15; +const SAFE: u64 = 20; +const UNSAFE: u64 = 30; + +/// The L2 genesis block, which carries no L1-info deposit. +fn genesis_block() -> RpcBlock { + RpcBlock::new(RpcHeader::new(Header::default()), BlockTransactions::Full(vec![])) +} + +fn rollup_config() -> Arc { + let genesis = BlockNumHash { hash: genesis_block().header.hash, number: 0 }; + Arc::new(RollupConfig { + genesis: ChainGenesis { l2: genesis, ..Default::default() }, + ..Default::default() + }) +} + +/// An L2 block at `number`. Blocks built from different `fork` values share a height but not a +/// hash. +fn l2_block(number: u64, fork: u8) -> RpcBlock { + test_rpc_block(&test_l2_block(number, B256::ZERO, fork)) +} + +fn canonical(number: u64) -> RpcBlock { + l2_block(number, 0) +} + +fn conflicting(number: u64) -> RpcBlock { + l2_block(number, 1) +} + +fn l2_block_info(block: &RpcBlock) -> L2BlockInfo { + L2BlockInfo::from_block_and_genesis(&block.clone().into_consensus(), &rollup_config().genesis) + .unwrap() +} + +fn by_hash(block: &RpcBlock) -> FinalizeBlockId { + FinalizeBlockId::ByHash(l2_block_info(block).block_info.id()) +} - let cfg = Arc::new(RollupConfig::default()); +fn by_number(block: &RpcBlock) -> FinalizeBlockId { + FinalizeBlockId::ByNumber(block.header.number) +} + +fn established() -> L2BlockInfo { + l2_block_info(&canonical(FINALIZED)) +} + +fn uninitialized() -> L2BlockInfo { + L2BlockInfo::default() +} + +fn engine_state(finalized_head: L2BlockInfo) -> EngineState { + TestEngineStateBuilder::new() + .with_unsafe_head(test_block_info(UNSAFE)) + .with_safe_head(test_block_info(SAFE)) + .with_finalized_head(finalized_head) + .build() +} + +fn with_finalized_head(state: &EngineState, finalized_head: L2BlockInfo) -> EngineSyncState { + state.sync_state.apply_update(EngineSyncStateUpdate { + finalized_head: Some(finalized_head), + ..Default::default() + }) +} + +async fn finalize( + target: FinalizeBlockId, + state: &mut EngineState, + configure_engine: impl FnOnce(&RpcMock), +) -> Result<(), FinalizeTaskError> { + let cfg = rollup_config(); let (engine_client, l1, l2) = test_engine_client(cfg.clone()); - l2.expect_params( - "eth_getBlockByHash", - serde_json::json!([hash_b, true]), - Option::>::None, - ); - - // Place the safe head at N so the sanity check (`safe_head.number >= block_id.number`) passes - // and `execute()` reaches the lookup we care about. - let safe_head = L2BlockInfo { - block_info: BlockInfo { - number: N, - hash: hash_a, - parent_hash: Default::default(), - timestamp: N * 2, - }, - l1_origin: BlockNumHash::default(), - seq_num: 0, - }; - let mut state = - TestEngineStateBuilder::new().with_unsafe_head(safe_head).with_safe_head(safe_head).build(); - - let task = FinalizeTask::new( - Arc::new(engine_client), - cfg, - FinalizeBlockId::ByHash(BlockNumHash { number: N, hash: hash_b }), - ); - - let result = task.execute(&mut state).await; - - assert!( - matches!(result, Err(FinalizeTaskError::BlockNotFound(n)) if n == N), - "expected BlockNotFound({N}) — got {result:?}. The by-hash lookup must fail loudly when \ - the engine lacks the requested hash; instead, the task either succeeded (finalizing the \ - wrong block) or surfaced a different error." - ); + configure_engine(&l2); + + let result = FinalizeTask::new(Arc::new(engine_client), cfg, target).execute(state).await; + l1.assert_finished(); l2.assert_finished(); + result +} + +const fn is_behind(err: &FinalizeTaskError) -> bool { + matches!(err, FinalizeTaskError::BehindFinalizedHead { finalized: FINALIZED, target: STALE }) +} + +const fn is_conflict(err: &FinalizeTaskError) -> bool { + matches!(err, FinalizeTaskError::ConflictsWithFinalizedHead { .. }) +} + +const fn is_not_canonical(err: &FinalizeTaskError) -> bool { + matches!(err, FinalizeTaskError::NotCanonical { .. }) +} + +const fn is_number_mismatch(err: &FinalizeTaskError) -> bool { + matches!(err, FinalizeTaskError::BlockNumberMismatch { requested: ADVANCED, .. }) +} + +const fn is_block_not_found(err: &FinalizeTaskError) -> bool { + matches!(err, FinalizeTaskError::BlockNotFound(ADVANCED)) +} + +/// Targets behind the finalized head, or a pinned hash that conflicts with it, are skipped before +/// asking the execution layer for the block: the test engine expects no request at all. +#[rstest] +#[case::stale_by_hash(by_hash(&canonical(STALE)), is_behind)] +#[case::stale_by_number(by_number(&canonical(STALE)), is_behind)] +#[case::conflicting_by_hash(by_hash(&conflicting(FINALIZED)), is_conflict)] +#[tokio::test] +async fn finalize_task_skips_target_without_lookup( + #[case] target: FinalizeBlockId, + #[case] is_expected_error: fn(&FinalizeTaskError) -> bool, +) { + let mut state = engine_state(established()); + let original_state = state; + + let err = finalize(target, &mut state, |_| {}).await.unwrap_err(); + + assert!(is_expected_error(&err), "{err:?}"); + assert_eq!(err.severity(), EngineTaskErrorSeverity::Skip); + assert_eq!(state, original_state); +} + +/// A target whose block differs from the engine's canonical block is skipped. A pinned hash is +/// never replaced by the canonical block at its height, even above the finalized head, where a +/// delegate on another fork names a block the engine does not consider canonical. +#[rstest] +#[case::conflicting_by_number(by_number(&conflicting(FINALIZED)), conflicting(FINALIZED), is_conflict)] +#[case::other_fork_by_hash(by_hash(&conflicting(ADVANCED)), canonical(ADVANCED), is_not_canonical)] +#[tokio::test] +async fn finalize_task_skips_target_off_the_canonical_chain( + #[case] target: FinalizeBlockId, + #[case] served_block: RpcBlock, + #[case] is_expected_error: fn(&FinalizeTaskError) -> bool, +) { + let mut state = engine_state(established()); + let original_state = state; + + let err = finalize(target, &mut state, |l2| { + l2.expect_block_by_number(target.number(), Some(served_block)) + }) + .await + .unwrap_err(); + + assert!(is_expected_error(&err), "{err:?}"); + assert_eq!(err.severity(), EngineTaskErrorSeverity::Skip); + assert_eq!(state, original_state); +} + +/// A canonical target above the finalized head becomes the finalized head through one forkchoice +/// update. The canonical block at the finalized height is already finalized, so it needs no update. +#[rstest] +#[case::advancing_by_hash(established(), by_hash, canonical(ADVANCED), 1)] +#[case::advancing_by_number(established(), by_number, canonical(ADVANCED), 1)] +#[case::identical_by_number(established(), by_number, canonical(FINALIZED), 0)] +#[case::uninitialized_adopts_genesis(uninitialized(), by_hash, genesis_block(), 1)] +#[tokio::test] +async fn finalize_task_finalizes_canonical_target( + #[case] finalized_head: L2BlockInfo, + #[case] identify: fn(&RpcBlock) -> FinalizeBlockId, + #[case] served_block: RpcBlock, + #[case] forkchoice_updates: usize, +) { + let mut state = engine_state(finalized_head); + let target = identify(&served_block); + let expected_sync_state = with_finalized_head(&state, l2_block_info(&served_block)); + + finalize(target, &mut state, |l2| { + l2.expect_block_by_number(target.number(), Some(served_block)); + for _ in 0..forkchoice_updates { + l2.expect_forkchoice_updated( + expected_sync_state.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + } + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected_sync_state); +} + +/// A target pinned to the finalized head is already finalized, so it needs no lookup. +#[tokio::test] +async fn finalize_task_skips_lookup_for_finalized_head() { + let mut state = engine_state(established()); + let original_state = state; + + finalize(by_hash(&canonical(FINALIZED)), &mut state, |_| {}).await.unwrap(); + + assert_eq!(state, original_state); +} + +/// The engine serving no block, or a block at a height other than the requested one, stops the +/// node instead of finalizing anything. A misplaced block could otherwise bypass the guards against +/// moving finality backward. +#[rstest] +#[case::missing_by_number(FinalizeBlockId::ByNumber(ADVANCED), None, is_block_not_found)] +#[case::missing_by_hash(by_hash(&canonical(ADVANCED)), None, is_block_not_found)] +#[case::stale_block_by_number( + /* target */ FinalizeBlockId::ByNumber(ADVANCED), + /* served_block */ Some(canonical(STALE)), + /* is_expected_error */ is_number_mismatch +)] +#[case::higher_block_by_number( + /* target */ FinalizeBlockId::ByNumber(ADVANCED), + /* served_block */ Some(canonical(SAFE)), + /* is_expected_error */ is_number_mismatch +)] +#[case::stale_block_by_hash(by_hash(&canonical(ADVANCED)), Some(canonical(STALE)), is_number_mismatch)] +#[tokio::test] +async fn finalize_task_stops_on_unexpected_lookup_result( + #[case] target: FinalizeBlockId, + #[case] served_block: Option, + #[case] is_expected_error: fn(&FinalizeTaskError) -> bool, +) { + let mut state = engine_state(established()); + let original_state = state; + + let err = + finalize(target, &mut state, |l2| l2.expect_block_by_number(target.number(), served_block)) + .await + .unwrap_err(); + + assert!(is_expected_error(&err), "{err:?}"); + assert_eq!(err.severity(), EngineTaskErrorSeverity::Critical); + assert_eq!(state, original_state); } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/task.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/task.rs index ee976a02201..6b3a3d1b4cb 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/task.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/task.rs @@ -32,6 +32,10 @@ pub enum EngineTaskErrorSeverity { /// The error indicates that the engine should be flushed. #[display("flush")] Flush, + /// The task no longer applies to the engine state, so it is discarded instead of retried and + /// the engine keeps running. + #[display("skip")] + Skip, } /// The interface for an engine task error. diff --git a/rust/kona/crates/node/engine/src/test_utils/blocks.rs b/rust/kona/crates/node/engine/src/test_utils/blocks.rs new file mode 100644 index 00000000000..434d3bd01eb --- /dev/null +++ b/rust/kona/crates/node/engine/src/test_utils/blocks.rs @@ -0,0 +1,40 @@ +//! L2 blocks, and their RPC form as the execution layer serves them. + +use super::l1_info_deposit; +use alloy_consensus::transaction::Recovered; +use alloy_primitives::{Address, B256, Bytes}; +use alloy_rpc_types_eth::{BlockTransactions, Header as RpcHeader}; +use op_alloy_consensus::OpBlock; + +/// A full L2 block as the execution layer serves it over RPC. +pub type TestRpcBlock = alloy_rpc_types_eth::Block; + +/// An L2 block at `number` on `parent_hash` whose only transaction is the L1-info deposit, so it +/// converts into an `L2BlockInfo` without being the configured genesis. Blocks with different +/// `extra_data` share a height but not a hash. +pub fn test_l2_block(number: u64, parent_hash: B256, extra_data: u8) -> OpBlock { + let mut block = OpBlock::default(); + block.header.extra_data = Bytes::from(vec![extra_data]); + block.header.number = number; + block.header.parent_hash = parent_hash; + block.header.timestamp = number * 2; + block.body.transactions.push(l1_info_deposit()); + block +} + +/// `block` as the execution layer serves it over RPC. +pub fn test_rpc_block(block: &OpBlock) -> TestRpcBlock { + let transactions = block + .body + .transactions + .iter() + .cloned() + .map(|transaction| { + op_alloy_rpc_types::Transaction::from_transaction( + Recovered::new_unchecked(transaction, Address::ZERO), + Default::default(), + ) + }) + .collect(); + TestRpcBlock::new(RpcHeader::new(block.header.clone()), BlockTransactions::Full(transactions)) +} diff --git a/rust/kona/crates/node/engine/src/test_utils/mod.rs b/rust/kona/crates/node/engine/src/test_utils/mod.rs index e5949f23225..0a60403a0d6 100644 --- a/rust/kona/crates/node/engine/src/test_utils/mod.rs +++ b/rust/kona/crates/node/engine/src/test_utils/mod.rs @@ -1,6 +1,9 @@ mod attributes; pub use attributes::TestAttributesBuilder; +mod blocks; +pub use blocks::{TestRpcBlock, test_l2_block, test_rpc_block}; + mod deposits; pub use deposits::l1_info_deposit; diff --git a/rust/kona/crates/node/engine/src/test_utils/rpc.rs b/rust/kona/crates/node/engine/src/test_utils/rpc.rs index 1cfc3995356..39f23f14de2 100644 --- a/rust/kona/crates/node/engine/src/test_utils/rpc.rs +++ b/rust/kona/crates/node/engine/src/test_utils/rpc.rs @@ -1,6 +1,6 @@ //! Scripted RPC expectations over Alloy's mock transport. -use crate::EngineClient; +use crate::{EngineClient, test_utils::TestRpcBlock}; use alloy_json_rpc::{ErrorPayload, RequestPacket, ResponsePacket}; use alloy_network::Network; use alloy_provider::RootProvider; @@ -59,6 +59,16 @@ impl RpcMock { ); } + /// Expects a full-block `eth_getBlockByNumber` call for `number` and answers it with `block`, + /// or with null for `None`. + pub fn expect_block_by_number(&self, number: u64, block: Option) { + self.expect_params( + "eth_getBlockByNumber", + serde_json::json!([format!("{number:#x}"), true]), + block, + ); + } + /// Expects an `engine_forkchoiceUpdatedV3` call for `forkchoice` without payload attributes and /// answers it with `status`. pub fn expect_forkchoice_updated( diff --git a/rust/kona/crates/node/service/src/actors/derivation/actor/core.rs b/rust/kona/crates/node/service/src/actors/derivation/actor/core.rs index e3a5ea1ba4d..830b7ce4eaf 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/actor/core.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/actor/core.rs @@ -369,8 +369,9 @@ where self.attempt_derivation().await?; } - DerivationActorRequest::ProcessEngineSyncCompletionRequest(safe_head) => { + DerivationActorRequest::ProcessEngineSyncCompletionRequest(sync_state) => { info!(target: "derivation", "Engine finished syncing, starting derivation."); + let safe_head = Box::new(sync_state.local_safe_head()); self.derivation_state_machine .update(&DerivationStateUpdate::ELSyncCompleted(safe_head.clone()))?; self.truncate_safe_db(*safe_head, "EL sync completion")?; diff --git a/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs b/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs index 5bd4f9cfc00..01eb187e119 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs @@ -10,7 +10,7 @@ use kona_derive::{ ActivationSignal, Pipeline, PipelineError, PipelineErrorKind, PipelineResult, ResetError, Signal, SignalReceiver, StepResult, }; -use kona_engine::FinalizeBlockId; +use kona_engine::{EngineSyncState, EngineSyncStateUpdate, FinalizeBlockId}; use kona_genesis::{HardForkConfig, RollupConfig, SystemConfig}; use kona_protocol::{BlockInfo, L2BlockInfo, OpAttributesWithParent}; use kona_safedb::{SafeDatabase, SafeDb, SafeDbError, SafeHeadRecord}; @@ -41,12 +41,7 @@ async fn reset_adopts_safe_head_and_resumes_atomically(#[case] previous_head: u6 l1_block(0).id(), watch::channel(None).0, ); - request_tx - .send(DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(l2_block( - previous_head, - )))) - .await - .unwrap(); + request_tx.send(engine_sync_completed_at(l2_block(previous_head))).await.unwrap(); actor.step().await.unwrap(); request_tx @@ -267,10 +262,7 @@ async fn test_origin_is_published(#[case] step_result: StepResult) { actor.step().await.unwrap(); assert_eq!(derivation_origin_rx.borrow().map(|origin| origin.id()), Some(safe_head.l1_origin)); - request_tx - .send(DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(safe_head))) - .await - .unwrap(); + request_tx.send(engine_sync_completed_at(safe_head)).await.unwrap(); actor.step().await.unwrap(); assert_eq!(derivation_origin_rx.borrow().unwrap().number, 11); @@ -465,6 +457,16 @@ fn l1_block(number: u64) -> BlockInfo { BlockInfo { hash: B256::with_last_byte(number as u8), number, ..Default::default() } } +/// The engine reporting that its sync completed with `safe_head` as its safe heads. +fn engine_sync_completed_at(safe_head: L2BlockInfo) -> DerivationActorRequest { + let sync_state = EngineSyncState::default().apply_update(EngineSyncStateUpdate { + local_safe_head: Some(safe_head), + safe_head: Some(safe_head), + ..Default::default() + }); + DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(sync_state)) +} + fn l2_block(number: u64) -> L2BlockInfo { L2BlockInfo { block_info: BlockInfo { @@ -535,10 +537,7 @@ impl RecordingActor { } async fn complete_el_sync_at(&mut self, head: u64) { - self.handle(DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new( - l2_block(head), - ))) - .await; + self.handle(engine_sync_completed_at(l2_block(head))).await; } async fn confirm_safe_head(&mut self, head: u64) { diff --git a/rust/kona/crates/node/service/src/actors/derivation/actor/tests/database.rs b/rust/kona/crates/node/service/src/actors/derivation/actor/tests/database.rs index 699bc1e6933..eeee778a78a 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/actor/tests/database.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/actor/tests/database.rs @@ -135,7 +135,7 @@ fn engine_reset_request() -> DerivationActorRequest { } fn sync_completion_request() -> DerivationActorRequest { - DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(l2_block(RESUME_HEAD))) + engine_sync_completed_at(l2_block(RESUME_HEAD)) } #[rstest] diff --git a/rust/kona/crates/node/service/src/actors/derivation/actor/tests/op_node_parity.rs b/rust/kona/crates/node/service/src/actors/derivation/actor/tests/op_node_parity.rs index 104544f2522..0df6cef81c1 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/actor/tests/op_node_parity.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/actor/tests/op_node_parity.rs @@ -16,7 +16,7 @@ fn l1_head_update(number: u64) -> DerivationActorRequest { } fn el_sync_completed(head: u64) -> DerivationActorRequest { - DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(l2_block(head))) + engine_sync_completed_at(l2_block(head)) } /// A reset whose pipeline signal fails once with a temporary provider error neither stops the diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs index 44fe763d969..7001d2f71d6 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs @@ -2,10 +2,11 @@ use crate::{ DerivationActorRequest, DerivationEngineClient, NodeActor, actors::derivation::{DerivationDelegateProvider, DerivationError}, }; +use alloy_eips::BlockNumHash; use alloy_primitives::BlockHash; use async_trait::async_trait; use kona_derive::{ChainProvider, Signal}; -use kona_engine::FinalizeBlockId; +use kona_engine::{EngineState, FinalizeBlockId}; use kona_protocol::{BlockInfo, L2BlockInfo, SyncStatus}; use thiserror::Error; use tokio::{ @@ -42,6 +43,10 @@ where /// Publishes the delegate's `current_l1`: the L1 block that its derivation has reached. derivation_origin_tx: watch::Sender>, + /// The engine's state as it publishes it. Delegate statuses are checked against the + /// finalized head the engine applied, not the last one forwarded to it, which the engine may + /// have skipped. + engine_state_rx: watch::Receiver, /// The engine's L2 safe head, according to updates from the Engine. engine_l2_safe_head: L2BlockInfo, /// Whether the engine sync has completed. This will only ever go from false -> true. @@ -64,6 +69,7 @@ where derivation_delegate_provider: DelegateProvider, l1_provider: L1Provider, derivation_origin_tx: watch::Sender>, + engine_state_rx: watch::Receiver, ) -> Self { let mut delegated_derivation_ticker = time::interval(Self::DERIVATION_DELEGATE_POLL_INTERVAL); @@ -74,6 +80,7 @@ where derivation_delegate_provider, l1_provider, derivation_origin_tx, + engine_state_rx, engine_l2_safe_head: L2BlockInfo::default(), has_engine_sync_completed: false, delegated_derivation_ticker, @@ -149,9 +156,44 @@ where Ok(()) } - /// Verifies that the L1 info reported by the derivation delegate - /// are consistent with canonical L1 chain. + /// The id of an established finalized head. The default head means finality has not been + /// established yet, so any finalized block from the delegate may initialize it. + fn established_finalized_l2(finalized_head: L2BlockInfo) -> Option { + (finalized_head != L2BlockInfo::default()).then(|| finalized_head.block_info.id()) + } + + /// Rejects a finalized L2 block behind, or at the same height as but different from, the + /// engine's. Either means the delegate lags or follows another chain, so none of its status + /// applies: its safe head would move the engine back or onto that chain too. + fn validate_finalized_l2_progress( + &self, + reported: BlockNumHash, + ) -> Result<(), DerivationDelegationError> { + let engine_finalized_head = self.engine_state_rx.borrow().sync_state.finalized_head(); + let Some(engine) = Self::established_finalized_l2(engine_finalized_head) else { + return Ok(()); + }; + if reported.number < engine.number { + return Err(DerivationDelegationError::FinalizedL2Regressed { engine, reported }); + } + if reported.number == engine.number && reported.hash != engine.hash { + return Err(DerivationDelegationError::FinalizedL2Conflict { engine, reported }); + } + Ok(()) + } + + /// Verifies that the finalized L2 block reported by the derivation delegate does not move back + /// from the engine's, and that its L1 info is consistent with the canonical L1 chain. async fn validate_sync_status(&mut self, v: &SyncStatus) -> bool { + if let Err(err) = self.validate_finalized_l2_progress(v.finalized_l2.block_info.id()) { + warn!( + target: "derivation", + error = %err, + "Finalized L2 inconsistency detected at sync status from delegate" + ); + return false; + } + let checks = [ ("L1 Origin of Safe L2", v.safe_l2.l1_origin.number, v.safe_l2.l1_origin.hash), ( @@ -227,9 +269,9 @@ where debug!(target: "derivation", safe_head = ?*safe_head, "Received safe head from engine."); self.engine_l2_safe_head = *safe_head; } - DerivationActorRequest::ProcessEngineSyncCompletionRequest(safe_head) => { + DerivationActorRequest::ProcessEngineSyncCompletionRequest(sync_state) => { info!(target: "derivation", "Engine finished syncing, starting derivation."); - self.engine_l2_safe_head = *safe_head; + self.engine_l2_safe_head = sync_state.local_safe_head(); self.has_engine_sync_completed = true; } DerivationActorRequest::ProcessEngineSignalRequest(signal) => match *signal { @@ -249,6 +291,14 @@ where #[derive(Error, Debug)] enum DerivationDelegationError { + /// The delegate's finalized L2 block is a different block at the engine's finalized height. + #[error("Finalized L2 {reported:?} conflicts with the engine's finalized L2 {engine:?}")] + FinalizedL2Conflict { engine: BlockNumHash, reported: BlockNumHash }, + + /// The delegate's finalized L2 block is behind the engine's. + #[error("Finalized L2 {reported:?} is behind the engine's finalized L2 {engine:?}")] + FinalizedL2Regressed { engine: BlockNumHash, reported: BlockNumHash }, + /// The L1 provider returned an error (network, RPC, etc.) #[error("L1 provider error: {0}")] L1Provider(String), diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/mod.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/mod.rs index e5460eceb11..5431e58aa93 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/mod.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/mod.rs @@ -5,3 +5,6 @@ mod client; pub use client::{ DerivationDelegateClient, DerivationDelegateClientError, DerivationDelegateProvider, }; + +#[cfg(test)] +mod tests; diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs new file mode 100644 index 00000000000..8f976aa256c --- /dev/null +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs @@ -0,0 +1,250 @@ +use super::{DelegateDerivationActor, DerivationDelegateClientError, DerivationDelegateProvider}; +use crate::{DerivationActorRequest, EngineActorRequest, NodeActor, QueuedDerivationEngineClient}; +use alloy_eips::BlockNumHash; +use alloy_primitives::B256; +use async_trait::async_trait; +use kona_derive::test_utils::TestChainProvider; +use kona_engine::{ + ConsolidateInput, EngineState, EngineSyncState, EngineSyncStateUpdate, FinalizeBlockId, +}; +use kona_protocol::{BlockInfo, L2BlockInfo, SyncStatus}; +use rstest::rstest; +use std::{collections::VecDeque, sync::Mutex}; +use tokio::sync::{mpsc, watch}; + +const SAFE_ORIGIN: u64 = 1; +const FINALIZED_ORIGIN: u64 = 2; +const CURRENT_L1: u64 = 3; + +type SyncStatusReply = Result; + +/// Answers each sync status poll with the next scripted reply. +#[derive(Debug)] +struct ScriptedDelegate(Mutex>); + +#[async_trait] +impl DerivationDelegateProvider for ScriptedDelegate { + async fn fetch_sync_status(&self) -> SyncStatusReply { + self.0.lock().unwrap().pop_front().expect("unexpected sync status poll") + } +} + +/// A delegate actor over a canonical L1 chain, with the engine requests it sends and the +/// `current_l1` it publishes. +struct Harness { + actor: + DelegateDerivationActor, + current_l1: watch::Receiver>, + derivation_requests: mpsc::Sender, + engine_requests: mpsc::Receiver, + engine_state: watch::Sender, +} + +impl Harness { + fn new(replies: impl IntoIterator) -> Self { + let mut l1 = TestChainProvider::default(); + for number in [SAFE_ORIGIN, FINALIZED_ORIGIN, CURRENT_L1] { + l1.insert_block(number, canonical_l1_block(number)); + } + let (derivation_requests, derivation_request_rx) = mpsc::channel(8); + let (engine_request_tx, engine_requests) = mpsc::channel(8); + let (current_l1_tx, current_l1) = watch::channel(None); + let (engine_state, engine_state_rx) = watch::channel(EngineState::default()); + let actor = DelegateDerivationActor::new( + QueuedDerivationEngineClient::new(engine_request_tx), + derivation_request_rx, + ScriptedDelegate(Mutex::new(replies.into_iter().collect())), + l1, + current_l1_tx, + engine_state_rx, + ); + Self { actor, current_l1, derivation_requests, engine_requests, engine_state } + } + + async fn complete_engine_sync(&mut self) { + self.complete_engine_sync_with(EngineSyncState::default()).await; + } + + /// Completes the engine sync with `sync_state` as the engine's labels, which the engine + /// publishes before it reports the completion. + async fn complete_engine_sync_with(&mut self, sync_state: EngineSyncState) { + self.publish_engine_sync_state(sync_state); + self.derivation_requests + .send(DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(sync_state))) + .await + .unwrap(); + self.actor.step().await.unwrap(); + } + + /// Publishes the engine's labels as the engine does after each task it runs. + fn publish_engine_sync_state(&self, sync_state: EngineSyncState) { + self.engine_state.send_modify(|state| state.sync_state = sync_state); + } + + /// The engine requests sent since the last call, in order. + fn forwarded(&mut self) -> Vec { + std::iter::from_fn(|| self.engine_requests.try_recv().ok()).map(Forwarded::from).collect() + } +} + +/// The parts of an engine request that the delegate actor decides. +#[derive(Debug, PartialEq, Eq)] +enum Forwarded { + Finalized(FinalizeBlockId), + Other(String), + Safe(L2BlockInfo), +} + +impl From for Forwarded { + fn from(request: EngineActorRequest) -> Self { + match request { + EngineActorRequest::ProcessFinalizedL2Block(finalized) => Self::Finalized(*finalized), + EngineActorRequest::ProcessSafeL2Signal(ConsolidateInput::BlockInfo(safe)) => { + Self::Safe(safe) + } + other => Self::Other(format!("{other:?}")), + } + } +} + +/// What one poll of `status` forwards: the safe head, then the finalized block pinned by hash. +fn forwarded_for(status: &SyncStatus) -> Vec { + vec![ + Forwarded::Safe(status.safe_l2), + Forwarded::Finalized(FinalizeBlockId::ByHash(status.finalized_l2.block_info.id())), + ] +} + +/// What one poll forwards when the status is accepted, and nothing when it is rejected. +fn forwarded_if(accepted: bool, status: &SyncStatus) -> Vec { + if accepted { forwarded_for(status) } else { Vec::new() } +} + +fn canonical_l1_block(number: u64) -> BlockInfo { + BlockInfo { hash: B256::with_last_byte(number as u8), number, ..Default::default() } +} + +fn l2_block(number: u64, l1_origin: u64) -> L2BlockInfo { + L2BlockInfo { + block_info: BlockInfo { + hash: B256::with_last_byte(0x80 | number as u8), + number, + ..Default::default() + }, + l1_origin: canonical_l1_block(l1_origin).id(), + seq_num: 0, + } +} + +/// A sync status whose safe origin, finalized origin and `current_l1` are all canonical. +fn delegate_status() -> SyncStatus { + SyncStatus { + current_l1: canonical_l1_block(CURRENT_L1), + current_l1_finalized: BlockInfo::default(), + finalized_l1: BlockInfo::default(), + finalized_l2: l2_block(15, FINALIZED_ORIGIN), + head_l1: BlockInfo::default(), + local_safe_l2: l2_block(20, SAFE_ORIGIN), + safe_l1: BlockInfo::default(), + safe_l2: l2_block(20, SAFE_ORIGIN), + unsafe_l2: l2_block(25, SAFE_ORIGIN), + } +} + +fn non_canonical(id: BlockNumHash) -> BlockNumHash { + BlockNumHash { hash: B256::repeat_byte(0xee), ..id } +} + +/// Returns `status` with its finalized L2 block replaced by `finalized_l2`. +fn with_finalized_l2(finalized_l2: L2BlockInfo) -> SyncStatus { + SyncStatus { finalized_l2, ..delegate_status() } +} + +/// An engine sync state whose finalized head is `finalized_l2`. +fn engine_finalized_at(finalized_l2: L2BlockInfo) -> EngineSyncState { + EngineSyncState::default().apply_update(EngineSyncStateUpdate { + finalized_head: Some(finalized_l2), + ..Default::default() + }) +} + +fn conflicting(block: L2BlockInfo) -> L2BlockInfo { + L2BlockInfo { + block_info: BlockInfo { + hash: non_canonical(block.block_info.id()).hash, + ..block.block_info + }, + ..block + } +} + +/// The engine's finalized head when its sync completes is the baseline, so a delegate that starts +/// behind it, or on another chain at its height, is rejected whole from the first poll on. +#[rstest] +#[case::advancing(l2_block(16, FINALIZED_ORIGIN), true)] +#[case::unchanged(l2_block(15, FINALIZED_ORIGIN), true)] +#[case::regressed(l2_block(14, FINALIZED_ORIGIN), false)] +#[case::conflicting(conflicting(l2_block(15, FINALIZED_ORIGIN)), false)] +#[tokio::test(start_paused = true)] +async fn first_poll_rejects_status_behind_engine_finality( + #[case] finalized_l2: L2BlockInfo, + #[case] accepted: bool, +) { + let status = with_finalized_l2(finalized_l2); + let mut harness = Harness::new([Ok(status.clone())]); + harness.complete_engine_sync_with(engine_finalized_at(l2_block(15, FINALIZED_ORIGIN))).await; + + harness.actor.step().await.unwrap(); + + assert_eq!(harness.forwarded(), forwarded_if(accepted, &status)); + assert_eq!(harness.current_l1.borrow().is_some(), accepted); +} + +/// Once the engine applies a forwarded finalized block, a status whose finalized block is behind +/// it, or is a different block at its height, is rejected whole: a lagging or diverged delegate's +/// safe head must not reach the engine either, and its `current_l1` is not published. +#[rstest] +#[case::advancing(l2_block(16, FINALIZED_ORIGIN), true)] +#[case::unchanged(l2_block(15, FINALIZED_ORIGIN), true)] +#[case::regressed(l2_block(14, FINALIZED_ORIGIN), false)] +#[case::conflicting(conflicting(l2_block(15, FINALIZED_ORIGIN)), false)] +#[tokio::test(start_paused = true)] +async fn poll_rejects_status_whose_finality_moves_back( + #[case] next_finalized_l2: L2BlockInfo, + #[case] accepted: bool, +) { + let next_status = with_finalized_l2(next_finalized_l2); + let mut harness = Harness::new([Ok(delegate_status()), Ok(next_status.clone())]); + harness.complete_engine_sync().await; + harness.actor.step().await.unwrap(); + assert_eq!(harness.forwarded(), forwarded_for(&delegate_status())); + harness.publish_engine_sync_state(engine_finalized_at(delegate_status().finalized_l2)); + harness.current_l1.borrow_and_update(); + + harness.actor.step().await.unwrap(); + + assert_eq!(harness.forwarded(), forwarded_if(accepted, &next_status)); + assert_eq!(harness.current_l1.has_changed().unwrap(), accepted); +} + +/// A finalized block the engine skipped, such as one on another chain, does not become the +/// baseline: the canonical block at its height, or below it but above the engine's finalized +/// head, is still accepted, so the delegate's safe head keeps reaching the engine. +#[rstest] +#[case::canonical_at_skipped_height(l2_block(17, FINALIZED_ORIGIN))] +#[case::canonical_below_skipped_height(l2_block(16, FINALIZED_ORIGIN))] +#[tokio::test(start_paused = true)] +async fn poll_accepts_status_after_engine_skipped_forwarded_finality( + #[case] canonical_finalized_l2: L2BlockInfo, +) { + let skipped_status = with_finalized_l2(conflicting(l2_block(17, FINALIZED_ORIGIN))); + let canonical_status = with_finalized_l2(canonical_finalized_l2); + let mut harness = Harness::new([Ok(skipped_status.clone()), Ok(canonical_status.clone())]); + harness.complete_engine_sync_with(engine_finalized_at(l2_block(15, FINALIZED_ORIGIN))).await; + harness.actor.step().await.unwrap(); + assert_eq!(harness.forwarded(), forwarded_for(&skipped_status)); + + harness.actor.step().await.unwrap(); + + assert_eq!(harness.forwarded(), forwarded_for(&canonical_status)); +} diff --git a/rust/kona/crates/node/service/src/actors/derivation/request.rs b/rust/kona/crates/node/service/src/actors/derivation/request.rs index 9482eef5582..caa1e2b9ad2 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/request.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/request.rs @@ -1,4 +1,5 @@ use kona_derive::Signal; +use kona_engine::EngineSyncState; use kona_protocol::{BlockInfo, L2BlockInfo}; use thiserror::Error; @@ -21,9 +22,9 @@ pub enum DerivationClientError { /// Inbound requests that the [`crate::DerivationActor`] can process. #[derive(Debug)] pub enum DerivationActorRequest { - /// Request to process the fact that Engine sync has completed, along with the current safe - /// head. - ProcessEngineSyncCompletionRequest(Box), + /// Request to process the fact that Engine sync has completed, along with the engine's sync + /// state at that point. + ProcessEngineSyncCompletionRequest(Box), /// Request to process the provided L2 engine safe head update. ProcessEngineSafeHeadUpdateRequest(Box), /// A request containing a [`Signal`] to the derivation pipeline. diff --git a/rust/kona/crates/node/service/src/actors/engine/actor.rs b/rust/kona/crates/node/service/src/actors/engine/actor.rs index c9affd16523..87619ba2756 100644 --- a/rust/kona/crates/node/service/src/actors/engine/actor.rs +++ b/rust/kona/crates/node/service/src/actors/engine/actor.rs @@ -5,8 +5,8 @@ use async_trait::async_trait; use kona_derive::{ResetSignal, Signal}; use kona_engine::{ BuildTask, CanonicalizeTask, ConsolidateTask, Engine, EngineActorRequest, EngineClient, - EngineTask, EngineTaskError, EngineTaskErrorSeverity, FinalizeTask, ImportedBlockSink, - InsertTask, SealTask, + EngineTask, EngineTaskError, EngineTaskErrorSeverity, EngineTaskErrors, FinalizeTask, + ImportedBlockSink, InsertTask, SealTask, }; use kona_genesis::RollupConfig; use kona_protocol::L2BlockInfo; @@ -34,6 +34,9 @@ where el_sync_complete: bool, /// The last local-safe head sent to the derivation actor. last_engine_safe_head_sent: L2BlockInfo, + /// The error of the last skipped engine task, so an identical repeat logs no new warning + /// until a reset changes the state the task was judged against. + last_skipped_task_error: Option, /// A channel to use to relay the current unsafe head. /// ## Note /// This is `Some` when the node is in sequencer mode, and `None` when the node is in validator @@ -81,6 +84,7 @@ where el_sync_complete: false, engine, last_engine_safe_head_sent: L2BlockInfo::default(), + last_skipped_task_error: None, rollup: config, unsafe_head_tx, inbound_request_rx, @@ -92,6 +96,18 @@ where } } + /// Logs a skipped task at `warn!` when its error differs from the last skip, and at `debug!` + /// while the identical error repeats, so a target resubmitted on every poll warns once. + fn log_skipped_task(&mut self, err: &EngineTaskErrors) { + let message = err.to_string(); + if self.last_skipped_task_error.as_ref() == Some(&message) { + debug!(target: "engine", %err, "Skipped the same engine task again"); + return; + } + warn!(target: "engine", %err, "Skipped an engine task that no longer applies"); + self.last_skipped_task_error = Some(message); + } + fn defer_retry(&mut self) { self.retry_at = Instant::now() + self.retry_delay; self.retry_delay = (self.retry_delay * 2).min(Duration::from_secs(5)); @@ -100,6 +116,7 @@ where fn begin_reset(&mut self) { if !self.reset_pending { self.engine.clear(); + self.last_skipped_task_error = None; self.reset_pending = true; self.retry_at = Instant::now(); } @@ -210,6 +227,7 @@ where self.defer_retry(); return Ok(()); } + EngineTaskErrorSeverity::Skip => self.log_skipped_task(&err), } } } @@ -233,7 +251,7 @@ where self.el_sync_complete = true; self.derivation_client - .notify_sync_completed(self.engine.state().sync_state.local_safe_head()) + .notify_sync_completed(self.engine.state().sync_state) .await .map(|_| Ok(())) .map_err(|e| { diff --git a/rust/kona/crates/node/service/src/actors/engine/client.rs b/rust/kona/crates/node/service/src/actors/engine/client.rs index 25322a5a7f2..518e7b957d0 100644 --- a/rust/kona/crates/node/service/src/actors/engine/client.rs +++ b/rust/kona/crates/node/service/src/actors/engine/client.rs @@ -2,6 +2,7 @@ use crate::{DerivationActorRequest, DerivationClientError, DerivationClientResul use async_trait::async_trait; use derive_more::Constructor; use kona_derive::Signal; +use kona_engine::EngineSyncState; use kona_protocol::L2BlockInfo; use std::fmt::Debug; use tokio::sync::mpsc; @@ -10,9 +11,13 @@ use tokio::sync::mpsc; #[cfg_attr(test, mockall::automock)] #[async_trait] pub trait EngineDerivationClient: Debug + Send + Sync { - /// Notifies the [`crate::DerivationActor`] that engine syncing has completed. + /// Notifies the [`crate::DerivationActor`] that engine syncing has completed, with the engine's + /// sync state at that point. /// Note: Does not wait for the derivation client to process this message. - async fn notify_sync_completed(&self, safe_head: L2BlockInfo) -> DerivationClientResult<()>; + async fn notify_sync_completed( + &self, + sync_state: EngineSyncState, + ) -> DerivationClientResult<()>; /// Sends the new engine `safe_head` to the [`crate::DerivationActor`]. /// Note: Does not wait for the derivation client to process this message. @@ -33,11 +38,14 @@ pub struct QueuedEngineDerivationClient { #[async_trait] impl EngineDerivationClient for QueuedEngineDerivationClient { - async fn notify_sync_completed(&self, safe_head: L2BlockInfo) -> DerivationClientResult<()> { + async fn notify_sync_completed( + &self, + sync_state: EngineSyncState, + ) -> DerivationClientResult<()> { info!(target: "engine", "Sending sync completed to derivation actor"); self.derivation_actor_request_tx - .send(DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(safe_head))) + .send(DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(sync_state))) .await .map_err(|_| { DerivationClientError::RequestError("request channel closed.".to_string()) diff --git a/rust/kona/crates/node/service/src/actors/engine/tests.rs b/rust/kona/crates/node/service/src/actors/engine/tests.rs index b61712d95b5..d000710cbf1 100644 --- a/rust/kona/crates/node/service/src/actors/engine/tests.rs +++ b/rust/kona/crates/node/service/src/actors/engine/tests.rs @@ -312,5 +312,5 @@ async fn derivation_is_handed_the_local_safe_head() { else { panic!("expected the sync completion second"); }; - assert_eq!(*sent, local_safe_head); + assert_eq!(sent.local_safe_head(), local_safe_head); } diff --git a/rust/kona/crates/node/service/src/service/node/rollup.rs b/rust/kona/crates/node/service/src/service/node/rollup.rs index d1b5fb0a12e..ed509b22101 100644 --- a/rust/kona/crates/node/service/src/service/node/rollup.rs +++ b/rust/kona/crates/node/service/src/service/node/rollup.rs @@ -289,6 +289,7 @@ impl RollupNode { engine_actor_request_tx: mpsc::Sender, derivation_actor_request_rx: mpsc::Receiver, derivation_origin_tx: watch::Sender>, + engine_state_rx: watch::Receiver, l1_head: watch::Receiver>, ) -> Result { if let Some(provider) = self.derivation_delegate_provider.clone() { @@ -303,6 +304,7 @@ impl RollupNode { provider, l1_provider, derivation_origin_tx, + engine_state_rx, )))) } else { let pipeline = self @@ -621,6 +623,7 @@ impl RollupNode { engine_actor_request_tx.clone(), derivation_actor_request_rx, derivation_origin_tx, + engine_state_rx.clone(), l1_head_updates_rx.clone(), ) .await?; From b73f56aa627ec55b7d4a3d84ebe09f34bddd208a Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 12 Nov 2026 14:30:53 +0100 Subject: [PATCH 3/7] kona-engine: replace ForkchoiceDelivery with two synchronize functions `synchronize` skips a forkchoice update that leaves the labels unchanged; `synchronize_unconditionally` always sends it. Both share a private `send_forkchoice`. The reset and the delegated paths of the consolidate task call `synchronize_unconditionally`; the insert, finalize and derived consolidate paths call `synchronize`. The consolidate task's thin wrappers `reconcile_to_safe_head`, `follow_safe_head_during_el_sync` and `apply_delegated_safe_head` are inlined at their call sites, keeping their logs and comments. Behavior is unchanged. Co-Authored-By: Claude Opus 5.5 --- .../crates/node/engine/src/task_queue/core.rs | 5 +- .../src/task_queue/tasks/consolidate/task.rs | 198 ++++++++---------- .../src/task_queue/tasks/finalize/task.rs | 4 +- .../src/task_queue/tasks/insert/task.rs | 4 +- .../node/engine/src/task_queue/tasks/mod.rs | 2 +- .../task_queue/tasks/synchronize/helper.rs | 70 ++++--- .../src/task_queue/tasks/synchronize/mod.rs | 2 +- .../src/task_queue/tasks/synchronize/tests.rs | 39 +++- 8 files changed, 159 insertions(+), 165 deletions(-) diff --git a/rust/kona/crates/node/engine/src/task_queue/core.rs b/rust/kona/crates/node/engine/src/task_queue/core.rs index acc3fa27d40..2167623aaa2 100644 --- a/rust/kona/crates/node/engine/src/task_queue/core.rs +++ b/rust/kona/crates/node/engine/src/task_queue/core.rs @@ -5,7 +5,7 @@ use crate::{ EngineClient, EngineState, EngineSyncStateUpdate, EngineTask, EngineTaskError, EngineTaskErrorSeverity, Metrics, SyncStartError, SynchronizeTaskError, find_starting_forkchoice, - task_queue::{EngineTaskErrors, ForkchoiceDelivery, synchronize}, + task_queue::{EngineTaskErrors, synchronize_unconditionally}, }; use kona_genesis::RollupConfig; use kona_protocol::L2BlockInfo; @@ -118,7 +118,7 @@ impl Engine { // One attempt. The actor retains the reset request and retries transient failures. The // execution layer may have lost or changed its forkchoice, so the reset sends one even when // the recovered labels equal the current ones. - synchronize( + synchronize_unconditionally( client.as_ref(), &mut self.state, EngineSyncStateUpdate { @@ -127,7 +127,6 @@ impl Engine { safe_head: Some(start.safe), finalized_head: Some(start.finalized), }, - ForkchoiceDelivery::Always, ) .await?; diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs index 9531eac4aa3..d3183cdafa5 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs @@ -3,7 +3,7 @@ use crate::{ ConsolidateTaskError, EngineClient, EngineState, ExecutableEngineTask, ImportedBlockSink, state::EngineSyncStateUpdate, - task_queue::{ForkchoiceDelivery, build_and_import, synchronize}, + task_queue::{build_and_import, synchronize, synchronize_unconditionally}, }; use alloy_rpc_types_eth::Block; use async_trait::async_trait; @@ -56,16 +56,6 @@ impl ConsolidateInput { const fn is_intermediate_span_block(&self) -> bool { matches!(self, Self::Attributes(attributes) if !attributes.is_last_in_span) } - - /// Derived attributes move the safe head on every consolidation, so an unchanged forkchoice was - /// already sent. A delegated safe head can repeat a forkchoice the execution layer answered - /// `SYNCING` or lost, so it is always sent. - const fn forkchoice_delivery(&self) -> ForkchoiceDelivery { - match self { - Self::Attributes(_) => ForkchoiceDelivery::WhenChanged, - Self::BlockInfo(_) => ForkchoiceDelivery::Always, - } - } } /// The [`ConsolidateTask`] attempts to consolidate the engine state @@ -110,96 +100,6 @@ impl ConsolidateTask { Ok(()) } - /// This provides symmetric fallback behavior with `build_and_import`. - async fn reconcile_to_safe_head( - &self, - state: &mut EngineState, - safe_l2: &L2BlockInfo, - ) -> Result<(), ConsolidateTaskError> { - warn!( - target: "engine", - safe_l2 = %safe_l2, - "Apply safe head" - ); - - // We intentionally set the unsafe head to safe_l2 to ensure the engine observes a - // self-consistent head state. This is required to correctly handle reorgs (where unsafe - // may be ahead on a non-canonical fork) and to trigger EL sync when the local unsafe head - // lags behind the safe head. - self.apply_delegated_safe_head( - state, - safe_l2, - EngineSyncStateUpdate { - local_safe_head: Some(*safe_l2), - safe_head: Some(*safe_l2), - unsafe_head: Some(*safe_l2), - ..Default::default() - }, - "Updated safe head via follow safe", - ) - .await - } - - /// Moves the safe labels to a delegated safe head that the execution layer does not have yet, - /// leaving the unsafe head in place. - /// - /// A block missing below the unsafe head means the execution layer is still syncing toward the - /// unsafe head. Resetting would restart that sync from the execution layer's old labels, and - /// moving the unsafe head back would retarget it. Like op-node's follow-source mode, only the - /// safe labels follow the delegate, and the forkchoice update carries them to the execution - /// layer. - async fn follow_safe_head_during_el_sync( - &self, - state: &mut EngineState, - safe_l2: &L2BlockInfo, - ) -> Result<(), ConsolidateTaskError> { - warn!( - target: "engine", - safe_l2 = %safe_l2, - unsafe_number = state.sync_state.unsafe_head().block_info.number, - "Execution layer does not have the delegated safe head yet; keeping its sync target" - ); - - self.apply_delegated_safe_head( - state, - safe_l2, - EngineSyncStateUpdate { - local_safe_head: Some(*safe_l2), - safe_head: Some(*safe_l2), - ..Default::default() - }, - "Updated safe head while the execution layer syncs", - ) - .await - } - - /// Applies `update`, whose labels follow the delegated `safe_l2`, and always sends its - /// forkchoice update: the execution layer may have answered `SYNCING` to the same labels - /// before, or lost them in a restart. - async fn apply_delegated_safe_head( - &self, - state: &mut EngineState, - safe_l2: &L2BlockInfo, - update: EngineSyncStateUpdate, - outcome: &'static str, - ) -> Result<(), ConsolidateTaskError> { - synchronize(self.client.as_ref(), state, update, ForkchoiceDelivery::Always) - .await - .map_err(|e| { - warn!(target: "engine", ?e, safe_l2 = %safe_l2, "Applying the delegated safe head failed"); - e - })?; - - info!( - target: "engine", - hash = %safe_l2.block_info.hash, - number = safe_l2.block_info.number, - "{outcome}" - ); - - Ok(()) - } - /// Handles the fallback case when the block doesn't match the input or does not exist. async fn reconcile_unsafe_to_safe( &self, @@ -210,7 +110,37 @@ impl ConsolidateTask { self.build_and_import_attributes(state, attributes).await } ConsolidateInput::BlockInfo(safe_l2) => { - self.reconcile_to_safe_head(state, safe_l2).await + warn!(target: "engine", safe_l2 = %safe_l2, "Apply safe head"); + + // We intentionally set the unsafe head to safe_l2 to ensure the engine observes a + // self-consistent head state. This is required to correctly handle reorgs (where + // unsafe may be ahead on a non-canonical fork) and to trigger EL sync when the + // local unsafe head lags behind the safe head. The forkchoice update is always + // sent: the execution layer may have answered `SYNCING` to the same labels before, + // or lost them in a restart. + synchronize_unconditionally( + self.client.as_ref(), + state, + EngineSyncStateUpdate { + local_safe_head: Some(*safe_l2), + safe_head: Some(*safe_l2), + unsafe_head: Some(*safe_l2), + ..Default::default() + }, + ) + .await + .map_err(|e| { + warn!(target: "engine", ?e, safe_l2 = %safe_l2, "Applying the delegated safe head failed"); + e + })?; + + info!( + target: "engine", + hash = %safe_l2.block_info.hash, + number = safe_l2.block_info.number, + "Updated safe head via follow safe" + ); + Ok(()) } } } @@ -230,7 +160,39 @@ impl ConsolidateTask { Ok(Some(block)) => block, Ok(None) => { if let ConsolidateInput::BlockInfo(safe_l2) = &self.input { - return self.follow_safe_head_during_el_sync(state, safe_l2).await; + // A block missing below the unsafe head means the execution layer is still + // syncing toward the unsafe head. Resetting would restart that sync from the + // execution layer's old labels, and moving the unsafe head back would retarget + // it. Like op-node's follow-source mode, only the safe labels follow the + // delegate, and the forkchoice update, sent even when unchanged, carries them + // to the execution layer. + warn!( + target: "engine", + safe_l2 = %safe_l2, + unsafe_number = state.sync_state.unsafe_head().block_info.number, + "Execution layer does not have the delegated safe head yet; keeping its sync target" + ); + synchronize_unconditionally( + self.client.as_ref(), + state, + EngineSyncStateUpdate { + local_safe_head: Some(*safe_l2), + safe_head: Some(*safe_l2), + ..Default::default() + }, + ) + .await + .map_err(|e| { + warn!(target: "engine", ?e, safe_l2 = %safe_l2, "Applying the delegated safe head failed"); + e + })?; + info!( + target: "engine", + hash = %safe_l2.block_info.hash, + number = safe_l2.block_info.number, + "Updated safe head while the execution layer syncs" + ); + return Ok(()); } warn!(target: "engine", "Received `None` block for {}", block_num); return Err(ConsolidateTaskError::MissingUnsafeL2Block(block_num)); @@ -278,18 +240,24 @@ impl ConsolidateTask { // The next attributes built are this block's child, and ask for its config. self.block_sink.block_imported(consensus_block, block_info); - synchronize( - self.client.as_ref(), - state, - EngineSyncStateUpdate { - safe_head: Some(block_info), - local_safe_head: Some(block_info), - ..Default::default() - }, - self.input.forkchoice_delivery(), - ) - .await - .map_err(|e| { + let update = EngineSyncStateUpdate { + safe_head: Some(block_info), + local_safe_head: Some(block_info), + ..Default::default() + }; + let synchronized = match &self.input { + // Derived attributes move the safe head on every consolidation, so an + // unchanged forkchoice was already sent. + ConsolidateInput::Attributes(_) => { + synchronize(self.client.as_ref(), state, update).await + } + // A delegated safe head can repeat a forkchoice the execution layer + // answered `SYNCING` or lost, so it is always sent. + ConsolidateInput::BlockInfo(_) => { + synchronize_unconditionally(self.client.as_ref(), state, update).await + } + }; + synchronized.map_err(|e| { warn!(target: "engine", ?e, "Consolidation failed"); e })?; diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs index d00bb74b624..3461d7d860f 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs @@ -2,8 +2,7 @@ use crate::{ EngineClient, EngineState, ExecutableEngineTask, FinalizeBlockId, FinalizeTaskError, - state::EngineSyncStateUpdate, - task_queue::{ForkchoiceDelivery, synchronize}, + state::EngineSyncStateUpdate, task_queue::synchronize, }; use alloy_eips::BlockId; use async_trait::async_trait; @@ -72,7 +71,6 @@ impl ExecutableEngineTask for FinalizeTask { self.client.as_ref(), state, EngineSyncStateUpdate { finalized_head: Some(block_info), ..Default::default() }, - ForkchoiceDelivery::WhenChanged, ) .await?; diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/insert/task.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/insert/task.rs index 10110e3b622..54d2d380709 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/insert/task.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/insert/task.rs @@ -2,8 +2,7 @@ use crate::{ EngineClient, EngineState, ExecutableEngineTask, ImportedBlockSink, InsertTaskError, - state::EngineSyncStateUpdate, - task_queue::{ForkchoiceDelivery, synchronize}, + state::EngineSyncStateUpdate, task_queue::synchronize, }; use alloy_rpc_types_engine::{ExecutionPayloadInputV2, PayloadStatusEnum}; use async_trait::async_trait; @@ -126,7 +125,6 @@ pub(in crate::task_queue) async fn insert_payload( safe_head: is_payload_safe.then_some(new_unsafe_ref), ..Default::default() }, - ForkchoiceDelivery::WhenChanged, ) .await?; diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/mod.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/mod.rs index c5aa7a8ad47..9eddaa76d00 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/mod.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/mod.rs @@ -7,7 +7,7 @@ pub use task::{ mod synchronize; pub use synchronize::SynchronizeTaskError; -pub(super) use synchronize::{ForkchoiceDelivery, synchronize}; +pub(super) use synchronize::{synchronize, synchronize_unconditionally}; mod insert; pub(super) use insert::insert_payload; diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs index 219416bd6f1..a9bc15146ff 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs @@ -1,61 +1,73 @@ //! The `engine_forkchoiceUpdated` call, with no attributes, that the engine tasks share. -use crate::{EngineClient, EngineState, SynchronizeTaskError, state::EngineSyncStateUpdate}; +use crate::{ + EngineClient, EngineState, EngineSyncState, SynchronizeTaskError, state::EngineSyncStateUpdate, +}; use alloy_rpc_types_engine::{INVALID_FORK_CHOICE_STATE_ERROR, PayloadStatusEnum}; use op_alloy_provider::ext::engine::OpEngineApi; -/// Whether [`synchronize`] may skip a forkchoice update that leaves the consensus labels unchanged. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(in crate::task_queue) enum ForkchoiceDelivery { - /// Send the update even when it leaves the consensus labels unchanged. - /// - /// Unchanged labels do not prove that the execution layer holds their forkchoice: a `SYNCING` - /// answer moves the labels too, and an execution layer that restarts can lose its forkchoice. - Always, - /// Skip the update when it leaves the consensus labels unchanged. - WhenChanged, -} - /// Applies `state_update` to the engine's sync state and synchronizes the execution layer's -/// forkchoice with it, using an `engine_forkchoiceUpdated` call without payload attributes. +/// forkchoice with it, using an `engine_forkchoiceUpdated` call without payload attributes. An +/// update that leaves the labels unchanged is skipped. /// -/// Engine tasks call this whenever they move the unsafe, safe, or finalized head, and the engine -/// calls it during a reset to establish the initial forkchoice state. Forkchoice updates that -/// start a block build are made by `start_build` instead. +/// Engine tasks call this whenever they move the unsafe, safe, or finalized head. Forkchoice +/// updates that start a block build are made by `start_build` instead. #[tracing::instrument( name = "synchronize", target = "engine", level = "info", skip_all, - fields(?delivery, ?state_update, forkchoice = tracing::field::Empty) + fields(?state_update, forkchoice = tracing::field::Empty) )] pub(in crate::task_queue) async fn synchronize( client: &EngineClient, state: &mut EngineState, state_update: EngineSyncStateUpdate, - delivery: ForkchoiceDelivery, ) -> Result<(), SynchronizeTaskError> { - // Apply the sync state update to the engine state. let new_sync_state = state.sync_state.apply_update(state_update); - // Check if a forkchoice update is not needed, return early. - // A forkchoice update is not needed if... - // 1. The caller allows skipping an unchanged forkchoice, - // 2. The engine state is not default (initial forkchoice state has been emitted), and - // 3. The new sync state is the same as the current sync state (no changes to the sync state). + // A forkchoice update is not needed once the initial forkchoice state has been emitted and the + // update leaves the labels unchanged. // // NOTE: // We shouldn't retry the synchronization here. Since the `sync_state` is only updated inside // `synchronize` (except inside the ConsolidateTask, when the block is not the last in the // batch) - the engine would get stuck retrying the synchronization. - if delivery == ForkchoiceDelivery::WhenChanged && - state.sync_state != Default::default() && - state.sync_state == new_sync_state - { + if state.sync_state != Default::default() && state.sync_state == new_sync_state { debug!(target: "engine", ?new_sync_state, "No forkchoice update needed"); return Ok(()); } + send_forkchoice(client, state, new_sync_state).await +} + +/// Like [`synchronize`], but sends the forkchoice update even when it leaves the labels unchanged. +/// +/// Unchanged labels do not prove that the execution layer holds their forkchoice: a `SYNCING` +/// answer moves the labels too, and an execution layer that restarts can lose its forkchoice. +#[tracing::instrument( + name = "synchronize_unconditionally", + target = "engine", + level = "info", + skip_all, + fields(?state_update, forkchoice = tracing::field::Empty) +)] +pub(in crate::task_queue) async fn synchronize_unconditionally( + client: &EngineClient, + state: &mut EngineState, + state_update: EngineSyncStateUpdate, +) -> Result<(), SynchronizeTaskError> { + let new_sync_state = state.sync_state.apply_update(state_update); + send_forkchoice(client, state, new_sync_state).await +} + +/// Sends the forkchoice of `new_sync_state` and, once the execution layer accepts it, makes it the +/// engine's sync state. +async fn send_forkchoice( + client: &EngineClient, + state: &mut EngineState, + new_sync_state: EngineSyncState, +) -> Result<(), SynchronizeTaskError> { // Check if the head is behind the finalized head. if new_sync_state.unsafe_head().block_info.number < new_sync_state.finalized_head().block_info.number diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/mod.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/mod.rs index e73923b3228..9de83da3a5c 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/mod.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/mod.rs @@ -1,7 +1,7 @@ //! The forkchoice engine update shared by the engine tasks, and its error type. mod helper; -pub(in crate::task_queue) use helper::{ForkchoiceDelivery, synchronize}; +pub(in crate::task_queue) use helper::{synchronize, synchronize_unconditionally}; mod error; pub use error::SynchronizeTaskError; diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/tests.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/tests.rs index 1ebc3a56248..64909cb394f 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/tests.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/tests.rs @@ -1,4 +1,4 @@ -use super::{ForkchoiceDelivery, synchronize}; +use super::{synchronize, synchronize_unconditionally}; use crate::{ EngineState, EngineSyncState, EngineSyncStateUpdate, EngineTaskError, EngineTaskErrorSeverity::{self, Reset, Temporary}, @@ -72,16 +72,13 @@ fn expect_reply(l2: &RpcMock, forkchoice: ForkchoiceState, reply: Reply) { } } -/// An update that leaves the labels unchanged is skipped only when the caller allows it. +/// `synchronize` skips an update that leaves the labels unchanged. #[rstest] -#[case::changed_labels_are_sent(true, ForkchoiceDelivery::WhenChanged, 1)] -#[case::unchanged_labels_are_skipped(false, ForkchoiceDelivery::WhenChanged, 0)] -#[case::changed_labels_are_sent_always(true, ForkchoiceDelivery::Always, 1)] -#[case::unchanged_labels_are_sent_always(false, ForkchoiceDelivery::Always, 1)] +#[case::changed_labels_are_sent(true, 1)] +#[case::unchanged_labels_are_skipped(false, 0)] #[tokio::test] -async fn synchronize_skips_unchanged_labels_only_when_allowed( +async fn synchronize_skips_unchanged_labels( #[case] moves_safe_head: bool, - #[case] delivery: ForkchoiceDelivery, #[case] forkchoice_updates: usize, ) { let mut state = engine_state(); @@ -95,7 +92,29 @@ async fn synchronize_skips_unchanged_labels_only_when_allowed( ); } - synchronize(&client, &mut state, update, delivery).await.unwrap(); + synchronize(&client, &mut state, update).await.unwrap(); + + assert_eq!(state.sync_state, expected_sync_state); + l1.assert_finished(); + l2.assert_finished(); +} + +/// `synchronize_unconditionally` sends the update even when it leaves the labels unchanged. +#[rstest] +#[case::changed_labels_are_sent(true)] +#[case::unchanged_labels_are_sent(false)] +#[tokio::test] +async fn synchronize_unconditionally_sends_unchanged_labels(#[case] moves_safe_head: bool) { + let mut state = engine_state(); + let update = state_update(moves_safe_head); + let expected_sync_state = state.sync_state.apply_update(update); + let (client, l1, l2) = test_engine_client(Arc::new(RollupConfig::default())); + l2.expect_forkchoice_updated( + expected_sync_state.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + + synchronize_unconditionally(&client, &mut state, update).await.unwrap(); assert_eq!(state.sync_state, expected_sync_state); l1.assert_finished(); @@ -124,7 +143,7 @@ async fn synchronize_applies_labels_only_when_accepted( let (client, l1, l2) = test_engine_client(Arc::new(RollupConfig::default())); expect_reply(&l2, moved_sync_state.create_forkchoice_state(), reply); - let result = synchronize(&client, &mut state, update, ForkchoiceDelivery::WhenChanged).await; + let result = synchronize(&client, &mut state, update).await; assert_eq!(result.as_ref().err().map(EngineTaskError::severity), error_severity, "{result:?}"); assert_eq!(state.sync_state, expected_sync_state); From e9305c437df6547fbe172cb14d7acafb0082affd Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 12 Nov 2026 14:38:20 +0100 Subject: [PATCH 4/7] kona-engine: inline the finalize task's finality checks `ensure_not_behind_finalized_head` and `ensure_no_conflict_with_finalized_head` each had one caller, so their checks move inline. A new test pins a target above the safe head: it fails with `BlockNotSafe` before any lookup. Co-Authored-By: Claude Opus 5.5 --- .../src/task_queue/tasks/finalize/task.rs | 48 +++++-------------- .../task_queue/tasks/finalize/task_test.rs | 14 ++++++ 2 files changed, 27 insertions(+), 35 deletions(-) diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs index 2099ccd7403..dac3404750a 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs @@ -28,48 +28,21 @@ pub struct FinalizeTask { } impl FinalizeTask { - /// Rejects a target that would move finality backward. - const fn ensure_not_behind_finalized_head( - finalized_head: &L2BlockInfo, - target_number: u64, - ) -> Result<(), FinalizeTaskError> { - if target_number < finalized_head.block_info.number { - return Err(FinalizeTaskError::BehindFinalizedHead { - finalized: finalized_head.block_info.number, - target: target_number, - }); - } - Ok(()) - } - - /// Rejects a different block at the finalized head's height. - /// - /// The default head means finality has not been established yet, so any target, including the - /// configured genesis, may initialize it. - fn ensure_no_conflict_with_finalized_head( - finalized_head: &L2BlockInfo, - target: BlockNumHash, - ) -> Result<(), FinalizeTaskError> { - let finalized = finalized_head.block_info.id(); - if *finalized_head != L2BlockInfo::default() && - target.number == finalized.number && - target.hash != finalized.hash - { - return Err(FinalizeTaskError::ConflictsWithFinalizedHead { finalized, target }); - } - Ok(()) - } - /// Whether `target` is the finalized head itself; a different block at its height is an error. fn is_already_finalized( finalized_head: &L2BlockInfo, target: BlockNumHash, ) -> Result { - if target == finalized_head.block_info.id() { + let finalized = finalized_head.block_info.id(); + if target == finalized { debug!(target: "engine", ?target, "Target is already finalized"); return Ok(true); } - Self::ensure_no_conflict_with_finalized_head(finalized_head, target)?; + // The default head means finality has not been established yet, so any target, including + // the configured genesis, may initialize it. + if *finalized_head != L2BlockInfo::default() && target.number == finalized.number { + return Err(FinalizeTaskError::ConflictsWithFinalizedHead { finalized, target }); + } Ok(false) } @@ -121,7 +94,12 @@ impl ExecutableEngineTask for FinalizeTask { // A lagging source can report an older finalized block, or one from another chain. A // pinned hash is enough to detect either without asking the execution layer. - Self::ensure_not_behind_finalized_head(&finalized_head, target_number)?; + if target_number < finalized_head.block_info.number { + return Err(FinalizeTaskError::BehindFinalizedHead { + finalized: finalized_head.block_info.number, + target: target_number, + }); + } if let FinalizeBlockId::ByHash(target) = self.block_id && Self::is_already_finalized(&finalized_head, target)? { diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs index fc2243dcb6c..fb273f99831 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs @@ -142,6 +142,20 @@ async fn finalize_task_skips_target_without_lookup( assert_eq!(state, original_state); } +/// A target above the safe head is refused before asking the execution layer for the block: the +/// test engine expects no request at all. +#[tokio::test] +async fn finalize_task_refuses_target_above_safe_head_without_lookup() { + let mut state = engine_state(established()); + let original_state = state; + + let err = finalize(by_hash(&canonical(SAFE + 5)), &mut state, |_| {}).await.unwrap_err(); + + assert!(matches!(err, FinalizeTaskError::BlockNotSafe), "{err:?}"); + assert_eq!(err.severity(), EngineTaskErrorSeverity::Critical); + assert_eq!(state, original_state); +} + /// A target whose block differs from the engine's canonical block is skipped. A pinned hash is /// never replaced by the canonical block at its height, even above the finalized head, where a /// delegate on another fork names a block the engine does not consider canonical. From d92d1fcb5e390b99f4b4456541af403af0bb6a5b Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 12 Nov 2026 14:38:21 +0100 Subject: [PATCH 5/7] kona-node-service: report sync completion with the safe head only `ProcessEngineSyncCompletionRequest` and `notify_sync_completed` carry the local-safe head again instead of the whole `EngineSyncState`. Both consumers only read the local-safe head; the delegate actor takes its finalized baseline from the engine's published state. Co-Authored-By: Claude Opus 5.5 --- .../src/actors/derivation/actor/core.rs | 3 +- .../src/actors/derivation/actor/tests.rs | 29 ++++++++++--------- .../actors/derivation/actor/tests/database.rs | 2 +- .../derivation/actor/tests/op_node_parity.rs | 2 +- .../src/actors/derivation/delegated/actor.rs | 4 +-- .../src/actors/derivation/delegated/tests.rs | 4 ++- .../service/src/actors/derivation/request.rs | 7 ++--- .../node/service/src/actors/engine/actor.rs | 2 +- .../node/service/src/actors/engine/client.rs | 16 +++------- .../node/service/src/actors/engine/tests.rs | 2 +- 10 files changed, 32 insertions(+), 39 deletions(-) diff --git a/rust/kona/crates/node/service/src/actors/derivation/actor/core.rs b/rust/kona/crates/node/service/src/actors/derivation/actor/core.rs index 830b7ce4eaf..e3a5ea1ba4d 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/actor/core.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/actor/core.rs @@ -369,9 +369,8 @@ where self.attempt_derivation().await?; } - DerivationActorRequest::ProcessEngineSyncCompletionRequest(sync_state) => { + DerivationActorRequest::ProcessEngineSyncCompletionRequest(safe_head) => { info!(target: "derivation", "Engine finished syncing, starting derivation."); - let safe_head = Box::new(sync_state.local_safe_head()); self.derivation_state_machine .update(&DerivationStateUpdate::ELSyncCompleted(safe_head.clone()))?; self.truncate_safe_db(*safe_head, "EL sync completion")?; diff --git a/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs b/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs index 01eb187e119..5bd4f9cfc00 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs @@ -10,7 +10,7 @@ use kona_derive::{ ActivationSignal, Pipeline, PipelineError, PipelineErrorKind, PipelineResult, ResetError, Signal, SignalReceiver, StepResult, }; -use kona_engine::{EngineSyncState, EngineSyncStateUpdate, FinalizeBlockId}; +use kona_engine::FinalizeBlockId; use kona_genesis::{HardForkConfig, RollupConfig, SystemConfig}; use kona_protocol::{BlockInfo, L2BlockInfo, OpAttributesWithParent}; use kona_safedb::{SafeDatabase, SafeDb, SafeDbError, SafeHeadRecord}; @@ -41,7 +41,12 @@ async fn reset_adopts_safe_head_and_resumes_atomically(#[case] previous_head: u6 l1_block(0).id(), watch::channel(None).0, ); - request_tx.send(engine_sync_completed_at(l2_block(previous_head))).await.unwrap(); + request_tx + .send(DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(l2_block( + previous_head, + )))) + .await + .unwrap(); actor.step().await.unwrap(); request_tx @@ -262,7 +267,10 @@ async fn test_origin_is_published(#[case] step_result: StepResult) { actor.step().await.unwrap(); assert_eq!(derivation_origin_rx.borrow().map(|origin| origin.id()), Some(safe_head.l1_origin)); - request_tx.send(engine_sync_completed_at(safe_head)).await.unwrap(); + request_tx + .send(DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(safe_head))) + .await + .unwrap(); actor.step().await.unwrap(); assert_eq!(derivation_origin_rx.borrow().unwrap().number, 11); @@ -457,16 +465,6 @@ fn l1_block(number: u64) -> BlockInfo { BlockInfo { hash: B256::with_last_byte(number as u8), number, ..Default::default() } } -/// The engine reporting that its sync completed with `safe_head` as its safe heads. -fn engine_sync_completed_at(safe_head: L2BlockInfo) -> DerivationActorRequest { - let sync_state = EngineSyncState::default().apply_update(EngineSyncStateUpdate { - local_safe_head: Some(safe_head), - safe_head: Some(safe_head), - ..Default::default() - }); - DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(sync_state)) -} - fn l2_block(number: u64) -> L2BlockInfo { L2BlockInfo { block_info: BlockInfo { @@ -537,7 +535,10 @@ impl RecordingActor { } async fn complete_el_sync_at(&mut self, head: u64) { - self.handle(engine_sync_completed_at(l2_block(head))).await; + self.handle(DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new( + l2_block(head), + ))) + .await; } async fn confirm_safe_head(&mut self, head: u64) { diff --git a/rust/kona/crates/node/service/src/actors/derivation/actor/tests/database.rs b/rust/kona/crates/node/service/src/actors/derivation/actor/tests/database.rs index eeee778a78a..699bc1e6933 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/actor/tests/database.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/actor/tests/database.rs @@ -135,7 +135,7 @@ fn engine_reset_request() -> DerivationActorRequest { } fn sync_completion_request() -> DerivationActorRequest { - engine_sync_completed_at(l2_block(RESUME_HEAD)) + DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(l2_block(RESUME_HEAD))) } #[rstest] diff --git a/rust/kona/crates/node/service/src/actors/derivation/actor/tests/op_node_parity.rs b/rust/kona/crates/node/service/src/actors/derivation/actor/tests/op_node_parity.rs index 0df6cef81c1..104544f2522 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/actor/tests/op_node_parity.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/actor/tests/op_node_parity.rs @@ -16,7 +16,7 @@ fn l1_head_update(number: u64) -> DerivationActorRequest { } fn el_sync_completed(head: u64) -> DerivationActorRequest { - engine_sync_completed_at(l2_block(head)) + DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(l2_block(head))) } /// A reset whose pipeline signal fails once with a temporary provider error neither stops the diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs index 7001d2f71d6..c7ec925ecd2 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs @@ -269,9 +269,9 @@ where debug!(target: "derivation", safe_head = ?*safe_head, "Received safe head from engine."); self.engine_l2_safe_head = *safe_head; } - DerivationActorRequest::ProcessEngineSyncCompletionRequest(sync_state) => { + DerivationActorRequest::ProcessEngineSyncCompletionRequest(safe_head) => { info!(target: "derivation", "Engine finished syncing, starting derivation."); - self.engine_l2_safe_head = sync_state.local_safe_head(); + self.engine_l2_safe_head = *safe_head; self.has_engine_sync_completed = true; } DerivationActorRequest::ProcessEngineSignalRequest(signal) => match *signal { diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs index 8f976aa256c..0eda07f0beb 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs @@ -70,7 +70,9 @@ impl Harness { async fn complete_engine_sync_with(&mut self, sync_state: EngineSyncState) { self.publish_engine_sync_state(sync_state); self.derivation_requests - .send(DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(sync_state))) + .send(DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new( + sync_state.local_safe_head(), + ))) .await .unwrap(); self.actor.step().await.unwrap(); diff --git a/rust/kona/crates/node/service/src/actors/derivation/request.rs b/rust/kona/crates/node/service/src/actors/derivation/request.rs index caa1e2b9ad2..9482eef5582 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/request.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/request.rs @@ -1,5 +1,4 @@ use kona_derive::Signal; -use kona_engine::EngineSyncState; use kona_protocol::{BlockInfo, L2BlockInfo}; use thiserror::Error; @@ -22,9 +21,9 @@ pub enum DerivationClientError { /// Inbound requests that the [`crate::DerivationActor`] can process. #[derive(Debug)] pub enum DerivationActorRequest { - /// Request to process the fact that Engine sync has completed, along with the engine's sync - /// state at that point. - ProcessEngineSyncCompletionRequest(Box), + /// Request to process the fact that Engine sync has completed, along with the current safe + /// head. + ProcessEngineSyncCompletionRequest(Box), /// Request to process the provided L2 engine safe head update. ProcessEngineSafeHeadUpdateRequest(Box), /// A request containing a [`Signal`] to the derivation pipeline. diff --git a/rust/kona/crates/node/service/src/actors/engine/actor.rs b/rust/kona/crates/node/service/src/actors/engine/actor.rs index 87619ba2756..f09047db8a2 100644 --- a/rust/kona/crates/node/service/src/actors/engine/actor.rs +++ b/rust/kona/crates/node/service/src/actors/engine/actor.rs @@ -251,7 +251,7 @@ where self.el_sync_complete = true; self.derivation_client - .notify_sync_completed(self.engine.state().sync_state) + .notify_sync_completed(self.engine.state().sync_state.local_safe_head()) .await .map(|_| Ok(())) .map_err(|e| { diff --git a/rust/kona/crates/node/service/src/actors/engine/client.rs b/rust/kona/crates/node/service/src/actors/engine/client.rs index 518e7b957d0..25322a5a7f2 100644 --- a/rust/kona/crates/node/service/src/actors/engine/client.rs +++ b/rust/kona/crates/node/service/src/actors/engine/client.rs @@ -2,7 +2,6 @@ use crate::{DerivationActorRequest, DerivationClientError, DerivationClientResul use async_trait::async_trait; use derive_more::Constructor; use kona_derive::Signal; -use kona_engine::EngineSyncState; use kona_protocol::L2BlockInfo; use std::fmt::Debug; use tokio::sync::mpsc; @@ -11,13 +10,9 @@ use tokio::sync::mpsc; #[cfg_attr(test, mockall::automock)] #[async_trait] pub trait EngineDerivationClient: Debug + Send + Sync { - /// Notifies the [`crate::DerivationActor`] that engine syncing has completed, with the engine's - /// sync state at that point. + /// Notifies the [`crate::DerivationActor`] that engine syncing has completed. /// Note: Does not wait for the derivation client to process this message. - async fn notify_sync_completed( - &self, - sync_state: EngineSyncState, - ) -> DerivationClientResult<()>; + async fn notify_sync_completed(&self, safe_head: L2BlockInfo) -> DerivationClientResult<()>; /// Sends the new engine `safe_head` to the [`crate::DerivationActor`]. /// Note: Does not wait for the derivation client to process this message. @@ -38,14 +33,11 @@ pub struct QueuedEngineDerivationClient { #[async_trait] impl EngineDerivationClient for QueuedEngineDerivationClient { - async fn notify_sync_completed( - &self, - sync_state: EngineSyncState, - ) -> DerivationClientResult<()> { + async fn notify_sync_completed(&self, safe_head: L2BlockInfo) -> DerivationClientResult<()> { info!(target: "engine", "Sending sync completed to derivation actor"); self.derivation_actor_request_tx - .send(DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(sync_state))) + .send(DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(safe_head))) .await .map_err(|_| { DerivationClientError::RequestError("request channel closed.".to_string()) diff --git a/rust/kona/crates/node/service/src/actors/engine/tests.rs b/rust/kona/crates/node/service/src/actors/engine/tests.rs index d000710cbf1..b61712d95b5 100644 --- a/rust/kona/crates/node/service/src/actors/engine/tests.rs +++ b/rust/kona/crates/node/service/src/actors/engine/tests.rs @@ -312,5 +312,5 @@ async fn derivation_is_handed_the_local_safe_head() { else { panic!("expected the sync completion second"); }; - assert_eq!(sent.local_safe_head(), local_safe_head); + assert_eq!(*sent, local_safe_head); } From e2bcf65ec2a0e8794c37936808398f103b90f775 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 12 Nov 2026 14:38:22 +0100 Subject: [PATCH 6/7] kona-node-service: reject a delegate status whose finalized is above its safe Such a snapshot is inconsistent: the engine would refuse to finalize it. op-node's follow mode rejects it the same way. A test also pins that any finalized block, genesis included, is accepted before the engine establishes finality, and the doc on the finality check now states that it only catches a fork at the engine's finalized height. Co-Authored-By: Claude Opus 5.5 --- .../src/actors/derivation/delegated/actor.rs | 23 ++++++++++++-- .../src/actors/derivation/delegated/tests.rs | 31 +++++++++++++++++++ 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs index c7ec925ecd2..9962dd9e388 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs @@ -165,6 +165,10 @@ where /// Rejects a finalized L2 block behind, or at the same height as but different from, the /// engine's. Either means the delegate lags or follows another chain, so none of its status /// applies: its safe head would move the engine back or onto that chain too. + /// + /// Only a fork at the engine's finalized height is caught here. A chain that forked below it + /// and reports a higher finalized block passes, and the execution layer rejects the forkchoice + /// update that would move onto it. fn validate_finalized_l2_progress( &self, reported: BlockNumHash, @@ -182,9 +186,20 @@ where Ok(()) } - /// Verifies that the finalized L2 block reported by the derivation delegate does not move back - /// from the engine's, and that its L1 info is consistent with the canonical L1 chain. + /// Verifies that the derivation delegate's finalized L2 block is not above its safe L2 block + /// and does not move back from the engine's, and that its L1 info is consistent with the + /// canonical L1 chain. async fn validate_sync_status(&mut self, v: &SyncStatus) -> bool { + // A finalized block above the safe head is an inconsistent snapshot; the engine would only + // refuse to finalize it. + if v.finalized_l2.block_info.number > v.safe_l2.block_info.number { + let err = DerivationDelegationError::FinalizedAheadOfSafe { + finalized: v.finalized_l2.block_info.number, + safe: v.safe_l2.block_info.number, + }; + warn!(target: "derivation", error = %err, "Inconsistent sync status from delegate"); + return false; + } if let Err(err) = self.validate_finalized_l2_progress(v.finalized_l2.block_info.id()) { warn!( target: "derivation", @@ -291,6 +306,10 @@ where #[derive(Error, Debug)] enum DerivationDelegationError { + /// The delegate's finalized L2 block is above its safe L2 block. + #[error("Finalized L2 {finalized} is ahead of safe L2 {safe}")] + FinalizedAheadOfSafe { finalized: u64, safe: u64 }, + /// The delegate's finalized L2 block is a different block at the engine's finalized height. #[error("Finalized L2 {reported:?} conflicts with the engine's finalized L2 {engine:?}")] FinalizedL2Conflict { engine: BlockNumHash, reported: BlockNumHash }, diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs index 0eda07f0beb..2468e203727 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs @@ -250,3 +250,34 @@ async fn poll_accepts_status_after_engine_skipped_forwarded_finality( assert_eq!(harness.forwarded(), forwarded_for(&canonical_status)); } + +/// Before the engine establishes finality, any finalized block from the delegate may initialize +/// it, including one at genesis height. +#[tokio::test(start_paused = true)] +async fn first_poll_accepts_finality_before_the_engine_establishes_it() { + let status = with_finalized_l2(l2_block(0, FINALIZED_ORIGIN)); + let mut harness = Harness::new([Ok(status.clone())]); + harness.complete_engine_sync().await; + + harness.actor.step().await.unwrap(); + + assert_eq!(harness.forwarded(), forwarded_for(&status)); +} + +/// A status whose finalized block is above its safe block is inconsistent and rejected whole. +#[rstest] +#[case::at_safe_head(l2_block(20, FINALIZED_ORIGIN), true)] +#[case::ahead_of_safe_head(l2_block(21, FINALIZED_ORIGIN), false)] +#[tokio::test(start_paused = true)] +async fn poll_rejects_status_whose_finalized_is_ahead_of_its_safe( + #[case] finalized_l2: L2BlockInfo, + #[case] accepted: bool, +) { + let status = with_finalized_l2(finalized_l2); + let mut harness = Harness::new([Ok(status.clone())]); + harness.complete_engine_sync().await; + + harness.actor.step().await.unwrap(); + + assert_eq!(harness.forwarded(), forwarded_if(accepted, &status)); +} From 5c7d4925978a6aaadb62fd8d98f8bdd6dcdf6e2c Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 12 Nov 2026 14:38:22 +0100 Subject: [PATCH 7/7] kona-node-service: drive a skipped finalization through the engine actor The test sends stale finalizations through the actor: it keeps running, an identical repeat keeps the same skip in memory, and a reset forgets it. Co-Authored-By: Claude Opus 5.5 --- .../service/src/actors/engine/queue_tests.rs | 46 ++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs b/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs index 3dc21d249e1..0b4bc8f58ff 100644 --- a/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs +++ b/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs @@ -4,7 +4,7 @@ use alloy_rpc_types_engine::{ ExecutionPayloadV1, ForkchoiceUpdated, PayloadStatus, PayloadStatusEnum, }; use kona_engine::{ - ConsolidateInput, EngineSyncStateUpdate, NoopBlockSink, + ConsolidateInput, EngineSyncStateUpdate, FinalizeBlockId, NoopBlockSink, test_utils::{TestEngineStateBuilder, l1_info_deposit, test_block_info, test_engine_client}, }; use op_alloy_consensus::OpBlock; @@ -140,3 +140,47 @@ async fn delegated_safe_head_is_sent_on_every_poll() { l1.assert_finished(); l2.assert_finished(); } + +/// A finalization the engine skips leaves the actor running, an identical repeat keeps the same +/// skip in memory, and a reset forgets it. +#[tokio::test(start_paused = true)] +async fn skipped_finalization_keeps_the_actor_running_until_a_reset_forgets_it() { + let config = Arc::new(RollupConfig::default()); + let (client, l1, l2) = test_engine_client(config.clone()); + let state = TestEngineStateBuilder::new() + .with_unsafe_head(test_block_info(30)) + .with_safe_head(test_block_info(20)) + .with_finalized_head(test_block_info(10)) + .build(); + let (state_tx, _state_rx) = watch::channel(state); + let (requests, request_rx) = mpsc::channel(8); + let (derivation_tx, _derivation_rx) = mpsc::channel(8); + let mut actor = EngineActor::new( + Arc::new(client), + config, + QueuedEngineDerivationClient { derivation_actor_request_tx: derivation_tx }, + Engine::new(state, state_tx), + None, + request_rx, + Arc::new(NoopBlockSink), + ); + let stale_finalization = + || EngineActorRequest::ProcessFinalizedL2Block(Box::new(FinalizeBlockId::ByNumber(5))); + + // Each step drains the previous request's task, then accepts the next request. + requests.send(stale_finalization()).await.unwrap(); + time::timeout(Duration::from_secs(1), actor.step()).await.unwrap().unwrap(); + requests.send(stale_finalization()).await.unwrap(); + time::timeout(Duration::from_secs(1), actor.step()).await.unwrap().unwrap(); + let first_skip = actor.last_skipped_task_error.clone(); + assert!(first_skip.is_some(), "the skipped finalization is remembered"); + + requests.send(stale_finalization()).await.unwrap(); + time::timeout(Duration::from_secs(1), actor.step()).await.unwrap().unwrap(); + assert_eq!(actor.last_skipped_task_error, first_skip); + + actor.begin_reset(); + assert_eq!(actor.last_skipped_task_error, None); + l1.assert_finished(); + l2.assert_finished(); +}