diff --git a/rust/kona/crates/node/engine/src/task_queue/core.rs b/rust/kona/crates/node/engine/src/task_queue/core.rs index 816c8d0519e..2628f214302 100644 --- a/rust/kona/crates/node/engine/src/task_queue/core.rs +++ b/rust/kona/crates/node/engine/src/task_queue/core.rs @@ -5,7 +5,7 @@ use crate::{ EngineClient, EngineState, EngineSyncStateUpdate, EngineTask, EngineTaskError, EngineTaskErrorSeverity, Metrics, SyncStartError, SynchronizeTaskError, find_starting_forkchoice, - task_queue::{EngineTaskErrors, synchronize}, + task_queue::{EngineTaskErrors, synchronize_unconditionally}, }; use kona_genesis::RollupConfig; use kona_protocol::L2BlockInfo; @@ -115,8 +115,10 @@ impl Engine { ) -> Result { let start = find_starting_forkchoice(&config, client.as_ref()).await?; - // One attempt. The actor retains the reset request and retries transient failures. - synchronize( + // One attempt. The actor retains the reset request and retries transient failures. The + // execution layer may have lost or changed its forkchoice, so the reset sends one even when + // the recovered labels equal the current ones. + synchronize_unconditionally( client.as_ref(), &mut self.state, EngineSyncStateUpdate { @@ -142,7 +144,7 @@ impl Engine { /// Attempts to drain the queue by executing all [`EngineTask`]s in-order. If any task returns /// an error along the way, the error is returned and the task is not popped from the queue (in - /// case it must be retried), unless it is a flush error. + /// case it must be retried), unless it is a flush or skip error. pub async fn drain(&mut self) -> Result<(), EngineTaskErrors> { // Drain tasks in order of priority, halting on errors for a retry to be attempted. loop { @@ -153,9 +155,13 @@ impl Engine { let result = task.execute(&mut self.state).await; // A flush reports that the task replaced its block with a deposits-only block, so the - // task is done; retrying it would replace that block again. + // task is done; retrying it would replace that block again. A skip reports a task that + // no longer applies, so retrying it cannot succeed either. if let Err(err) = &result && - err.severity() != EngineTaskErrorSeverity::Flush + !matches!( + err.severity(), + EngineTaskErrorSeverity::Flush | EngineTaskErrorSeverity::Skip + ) { return result; } @@ -195,7 +201,8 @@ impl EngineResetError { mod recovery_tests { use super::*; use crate::{ - ConsolidateInput, ConsolidateTask, InsertTask, NoopBlockSink, + ConsolidateInput, ConsolidateTask, FinalizeBlockId, FinalizeTask, InsertTask, + NoopBlockSink, test_utils::{ TestAttributesBuilder, TestEngineStateBuilder, test_block_info, test_engine_client, }, @@ -345,4 +352,63 @@ mod recovery_tests { imported.header.hash_slow() ); } + + // An execution layer that restarted can have lost its forkchoice, so a reset sends the + // recovered forkchoice even when the labels already equal it. + #[tokio::test] + async fn reset_resends_an_unchanged_forkchoice() { + let block: ::BlockResponse = + Default::default(); + let mut config = RollupConfig::default(); + config.genesis.l2.hash = block.header.inner.hash_slow(); + let genesis = + L2BlockInfo::from_block_and_genesis(&block.clone().into_consensus(), &config.genesis) + .unwrap(); + let config = Arc::new(config); + let state = TestEngineStateBuilder::new().with_unsafe_head(genesis).build(); + let forkchoice = state.sync_state.create_forkchoice_state(); + let (client, l1, l2) = test_engine_client(config.clone()); + for tag in ["finalized", "safe", "latest"] { + l2.expect_params("eth_getBlockByNumber", serde_json::json!([tag, true]), &block); + } + l1.expect( + "eth_getBlockByHash", + alloy_rpc_types_eth::Block::::default(), + ); + l2.expect_forkchoice_updated(forkchoice, PayloadStatusEnum::Valid); + let (state_tx, _state_rx) = tokio::sync::watch::channel(state); + let mut engine = Engine::new(state, state_tx); + + engine.reset(Arc::new(client), config).await.unwrap(); + + assert_eq!(engine.state().sync_state, state.sync_state); + l1.assert_finished(); + l2.assert_finished(); + } + + // A skipped task is dropped instead of being kept for a retry, so it cannot hold the queue. + #[tokio::test] + async fn skipped_task_is_dropped_from_the_queue() { + let config = Arc::new(RollupConfig::default()); + let (client, l1, l2) = test_engine_client(config.clone()); + let state = TestEngineStateBuilder::new() + .with_unsafe_head(test_block_info(20)) + .with_finalized_head(test_block_info(10)) + .build(); + let (state_tx, _state_rx) = tokio::sync::watch::channel(state); + let mut engine = Engine::new(state, state_tx); + engine.enqueue(EngineTask::Finalize(Box::new(FinalizeTask::new( + Arc::new(client), + config, + FinalizeBlockId::ByNumber(5), + )))); + + let error = engine.drain().await.unwrap_err(); + + assert_eq!(error.severity(), EngineTaskErrorSeverity::Skip, "got {error:?}"); + assert!(engine.is_empty()); + assert_eq!(engine.state().sync_state, state.sync_state); + l1.assert_finished(); + l2.assert_finished(); + } } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/canonicalize_test.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/canonicalize_test.rs index 9da42da2fe8..f8d73461107 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/canonicalize_test.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/canonicalize_test.rs @@ -1,11 +1,14 @@ use super::*; -use crate::{InsertTaskError, NoopBlockSink, SynchronizeTaskError, test_utils::test_engine_client}; +use crate::{ + InsertTaskError, NoopBlockSink, SynchronizeTaskError, + test_utils::{l1_info_deposit, test_engine_client}, +}; use alloy_consensus::{Block, SignableTransaction, TxEip1559}; -use alloy_primitives::{Sealed, Signature}; +use alloy_primitives::Signature; use alloy_rpc_types_engine::{ ExecutionPayloadV1, ForkchoiceUpdated, PayloadStatus, PayloadStatusEnum, }; -use op_alloy_consensus::{OpTxEnvelope, TxDeposit}; +use op_alloy_consensus::OpTxEnvelope; #[derive(Debug, Default)] pub(crate) struct RecordingSink(pub(crate) std::sync::Mutex>); @@ -106,10 +109,7 @@ async fn invalid_committed_payload_does_not_build_an_uncommitted_replacement( ) { let PayloadFixture { cfg, .. } = payload_fixture(); let mut block = Block::::default(); - block.body.transactions.push(OpTxEnvelope::Deposit(Sealed::new(TxDeposit { - input: kona_protocol::L1BlockInfoBedrock::default().encode_calldata(), - ..Default::default() - }))); + block.body.transactions.push(l1_info_deposit()); if !deposits_only { block.body.transactions.push(OpTxEnvelope::Eip1559( TxEip1559::default().into_signed(Signature::test_signature()), diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs index a9ada74d3c0..d3183cdafa5 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs @@ -3,7 +3,7 @@ use crate::{ ConsolidateTaskError, EngineClient, EngineState, ExecutableEngineTask, ImportedBlockSink, state::EngineSyncStateUpdate, - task_queue::{build_and_import, synchronize}, + task_queue::{build_and_import, synchronize, synchronize_unconditionally}, }; use alloy_rpc_types_eth::Block; use async_trait::async_trait; @@ -52,13 +52,9 @@ impl ConsolidateInput { } } - /// Returns true if this is `Attributes` and `attributes.is_last_in_span` is true. - const fn is_attributes_last_in_span(&self) -> bool { - matches!( - self, - Self::Attributes(attributes) - if attributes.is_last_in_span - ) + /// Whether these are derived attributes for a block before the last block of its span batch. + const fn is_intermediate_span_block(&self) -> bool { + matches!(self, Self::Attributes(attributes) if !attributes.is_last_in_span) } } @@ -104,48 +100,6 @@ impl ConsolidateTask { Ok(()) } - /// This provides symmetric fallback behavior with `build_and_import`. - async fn reconcile_to_safe_head( - &self, - state: &mut EngineState, - safe_l2: &L2BlockInfo, - ) -> Result<(), ConsolidateTaskError> { - warn!( - target: "engine", - safe_l2 = %safe_l2, - "Apply safe head" - ); - - // We intentionally set the unsafe head to safe_l2 to ensure the engine observes a - // self-consistent head state. This is required to correctly handle reorgs (where unsafe - // may be ahead on a non-canonical fork) and to trigger EL sync when the local unsafe head - // lags behind the safe head. - synchronize( - self.client.as_ref(), - state, - EngineSyncStateUpdate { - unsafe_head: Some(*safe_l2), - safe_head: Some(*safe_l2), - local_safe_head: Some(*safe_l2), - ..Default::default() - }, - ) - .await - .map_err(|e| { - warn!(target: "engine", ?e, "Apply safe head failed"); - e - })?; - - info!( - target: "engine", - hash = %safe_l2.block_info.hash, - number = safe_l2.block_info.number, - "Updated safe head via follow safe" - ); - - Ok(()) - } - /// Handles the fallback case when the block doesn't match the input or does not exist. async fn reconcile_unsafe_to_safe( &self, @@ -156,7 +110,37 @@ impl ConsolidateTask { self.build_and_import_attributes(state, attributes).await } ConsolidateInput::BlockInfo(safe_l2) => { - self.reconcile_to_safe_head(state, safe_l2).await + warn!(target: "engine", safe_l2 = %safe_l2, "Apply safe head"); + + // We intentionally set the unsafe head to safe_l2 to ensure the engine observes a + // self-consistent head state. This is required to correctly handle reorgs (where + // unsafe may be ahead on a non-canonical fork) and to trigger EL sync when the + // local unsafe head lags behind the safe head. The forkchoice update is always + // sent: the execution layer may have answered `SYNCING` to the same labels before, + // or lost them in a restart. + synchronize_unconditionally( + self.client.as_ref(), + state, + EngineSyncStateUpdate { + local_safe_head: Some(*safe_l2), + safe_head: Some(*safe_l2), + unsafe_head: Some(*safe_l2), + ..Default::default() + }, + ) + .await + .map_err(|e| { + warn!(target: "engine", ?e, safe_l2 = %safe_l2, "Applying the delegated safe head failed"); + e + })?; + + info!( + target: "engine", + hash = %safe_l2.block_info.hash, + number = safe_l2.block_info.number, + "Updated safe head via follow safe" + ); + Ok(()) } } } @@ -175,6 +159,41 @@ impl ConsolidateTask { let block = match self.client.l2_block_by_label(block_num.into()).await { Ok(Some(block)) => block, Ok(None) => { + if let ConsolidateInput::BlockInfo(safe_l2) = &self.input { + // A block missing below the unsafe head means the execution layer is still + // syncing toward the unsafe head. Resetting would restart that sync from the + // execution layer's old labels, and moving the unsafe head back would retarget + // it. Like op-node's follow-source mode, only the safe labels follow the + // delegate, and the forkchoice update, sent even when unchanged, carries them + // to the execution layer. + warn!( + target: "engine", + safe_l2 = %safe_l2, + unsafe_number = state.sync_state.unsafe_head().block_info.number, + "Execution layer does not have the delegated safe head yet; keeping its sync target" + ); + synchronize_unconditionally( + self.client.as_ref(), + state, + EngineSyncStateUpdate { + local_safe_head: Some(*safe_l2), + safe_head: Some(*safe_l2), + ..Default::default() + }, + ) + .await + .map_err(|e| { + warn!(target: "engine", ?e, safe_l2 = %safe_l2, "Applying the delegated safe head failed"); + e + })?; + info!( + target: "engine", + hash = %safe_l2.block_info.hash, + number = safe_l2.block_info.number, + "Updated safe head while the execution layer syncs" + ); + return Ok(()); + } warn!(target: "engine", "Received `None` block for {}", block_num); return Err(ConsolidateTaskError::MissingUnsafeL2Block(block_num)); } @@ -194,10 +213,10 @@ impl ConsolidateTask { let consensus_block = block.into_consensus().map_transactions(|t| t.inner.inner.into_inner()); match L2BlockInfo::from_block_and_genesis(&consensus_block, &self.cfg.genesis) { - // Only issue a forkchoice update if the attributes are the last in the span - // batch. This is an optimization to avoid sending a FCU - // call for every block in the span batch. - Ok(block_info) if !self.input.is_attributes_last_in_span() => { + // A span batch's blocks before its last one only move the safe label; the last + // block's forkchoice update delivers them all. This avoids sending a FCU call for + // every block in the span batch. + Ok(block_info) if self.input.is_intermediate_span_block() => { // The next attributes built are this block's child, and ask for its config. self.block_sink.block_imported(consensus_block, block_info); @@ -221,17 +240,24 @@ impl ConsolidateTask { // The next attributes built are this block's child, and ask for its config. self.block_sink.block_imported(consensus_block, block_info); - synchronize( - self.client.as_ref(), - state, - EngineSyncStateUpdate { - safe_head: Some(block_info), - local_safe_head: Some(block_info), - ..Default::default() - }, - ) - .await - .map_err(|e| { + let update = EngineSyncStateUpdate { + safe_head: Some(block_info), + local_safe_head: Some(block_info), + ..Default::default() + }; + let synchronized = match &self.input { + // Derived attributes move the safe head on every consolidation, so an + // unchanged forkchoice was already sent. + ConsolidateInput::Attributes(_) => { + synchronize(self.client.as_ref(), state, update).await + } + // A delegated safe head can repeat a forkchoice the execution layer + // answered `SYNCING` or lost, so it is always sent. + ConsolidateInput::BlockInfo(_) => { + synchronize_unconditionally(self.client.as_ref(), state, update).await + } + }; + synchronized.map_err(|e| { warn!(target: "engine", ?e, "Consolidation failed"); e })?; @@ -297,10 +323,16 @@ impl ExecutableEngineTask for ConsolidateTask { mod tests { use super::*; use crate::{ - EngineTaskError, ImportedBlockSink, SealTaskError, test_utils::test_engine_client, + EngineSyncState, EngineTaskError, ImportedBlockSink, NoopBlockSink, SealTaskError, + test_utils::{ + TestAttributesBuilder, TestEngineStateBuilder, l1_info_deposit, test_block_info, + test_engine_client, + }, }; - use alloy_eips::BlockNumHash; - use alloy_primitives::B256; + use alloy_consensus::transaction::Recovered; + use alloy_eips::{BlockNumHash, Encodable2718}; + use alloy_primitives::{Address, B256}; + use alloy_rpc_types_engine::{ForkchoiceUpdated, PayloadStatus, PayloadStatusEnum}; use alloy_rpc_types_eth::{Block as RpcBlock, BlockTransactions, Header as RpcHeader}; use kona_genesis::ChainGenesis; use kona_protocol::BlockInfo; @@ -320,21 +352,14 @@ mod tests { // Consolidation reads the unsafe block in full to compare it against the derived // attributes; the next attributes built ask for this same block's system config, so it // must reach the sink rather than being dropped. - let header = RpcHeader::new(alloy_consensus::Header::default()); - let block_hash = header.hash; - let block: RpcBlock = - RpcBlock::new(header, BlockTransactions::Full(vec![])); - - // Pin genesis to this block so its L2BlockInfo needs no L1-info deposit. - let cfg = Arc::new(RollupConfig { - genesis: ChainGenesis { - l2: BlockNumHash { hash: block_hash, number: 0 }, - ..Default::default() - }, - ..Default::default() - }); + let (block, cfg) = genesis_pinned_block(0); + let block_hash = block.header.hash; let (client, l1, l2) = test_engine_client(cfg.clone()); l2.expect_params("eth_getBlockByNumber", serde_json::json!(["0x0", true]), block); + l2.expect( + "engine_forkchoiceUpdatedV3", + ForkchoiceUpdated::new(PayloadStatus::from_status(PayloadStatusEnum::Valid)), + ); let client = Arc::new(client); let safe_l2 = L2BlockInfo { @@ -479,4 +504,192 @@ mod tests { "got {err:?}" ); } + + /// An engine state with the unsafe head at 20, the finalized head at 5 and `safe_head` between. + fn engine_state_with_safe_head(safe_head: L2BlockInfo) -> EngineState { + TestEngineStateBuilder::new() + .with_unsafe_head(test_block_info(20)) + .with_safe_head(safe_head) + .with_finalized_head(test_block_info(5)) + .build() + } + + fn with_safe_head(state: &EngineState, safe_head: L2BlockInfo) -> EngineSyncState { + state.sync_state.apply_update(EngineSyncStateUpdate { + local_safe_head: Some(safe_head), + safe_head: Some(safe_head), + ..Default::default() + }) + } + + fn block_info_of(cfg: &RollupConfig, block: &RpcBlock) -> L2BlockInfo { + let consensus_block = + block.clone().into_consensus().map_transactions(|t| t.inner.inner.into_inner()); + L2BlockInfo::from_block_and_genesis(&consensus_block, &cfg.genesis).unwrap() + } + + /// An empty block at `number`, with genesis pinned to it so its `L2BlockInfo` needs no L1-info + /// deposit. + fn genesis_pinned_block(number: u64) -> (RpcBlock, Arc) { + let header = RpcHeader::new(alloy_consensus::Header { number, ..Default::default() }); + let cfg = Arc::new(RollupConfig { + genesis: ChainGenesis { + l2: BlockNumHash { hash: header.hash, number }, + ..Default::default() + }, + ..Default::default() + }); + (RpcBlock::new(header, BlockTransactions::Full(vec![])), cfg) + } + + /// What the execution layer answers for the block at the delegated safe head's height. + fn el_block(el_has_block: bool, block: RpcBlock) -> Option> { + el_has_block.then_some(block) + } + + /// The safe head before consolidation: either already the delegated target, as a `SYNCING` + /// answer or an earlier poll leaves it, or an older block. + fn initial_safe_head(starts_at_target: bool, target: L2BlockInfo) -> L2BlockInfo { + if starts_at_target { target } else { test_block_info(5) } + } + + /// Below the unsafe head, a delegated safe head always reaches the execution layer: when the + /// labels already hold it, and when the execution layer is still syncing and does not have its + /// block yet. A missing block then moves only the safe labels instead of resetting, and only a + /// block the execution layer has reaches the block sink. + #[rstest::rstest] + #[case::new_target(true, false, PayloadStatusEnum::Valid)] + #[case::repeated_target(true, true, PayloadStatusEnum::Valid)] + #[case::new_target_missing_from_syncing_el(false, false, PayloadStatusEnum::Syncing)] + #[case::repeated_target_missing_from_syncing_el(false, true, PayloadStatusEnum::Syncing)] + #[tokio::test] + async fn delegated_safe_head_below_unsafe_head_is_always_sent( + #[case] el_has_block: bool, + #[case] starts_at_target: bool, + #[case] reply: PayloadStatusEnum, + ) { + let (block, cfg) = genesis_pinned_block(10); + let delegated_safe_head = block_info_of(&cfg, &block); + let mut state = + engine_state_with_safe_head(initial_safe_head(starts_at_target, delegated_safe_head)); + let expected_sync_state = with_safe_head(&state, delegated_safe_head); + let (client, l1, l2) = test_engine_client(cfg.clone()); + l2.expect_params( + "eth_getBlockByNumber", + serde_json::json!(["0xa", true]), + el_block(el_has_block, block), + ); + l2.expect_forkchoice_updated(expected_sync_state.create_forkchoice_state(), reply); + let sink = Arc::new(RecordingSink::default()); + let task = ConsolidateTask::new( + Arc::new(client), + cfg, + ConsolidateInput::BlockInfo(delegated_safe_head), + sink.clone(), + ); + + task.execute(&mut state).await.unwrap(); + + assert_eq!(state.sync_state, expected_sync_state); + assert_eq!(sink.0.lock().unwrap().len(), usize::from(el_has_block)); + l1.assert_finished(); + l2.assert_finished(); + } + + /// Unlike a delegated safe head, derived attributes whose block is missing below the unsafe + /// head still reset the engine. + #[tokio::test] + async fn derived_attributes_missing_from_el_reset_the_engine() { + let cfg = Arc::new(RollupConfig::default()); + let attributes = derived_attributes(true); + let mut state = engine_state_with_safe_head(attributes.parent); + let state_before = state; + let (client, l1, l2) = test_engine_client(cfg.clone()); + l2.expect_params( + "eth_getBlockByNumber", + serde_json::json!(["0xa", true]), + None::>, + ); + let task = ConsolidateTask::new( + Arc::new(client), + cfg, + ConsolidateInput::Attributes(Box::new(attributes)), + Arc::new(NoopBlockSink), + ); + + let err = task.execute(&mut state).await.unwrap_err(); + + assert!(matches!(err, ConsolidateTaskError::MissingUnsafeL2Block(10)), "got {err:?}"); + assert_eq!(err.severity(), crate::EngineTaskErrorSeverity::Reset); + assert_eq!(state, state_before); + l1.assert_finished(); + l2.assert_finished(); + } + + fn derived_attributes(is_last_in_span: bool) -> OpAttributesWithParent { + let mut attributes = TestAttributesBuilder::new() + .with_parent(test_block_info(9)) + .with_transactions(vec![l1_info_deposit().encoded_2718().into()]) + .build(); + attributes.is_last_in_span = is_last_in_span; + attributes + } + + /// The unsafe block that `attributes` derive, so consolidation finds it consistent. + fn unsafe_block_matching(attributes: &OpAttributesWithParent) -> RpcBlock { + let payload_attributes = &attributes.attributes.payload_attributes; + let header = alloy_consensus::Header { + beneficiary: payload_attributes.suggested_fee_recipient, + gas_limit: attributes.attributes.gas_limit.unwrap(), + mix_hash: payload_attributes.prev_randao, + number: attributes.block_number(), + parent_beacon_block_root: payload_attributes.parent_beacon_block_root, + parent_hash: attributes.parent.block_info.hash, + timestamp: payload_attributes.timestamp, + ..Default::default() + }; + let deposit = Transaction::from_transaction( + Recovered::new_unchecked(l1_info_deposit(), Address::ZERO), + Default::default(), + ); + RpcBlock::new(RpcHeader::new(header), BlockTransactions::Full(vec![deposit])) + } + + /// Derived blocks inside a span batch only move the safe label; the span's last block sends + /// the forkchoice update that delivers it. + #[rstest::rstest] + #[case::intermediate_block_defers_forkchoice(false, 0)] + #[case::last_block_sends_forkchoice(true, 1)] + #[tokio::test] + async fn span_batch_sends_forkchoice_only_at_its_last_block( + #[case] is_last_in_span: bool, + #[case] forkchoice_updates: usize, + ) { + let cfg = Arc::new(RollupConfig::default()); + let attributes = derived_attributes(is_last_in_span); + let block = unsafe_block_matching(&attributes); + let derived_safe_head = block_info_of(&cfg, &block); + let mut state = engine_state_with_safe_head(attributes.parent); + let expected_sync_state = with_safe_head(&state, derived_safe_head); + let (client, l1, l2) = test_engine_client(cfg.clone()); + l2.expect_params("eth_getBlockByNumber", serde_json::json!(["0xa", true]), block); + for _ in 0..forkchoice_updates { + l2.expect_forkchoice_updated( + expected_sync_state.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + } + let task = ConsolidateTask::new( + Arc::new(client), + cfg, + ConsolidateInput::Attributes(Box::new(attributes)), + Arc::new(NoopBlockSink), + ); + + task.execute(&mut state).await.unwrap(); + + assert_eq!(state.sync_state, expected_sync_state); + l1.assert_finished(); + l2.assert_finished(); + } } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/error.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/error.rs index c8444c2595e..51a7f2c1dda 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/error.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/error.rs @@ -3,6 +3,7 @@ use crate::{ EngineTaskError, SynchronizeTaskError, task_queue::tasks::task::EngineTaskErrorSeverity, }; +use alloy_eips::BlockNumHash; use alloy_transport::{RpcError, TransportErrorKind}; use kona_protocol::FromBlockError; use thiserror::Error; @@ -10,33 +11,69 @@ use thiserror::Error; /// An error that occurs when running the [`crate::FinalizeTask`]. #[derive(Debug, Error)] pub enum FinalizeTaskError { - /// The block is not safe, and therefore cannot be finalized. - #[error("Attempted to finalize a block that is not yet safe")] - BlockNotSafe, + /// The target is below the finalized head, and finality never moves backward. + #[error("Finalization target {target} is behind the finalized head {finalized}")] + BehindFinalizedHead { + /// The number of the finalized head. + finalized: u64, + /// The number of the finalization target. + target: u64, + }, /// The block to finalize was not found. #[error("The block to finalize was not found: Number {0}")] BlockNotFound(u64), + /// The block is not safe, and therefore cannot be finalized. + #[error("Attempted to finalize a block that is not yet safe")] + BlockNotSafe, + /// The engine returned a block at a different height than the one requested. + #[error("Requested block {requested} to finalize, but the engine returned block {returned}")] + BlockNumberMismatch { + /// The requested block number. + requested: u64, + /// The number of the block the engine returned. + returned: u64, + }, + /// The target is a different block at the finalized head's height. + #[error("Finalization target {target:?} conflicts with the finalized head {finalized:?}")] + ConflictsWithFinalizedHead { + /// The finalized head. + finalized: BlockNumHash, + /// The finalization target. + target: BlockNumHash, + }, + /// The forkchoice update call to finalize the block failed. + #[error(transparent)] + ForkchoiceUpdateFailed(#[from] SynchronizeTaskError), /// An error occurred while transforming the RPC block into [`L2BlockInfo`]. /// /// [`L2BlockInfo`]: kona_protocol::L2BlockInfo #[error(transparent)] FromBlock(#[from] FromBlockError), + /// The pinned target is not the engine's canonical block at its height. + #[error("Finalization target {target:?} is not canonical; the engine has {canonical:?}")] + NotCanonical { + /// The engine's canonical block at the target's height. + canonical: BlockNumHash, + /// The finalization target. + target: BlockNumHash, + }, /// A temporary RPC failure. #[error(transparent)] TransportError(#[from] RpcError), - /// The forkchoice update call to finalize the block failed. - #[error(transparent)] - ForkchoiceUpdateFailed(#[from] SynchronizeTaskError), } impl EngineTaskError for FinalizeTaskError { fn severity(&self) -> EngineTaskErrorSeverity { match self { - Self::BlockNotSafe | Self::BlockNotFound(_) | Self::FromBlock(_) => { - EngineTaskErrorSeverity::Critical - } - Self::TransportError(_) => EngineTaskErrorSeverity::Temporary, + Self::BehindFinalizedHead { .. } | + Self::ConflictsWithFinalizedHead { .. } | + Self::NotCanonical { .. } => EngineTaskErrorSeverity::Skip, + Self::BlockNotFound(_) | + Self::BlockNotSafe | + Self::BlockNumberMismatch { .. } | + Self::FromBlock(_) => EngineTaskErrorSeverity::Critical, Self::ForkchoiceUpdateFailed(inner) => inner.severity(), + Self::TransportError(_) => EngineTaskErrorSeverity::Temporary, } } } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/id.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/id.rs index ab5a92e4ddb..c7c12fd6e73 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/id.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/id.rs @@ -10,9 +10,9 @@ use alloy_eips::BlockNumHash; /// authoritative source for the block at a given height (e.g. local L1-finality driven by /// `L2Finalizer`). The number is sufficient because there is only one chain to consult. /// - [`FinalizeBlockId::ByHash`] is used when an upstream supplies `(number, hash)` and the engine -/// must finalize the specific block identified by hash. If the engine does not have that hash, -/// the task must error rather than silently finalize the engine's canonical block at the same -/// height. +/// must finalize the specific block identified by hash. If the engine's canonical block at that +/// height has a different hash, the task skips the target rather than finalize the engine's block +/// in its place. /// /// EL finalization is irreversible, so finalizing the wrong block is unrecoverable. Forcing every /// caller to pick a variant prevents accidental hash-loss at the boundary. @@ -20,8 +20,8 @@ use alloy_eips::BlockNumHash; pub enum FinalizeBlockId { /// Finalize whatever block the engine has at the given L2 block number. ByNumber(u64), - /// Finalize the block matching the given `(number, hash)` pair. The task fails if the engine - /// does not have that hash. + /// Finalize the block matching the given `(number, hash)` pair, only once it is the engine's + /// canonical block at that height. ByHash(BlockNumHash), } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs index 3461d7d860f..dac3404750a 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs @@ -4,7 +4,7 @@ use crate::{ EngineClient, EngineState, ExecutableEngineTask, FinalizeBlockId, FinalizeTaskError, state::EngineSyncStateUpdate, task_queue::synchronize, }; -use alloy_eips::BlockId; +use alloy_eips::BlockNumHash; use async_trait::async_trait; use derive_more::Constructor; use kona_genesis::RollupConfig; @@ -13,6 +13,10 @@ use std::sync::Arc; /// The [`FinalizeTask`] fetches the [`L2BlockInfo`] identified by `block_id`, updates the /// [`EngineState`], and dispatches a forkchoice update to finalize the block. +/// +/// Finality only moves forward. A target behind the finalized head, a different block at the +/// finalized height, or a pinned hash that is not the engine's canonical block fails with an error +/// that skips the task and leaves the engine state unchanged. #[derive(Debug, Clone, Constructor)] pub struct FinalizeTask { /// The engine client. @@ -23,6 +27,61 @@ pub struct FinalizeTask { pub block_id: FinalizeBlockId, } +impl FinalizeTask { + /// Whether `target` is the finalized head itself; a different block at its height is an error. + fn is_already_finalized( + finalized_head: &L2BlockInfo, + target: BlockNumHash, + ) -> Result { + let finalized = finalized_head.block_info.id(); + if target == finalized { + debug!(target: "engine", ?target, "Target is already finalized"); + return Ok(true); + } + // The default head means finality has not been established yet, so any target, including + // the configured genesis, may initialize it. + if *finalized_head != L2BlockInfo::default() && target.number == finalized.number { + return Err(FinalizeTaskError::ConflictsWithFinalizedHead { finalized, target }); + } + Ok(false) + } + + /// Fetches the engine's canonical block at the target's height. + /// + /// A pinned hash must be that block: a forkchoice update cannot finalize a block off the + /// canonical chain, and finalizing the canonical block instead would finalize a block the + /// caller never named. The block's identity is checked before it is decoded, so a mismatch is + /// reported as such rather than as a decoding failure. + async fn fetch_canonical_block_info(&self) -> Result { + let requested = self.block_id.number(); + debug!(target: "engine", block_id = ?self.block_id, "Fetching the block to finalize"); + let block = self + .client + .get_l2_block(requested.into()) + .full() + .await + .map_err(FinalizeTaskError::TransportError)? + .ok_or(FinalizeTaskError::BlockNotFound(requested))?; + let canonical = BlockNumHash { hash: block.header.hash, number: block.header.number }; + debug!(target: "engine", ?canonical, "Fetched the block to finalize"); + + if canonical.number != requested { + return Err(FinalizeTaskError::BlockNumberMismatch { + requested, + returned: canonical.number, + }); + } + if let FinalizeBlockId::ByHash(target) = self.block_id && + target.hash != canonical.hash + { + return Err(FinalizeTaskError::NotCanonical { canonical, target }); + } + + L2BlockInfo::from_block_and_genesis(&block.into_consensus(), &self.cfg.genesis) + .map_err(FinalizeTaskError::FromBlock) + } +} + #[async_trait] impl ExecutableEngineTask for FinalizeTask { type Output = (); @@ -30,40 +89,35 @@ impl ExecutableEngineTask for FinalizeTask { type Error = FinalizeTaskError; async fn execute(&self, state: &mut EngineState) -> Result<(), FinalizeTaskError> { - let block_number = self.block_id.number(); + let finalized_head = state.sync_state.finalized_head(); + let target_number = self.block_id.number(); + + // A lagging source can report an older finalized block, or one from another chain. A + // pinned hash is enough to detect either without asking the execution layer. + if target_number < finalized_head.block_info.number { + return Err(FinalizeTaskError::BehindFinalizedHead { + finalized: finalized_head.block_info.number, + target: target_number, + }); + } + if let FinalizeBlockId::ByHash(target) = self.block_id && + Self::is_already_finalized(&finalized_head, target)? + { + return Ok(()); + } // Sanity check that the block that is being finalized is at least safe. - if state.sync_state.safe_head().block_info.number < block_number { + if state.sync_state.safe_head().block_info.number < target_number { return Err(FinalizeTaskError::BlockNotSafe); } - // Look up by hash when the caller pinned a specific hash (delegated polling supplies - // `(number, hash)`); otherwise look up by number. Finalization is irreversible, so a stale - // hash must produce a hard error, not silently finalize whatever the engine has at the - // same height. - let lookup: BlockId = match self.block_id { - FinalizeBlockId::ByHash(id) => id.hash.into(), - FinalizeBlockId::ByNumber(n) => n.into(), - }; - let block = self - .client - .get_l2_block(lookup) - .full() - .await - .map_err(FinalizeTaskError::TransportError)? - .ok_or(FinalizeTaskError::BlockNotFound(block_number))? - .into_consensus(); - let block_info = L2BlockInfo::from_block_and_genesis(&block, &self.client.cfg().genesis) - .map_err(FinalizeTaskError::FromBlock)?; - - // For ByHash, also assert the returned block's height matches the caller's claim. The - // engine should never serve a block at a different height for a given hash, but a height - // mismatch indicates either an upstream protocol bug or a misuse of the API and must - // never lead to silent finalization. - if let FinalizeBlockId::ByHash(id) = self.block_id && - block_info.block_info.number != id.number + let block_info = self.fetch_canonical_block_info().await?; + + // A by-number target only learns its hash from the lookup. + if matches!(self.block_id, FinalizeBlockId::ByNumber(_)) && + Self::is_already_finalized(&finalized_head, block_info.block_info.id())? { - return Err(FinalizeTaskError::BlockNotFound(id.number)); + return Ok(()); } // Dispatch a forkchoice update. diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs index 08c4f153e37..fb273f99831 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs @@ -1,62 +1,263 @@ use crate::{ + EngineState, EngineSyncState, EngineSyncStateUpdate, EngineTaskError, EngineTaskErrorSeverity, ExecutableEngineTask, FinalizeBlockId, FinalizeTask, FinalizeTaskError, - test_utils::{TestEngineStateBuilder, test_engine_client}, + test_utils::{ + RpcMock, TestEngineStateBuilder, TestRpcBlock as RpcBlock, test_block_info, + test_engine_client, test_l2_block, test_rpc_block, + }, }; +use alloy_consensus::Header; use alloy_eips::BlockNumHash; -use alloy_primitives::b256; -use kona_genesis::RollupConfig; -use kona_protocol::{BlockInfo, L2BlockInfo}; +use alloy_primitives::B256; +use alloy_rpc_types_engine::PayloadStatusEnum; +use alloy_rpc_types_eth::{BlockTransactions, Header as RpcHeader}; +use kona_genesis::{ChainGenesis, RollupConfig}; +use kona_protocol::L2BlockInfo; +use rstest::rstest; use std::sync::Arc; -/// When the engine receives a `ByHash` finalize request for a block hash it doesn't have, -/// [`FinalizeTask`] must fail with [`FinalizeTaskError::BlockNotFound`] rather than silently -/// finalize whatever it happens to have at the same height. -/// -/// Assert the exact by-hash RPC lookup, so a by-number lookup cannot silently finalize a -/// different canonical block at the same height. -#[tokio::test] -async fn finalize_task_by_hash_errors_when_engine_lacks_hash() { - const N: u64 = 10; - let hash_a = b256!("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); - let hash_b = b256!("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"); +const STALE: u64 = 5; +const FINALIZED: u64 = 10; +const ADVANCED: u64 = 15; +const SAFE: u64 = 20; +const UNSAFE: u64 = 30; + +/// The L2 genesis block, which carries no L1-info deposit. +fn genesis_block() -> RpcBlock { + RpcBlock::new(RpcHeader::new(Header::default()), BlockTransactions::Full(vec![])) +} + +fn rollup_config() -> Arc { + let genesis = BlockNumHash { hash: genesis_block().header.hash, number: 0 }; + Arc::new(RollupConfig { + genesis: ChainGenesis { l2: genesis, ..Default::default() }, + ..Default::default() + }) +} + +/// An L2 block at `number`. Blocks built from different `fork` values share a height but not a +/// hash. +fn l2_block(number: u64, fork: u8) -> RpcBlock { + test_rpc_block(&test_l2_block(number, B256::ZERO, fork)) +} + +fn canonical(number: u64) -> RpcBlock { + l2_block(number, 0) +} + +fn conflicting(number: u64) -> RpcBlock { + l2_block(number, 1) +} + +fn l2_block_info(block: &RpcBlock) -> L2BlockInfo { + L2BlockInfo::from_block_and_genesis(&block.clone().into_consensus(), &rollup_config().genesis) + .unwrap() +} + +fn by_hash(block: &RpcBlock) -> FinalizeBlockId { + FinalizeBlockId::ByHash(l2_block_info(block).block_info.id()) +} + +fn by_number(block: &RpcBlock) -> FinalizeBlockId { + FinalizeBlockId::ByNumber(block.header.number) +} - let cfg = Arc::new(RollupConfig::default()); +fn established() -> L2BlockInfo { + l2_block_info(&canonical(FINALIZED)) +} + +fn uninitialized() -> L2BlockInfo { + L2BlockInfo::default() +} + +fn engine_state(finalized_head: L2BlockInfo) -> EngineState { + TestEngineStateBuilder::new() + .with_unsafe_head(test_block_info(UNSAFE)) + .with_safe_head(test_block_info(SAFE)) + .with_finalized_head(finalized_head) + .build() +} + +fn with_finalized_head(state: &EngineState, finalized_head: L2BlockInfo) -> EngineSyncState { + state.sync_state.apply_update(EngineSyncStateUpdate { + finalized_head: Some(finalized_head), + ..Default::default() + }) +} + +async fn finalize( + target: FinalizeBlockId, + state: &mut EngineState, + configure_engine: impl FnOnce(&RpcMock), +) -> Result<(), FinalizeTaskError> { + let cfg = rollup_config(); let (engine_client, l1, l2) = test_engine_client(cfg.clone()); - l2.expect_params( - "eth_getBlockByHash", - serde_json::json!([hash_b, true]), - Option::>::None, - ); - - // Place the safe head at N so the sanity check (`safe_head.number >= block_id.number`) passes - // and `execute()` reaches the lookup we care about. - let safe_head = L2BlockInfo { - block_info: BlockInfo { - number: N, - hash: hash_a, - parent_hash: Default::default(), - timestamp: N * 2, - }, - l1_origin: BlockNumHash::default(), - seq_num: 0, - }; - let mut state = - TestEngineStateBuilder::new().with_unsafe_head(safe_head).with_safe_head(safe_head).build(); - - let task = FinalizeTask::new( - Arc::new(engine_client), - cfg, - FinalizeBlockId::ByHash(BlockNumHash { number: N, hash: hash_b }), - ); - - let result = task.execute(&mut state).await; - - assert!( - matches!(result, Err(FinalizeTaskError::BlockNotFound(n)) if n == N), - "expected BlockNotFound({N}) — got {result:?}. The by-hash lookup must fail loudly when \ - the engine lacks the requested hash; instead, the task either succeeded (finalizing the \ - wrong block) or surfaced a different error." - ); + configure_engine(&l2); + + let result = FinalizeTask::new(Arc::new(engine_client), cfg, target).execute(state).await; + l1.assert_finished(); l2.assert_finished(); + result +} + +const fn is_behind(err: &FinalizeTaskError) -> bool { + matches!(err, FinalizeTaskError::BehindFinalizedHead { finalized: FINALIZED, target: STALE }) +} + +const fn is_conflict(err: &FinalizeTaskError) -> bool { + matches!(err, FinalizeTaskError::ConflictsWithFinalizedHead { .. }) +} + +const fn is_not_canonical(err: &FinalizeTaskError) -> bool { + matches!(err, FinalizeTaskError::NotCanonical { .. }) +} + +const fn is_number_mismatch(err: &FinalizeTaskError) -> bool { + matches!(err, FinalizeTaskError::BlockNumberMismatch { requested: ADVANCED, .. }) +} + +const fn is_block_not_found(err: &FinalizeTaskError) -> bool { + matches!(err, FinalizeTaskError::BlockNotFound(ADVANCED)) +} + +/// Targets behind the finalized head, or a pinned hash that conflicts with it, are skipped before +/// asking the execution layer for the block: the test engine expects no request at all. +#[rstest] +#[case::stale_by_hash(by_hash(&canonical(STALE)), is_behind)] +#[case::stale_by_number(by_number(&canonical(STALE)), is_behind)] +#[case::conflicting_by_hash(by_hash(&conflicting(FINALIZED)), is_conflict)] +#[tokio::test] +async fn finalize_task_skips_target_without_lookup( + #[case] target: FinalizeBlockId, + #[case] is_expected_error: fn(&FinalizeTaskError) -> bool, +) { + let mut state = engine_state(established()); + let original_state = state; + + let err = finalize(target, &mut state, |_| {}).await.unwrap_err(); + + assert!(is_expected_error(&err), "{err:?}"); + assert_eq!(err.severity(), EngineTaskErrorSeverity::Skip); + assert_eq!(state, original_state); +} + +/// A target above the safe head is refused before asking the execution layer for the block: the +/// test engine expects no request at all. +#[tokio::test] +async fn finalize_task_refuses_target_above_safe_head_without_lookup() { + let mut state = engine_state(established()); + let original_state = state; + + let err = finalize(by_hash(&canonical(SAFE + 5)), &mut state, |_| {}).await.unwrap_err(); + + assert!(matches!(err, FinalizeTaskError::BlockNotSafe), "{err:?}"); + assert_eq!(err.severity(), EngineTaskErrorSeverity::Critical); + assert_eq!(state, original_state); +} + +/// A target whose block differs from the engine's canonical block is skipped. A pinned hash is +/// never replaced by the canonical block at its height, even above the finalized head, where a +/// delegate on another fork names a block the engine does not consider canonical. +#[rstest] +#[case::conflicting_by_number(by_number(&conflicting(FINALIZED)), conflicting(FINALIZED), is_conflict)] +#[case::other_fork_by_hash(by_hash(&conflicting(ADVANCED)), canonical(ADVANCED), is_not_canonical)] +#[tokio::test] +async fn finalize_task_skips_target_off_the_canonical_chain( + #[case] target: FinalizeBlockId, + #[case] served_block: RpcBlock, + #[case] is_expected_error: fn(&FinalizeTaskError) -> bool, +) { + let mut state = engine_state(established()); + let original_state = state; + + let err = finalize(target, &mut state, |l2| { + l2.expect_block_by_number(target.number(), Some(served_block)) + }) + .await + .unwrap_err(); + + assert!(is_expected_error(&err), "{err:?}"); + assert_eq!(err.severity(), EngineTaskErrorSeverity::Skip); + assert_eq!(state, original_state); +} + +/// A canonical target above the finalized head becomes the finalized head through one forkchoice +/// update. The canonical block at the finalized height is already finalized, so it needs no update. +#[rstest] +#[case::advancing_by_hash(established(), by_hash, canonical(ADVANCED), 1)] +#[case::advancing_by_number(established(), by_number, canonical(ADVANCED), 1)] +#[case::identical_by_number(established(), by_number, canonical(FINALIZED), 0)] +#[case::uninitialized_adopts_genesis(uninitialized(), by_hash, genesis_block(), 1)] +#[tokio::test] +async fn finalize_task_finalizes_canonical_target( + #[case] finalized_head: L2BlockInfo, + #[case] identify: fn(&RpcBlock) -> FinalizeBlockId, + #[case] served_block: RpcBlock, + #[case] forkchoice_updates: usize, +) { + let mut state = engine_state(finalized_head); + let target = identify(&served_block); + let expected_sync_state = with_finalized_head(&state, l2_block_info(&served_block)); + + finalize(target, &mut state, |l2| { + l2.expect_block_by_number(target.number(), Some(served_block)); + for _ in 0..forkchoice_updates { + l2.expect_forkchoice_updated( + expected_sync_state.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + } + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected_sync_state); +} + +/// A target pinned to the finalized head is already finalized, so it needs no lookup. +#[tokio::test] +async fn finalize_task_skips_lookup_for_finalized_head() { + let mut state = engine_state(established()); + let original_state = state; + + finalize(by_hash(&canonical(FINALIZED)), &mut state, |_| {}).await.unwrap(); + + assert_eq!(state, original_state); +} + +/// The engine serving no block, or a block at a height other than the requested one, stops the +/// node instead of finalizing anything. A misplaced block could otherwise bypass the guards against +/// moving finality backward. +#[rstest] +#[case::missing_by_number(FinalizeBlockId::ByNumber(ADVANCED), None, is_block_not_found)] +#[case::missing_by_hash(by_hash(&canonical(ADVANCED)), None, is_block_not_found)] +#[case::stale_block_by_number( + /* target */ FinalizeBlockId::ByNumber(ADVANCED), + /* served_block */ Some(canonical(STALE)), + /* is_expected_error */ is_number_mismatch +)] +#[case::higher_block_by_number( + /* target */ FinalizeBlockId::ByNumber(ADVANCED), + /* served_block */ Some(canonical(SAFE)), + /* is_expected_error */ is_number_mismatch +)] +#[case::stale_block_by_hash(by_hash(&canonical(ADVANCED)), Some(canonical(STALE)), is_number_mismatch)] +#[tokio::test] +async fn finalize_task_stops_on_unexpected_lookup_result( + #[case] target: FinalizeBlockId, + #[case] served_block: Option, + #[case] is_expected_error: fn(&FinalizeTaskError) -> bool, +) { + let mut state = engine_state(established()); + let original_state = state; + + let err = + finalize(target, &mut state, |l2| l2.expect_block_by_number(target.number(), served_block)) + .await + .unwrap_err(); + + assert!(is_expected_error(&err), "{err:?}"); + assert_eq!(err.severity(), EngineTaskErrorSeverity::Critical); + assert_eq!(state, original_state); } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/mod.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/mod.rs index 224c7a9e34f..9eddaa76d00 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/mod.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/mod.rs @@ -7,7 +7,7 @@ pub use task::{ mod synchronize; pub use synchronize::SynchronizeTaskError; -pub(super) use synchronize::synchronize; +pub(super) use synchronize::{synchronize, synchronize_unconditionally}; mod insert; pub(super) use insert::insert_payload; diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs index abae6d99464..a9bc15146ff 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs @@ -1,15 +1,17 @@ //! The `engine_forkchoiceUpdated` call, with no attributes, that the engine tasks share. -use crate::{EngineClient, EngineState, SynchronizeTaskError, state::EngineSyncStateUpdate}; +use crate::{ + EngineClient, EngineState, EngineSyncState, SynchronizeTaskError, state::EngineSyncStateUpdate, +}; use alloy_rpc_types_engine::{INVALID_FORK_CHOICE_STATE_ERROR, PayloadStatusEnum}; use op_alloy_provider::ext::engine::OpEngineApi; /// Applies `state_update` to the engine's sync state and synchronizes the execution layer's -/// forkchoice with it, using an `engine_forkchoiceUpdated` call without payload attributes. +/// forkchoice with it, using an `engine_forkchoiceUpdated` call without payload attributes. An +/// update that leaves the labels unchanged is skipped. /// -/// Engine tasks call this whenever they move the unsafe, safe, or finalized head, and the engine -/// calls it during a reset to establish the initial forkchoice state. Forkchoice updates that -/// start a block build are made by `start_build` instead. +/// Engine tasks call this whenever they move the unsafe, safe, or finalized head. Forkchoice +/// updates that start a block build are made by `start_build` instead. #[tracing::instrument( name = "synchronize", target = "engine", @@ -22,13 +24,10 @@ pub(in crate::task_queue) async fn synchronize( state: &mut EngineState, state_update: EngineSyncStateUpdate, ) -> Result<(), SynchronizeTaskError> { - // Apply the sync state update to the engine state. let new_sync_state = state.sync_state.apply_update(state_update); - // Check if a forkchoice update is not needed, return early. - // A forkchoice update is not needed if... - // 1. The engine state is not default (initial forkchoice state has been emitted), and - // 2. The new sync state is the same as the current sync state (no changes to the sync state). + // A forkchoice update is not needed once the initial forkchoice state has been emitted and the + // update leaves the labels unchanged. // // NOTE: // We shouldn't retry the synchronization here. Since the `sync_state` is only updated inside @@ -39,6 +38,36 @@ pub(in crate::task_queue) async fn synchronize( return Ok(()); } + send_forkchoice(client, state, new_sync_state).await +} + +/// Like [`synchronize`], but sends the forkchoice update even when it leaves the labels unchanged. +/// +/// Unchanged labels do not prove that the execution layer holds their forkchoice: a `SYNCING` +/// answer moves the labels too, and an execution layer that restarts can lose its forkchoice. +#[tracing::instrument( + name = "synchronize_unconditionally", + target = "engine", + level = "info", + skip_all, + fields(?state_update, forkchoice = tracing::field::Empty) +)] +pub(in crate::task_queue) async fn synchronize_unconditionally( + client: &EngineClient, + state: &mut EngineState, + state_update: EngineSyncStateUpdate, +) -> Result<(), SynchronizeTaskError> { + let new_sync_state = state.sync_state.apply_update(state_update); + send_forkchoice(client, state, new_sync_state).await +} + +/// Sends the forkchoice of `new_sync_state` and, once the execution layer accepts it, makes it the +/// engine's sync state. +async fn send_forkchoice( + client: &EngineClient, + state: &mut EngineState, + new_sync_state: EngineSyncState, +) -> Result<(), SynchronizeTaskError> { // Check if the head is behind the finalized head. if new_sync_state.unsafe_head().block_info.number < new_sync_state.finalized_head().block_info.number diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/mod.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/mod.rs index 67c378103a7..9de83da3a5c 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/mod.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/mod.rs @@ -1,7 +1,10 @@ //! The forkchoice engine update shared by the engine tasks, and its error type. mod helper; -pub(in crate::task_queue) use helper::synchronize; +pub(in crate::task_queue) use helper::{synchronize, synchronize_unconditionally}; mod error; pub use error::SynchronizeTaskError; + +#[cfg(test)] +mod tests; diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/tests.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/tests.rs new file mode 100644 index 00000000000..64909cb394f --- /dev/null +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/tests.rs @@ -0,0 +1,153 @@ +use super::{synchronize, synchronize_unconditionally}; +use crate::{ + EngineState, EngineSyncState, EngineSyncStateUpdate, EngineTaskError, + EngineTaskErrorSeverity::{self, Reset, Temporary}, + test_utils::{RpcMock, TestEngineStateBuilder, test_block_info, test_engine_client}, +}; +use alloy_rpc_types_engine::{ForkchoiceState, INVALID_FORK_CHOICE_STATE_ERROR, PayloadStatusEnum}; +use kona_genesis::RollupConfig; +use rstest::rstest; +use std::sync::Arc; + +const FORKCHOICE_UPDATED: &str = "engine_forkchoiceUpdatedV3"; + +/// How the execution layer answers the forkchoice update. +#[derive(Clone, Copy, Debug)] +enum Reply { + Invalid, + InvalidForkchoiceState, + Syncing, + TransportError, + Valid, +} + +/// Which labels the engine holds after the forkchoice update. +#[derive(Clone, Copy, Debug)] +enum Labels { + Kept, + Moved, +} + +/// An engine state that has not seen a `VALID` answer yet. +fn engine_state() -> EngineState { + TestEngineStateBuilder::new() + .with_unsafe_head(test_block_info(20)) + .with_safe_head(test_block_info(10)) + .with_finalized_head(test_block_info(5)) + .with_el_sync_finished(false) + .build() +} + +/// An update that moves the safe head, or one that leaves every label unchanged. +fn state_update(moves_safe_head: bool) -> EngineSyncStateUpdate { + EngineSyncStateUpdate { + safe_head: moves_safe_head.then(|| test_block_info(15)), + ..Default::default() + } +} + +const fn labels_after( + labels: Labels, + before: EngineSyncState, + moved: EngineSyncState, +) -> EngineSyncState { + match labels { + Labels::Kept => before, + Labels::Moved => moved, + } +} + +fn expect_reply(l2: &RpcMock, forkchoice: ForkchoiceState, reply: Reply) { + match reply { + Reply::Invalid => l2.expect_forkchoice_updated( + forkchoice, + PayloadStatusEnum::Invalid { validation_error: "invalid".into() }, + ), + Reply::InvalidForkchoiceState => { + l2.expect_error_code(FORKCHOICE_UPDATED, INVALID_FORK_CHOICE_STATE_ERROR as i64) + } + Reply::Syncing => l2.expect_forkchoice_updated(forkchoice, PayloadStatusEnum::Syncing), + Reply::TransportError => l2.expect_transport_error(FORKCHOICE_UPDATED), + Reply::Valid => l2.expect_forkchoice_updated(forkchoice, PayloadStatusEnum::Valid), + } +} + +/// `synchronize` skips an update that leaves the labels unchanged. +#[rstest] +#[case::changed_labels_are_sent(true, 1)] +#[case::unchanged_labels_are_skipped(false, 0)] +#[tokio::test] +async fn synchronize_skips_unchanged_labels( + #[case] moves_safe_head: bool, + #[case] forkchoice_updates: usize, +) { + let mut state = engine_state(); + let update = state_update(moves_safe_head); + let expected_sync_state = state.sync_state.apply_update(update); + let (client, l1, l2) = test_engine_client(Arc::new(RollupConfig::default())); + for _ in 0..forkchoice_updates { + l2.expect_forkchoice_updated( + expected_sync_state.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + } + + synchronize(&client, &mut state, update).await.unwrap(); + + assert_eq!(state.sync_state, expected_sync_state); + l1.assert_finished(); + l2.assert_finished(); +} + +/// `synchronize_unconditionally` sends the update even when it leaves the labels unchanged. +#[rstest] +#[case::changed_labels_are_sent(true)] +#[case::unchanged_labels_are_sent(false)] +#[tokio::test] +async fn synchronize_unconditionally_sends_unchanged_labels(#[case] moves_safe_head: bool) { + let mut state = engine_state(); + let update = state_update(moves_safe_head); + let expected_sync_state = state.sync_state.apply_update(update); + let (client, l1, l2) = test_engine_client(Arc::new(RollupConfig::default())); + l2.expect_forkchoice_updated( + expected_sync_state.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + + synchronize_unconditionally(&client, &mut state, update).await.unwrap(); + + assert_eq!(state.sync_state, expected_sync_state); + l1.assert_finished(); + l2.assert_finished(); +} + +/// Only a `VALID` or `SYNCING` answer moves the labels, and only `VALID` marks the execution layer +/// as synced. Failures leave the engine state untouched. +#[rstest] +#[case::valid(Reply::Valid, Labels::Moved, true, None)] +#[case::syncing(Reply::Syncing, Labels::Moved, false, None)] +#[case::invalid_payload(Reply::Invalid, Labels::Kept, false, Some(Reset))] +#[case::invalid_forkchoice_state(Reply::InvalidForkchoiceState, Labels::Kept, false, Some(Reset))] +#[case::transport_error(Reply::TransportError, Labels::Kept, false, Some(Temporary))] +#[tokio::test] +async fn synchronize_applies_labels_only_when_accepted( + #[case] reply: Reply, + #[case] labels: Labels, + #[case] el_sync_finished: bool, + #[case] error_severity: Option, +) { + let mut state = engine_state(); + let update = state_update(true); + let moved_sync_state = state.sync_state.apply_update(update); + let expected_sync_state = labels_after(labels, state.sync_state, moved_sync_state); + let (client, l1, l2) = test_engine_client(Arc::new(RollupConfig::default())); + expect_reply(&l2, moved_sync_state.create_forkchoice_state(), reply); + + let result = synchronize(&client, &mut state, update).await; + + assert_eq!(result.as_ref().err().map(EngineTaskError::severity), error_severity, "{result:?}"); + assert_eq!(state.sync_state, expected_sync_state); + assert_eq!(state.el_sync_finished, el_sync_finished); + l1.assert_finished(); + l2.assert_finished(); +} diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/task.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/task.rs index ee976a02201..6b3a3d1b4cb 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/task.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/task.rs @@ -32,6 +32,10 @@ pub enum EngineTaskErrorSeverity { /// The error indicates that the engine should be flushed. #[display("flush")] Flush, + /// The task no longer applies to the engine state, so it is discarded instead of retried and + /// the engine keeps running. + #[display("skip")] + Skip, } /// The interface for an engine task error. diff --git a/rust/kona/crates/node/engine/src/test_utils/blocks.rs b/rust/kona/crates/node/engine/src/test_utils/blocks.rs new file mode 100644 index 00000000000..434d3bd01eb --- /dev/null +++ b/rust/kona/crates/node/engine/src/test_utils/blocks.rs @@ -0,0 +1,40 @@ +//! L2 blocks, and their RPC form as the execution layer serves them. + +use super::l1_info_deposit; +use alloy_consensus::transaction::Recovered; +use alloy_primitives::{Address, B256, Bytes}; +use alloy_rpc_types_eth::{BlockTransactions, Header as RpcHeader}; +use op_alloy_consensus::OpBlock; + +/// A full L2 block as the execution layer serves it over RPC. +pub type TestRpcBlock = alloy_rpc_types_eth::Block; + +/// An L2 block at `number` on `parent_hash` whose only transaction is the L1-info deposit, so it +/// converts into an `L2BlockInfo` without being the configured genesis. Blocks with different +/// `extra_data` share a height but not a hash. +pub fn test_l2_block(number: u64, parent_hash: B256, extra_data: u8) -> OpBlock { + let mut block = OpBlock::default(); + block.header.extra_data = Bytes::from(vec![extra_data]); + block.header.number = number; + block.header.parent_hash = parent_hash; + block.header.timestamp = number * 2; + block.body.transactions.push(l1_info_deposit()); + block +} + +/// `block` as the execution layer serves it over RPC. +pub fn test_rpc_block(block: &OpBlock) -> TestRpcBlock { + let transactions = block + .body + .transactions + .iter() + .cloned() + .map(|transaction| { + op_alloy_rpc_types::Transaction::from_transaction( + Recovered::new_unchecked(transaction, Address::ZERO), + Default::default(), + ) + }) + .collect(); + TestRpcBlock::new(RpcHeader::new(block.header.clone()), BlockTransactions::Full(transactions)) +} diff --git a/rust/kona/crates/node/engine/src/test_utils/deposits.rs b/rust/kona/crates/node/engine/src/test_utils/deposits.rs new file mode 100644 index 00000000000..78d60d4bf01 --- /dev/null +++ b/rust/kona/crates/node/engine/src/test_utils/deposits.rs @@ -0,0 +1,14 @@ +//! Deposit transaction fixtures. + +use alloy_primitives::Sealed; +use kona_protocol::L1BlockInfoBedrock; +use op_alloy_consensus::{OpTxEnvelope, TxDeposit}; + +/// The L1-info deposit a block needs for `L2BlockInfo::from_block_and_genesis` to read its L1 +/// origin. +pub fn l1_info_deposit() -> OpTxEnvelope { + OpTxEnvelope::Deposit(Sealed::new(TxDeposit { + input: L1BlockInfoBedrock::default().encode_calldata(), + ..Default::default() + })) +} diff --git a/rust/kona/crates/node/engine/src/test_utils/mod.rs b/rust/kona/crates/node/engine/src/test_utils/mod.rs index a5575963a67..0a60403a0d6 100644 --- a/rust/kona/crates/node/engine/src/test_utils/mod.rs +++ b/rust/kona/crates/node/engine/src/test_utils/mod.rs @@ -1,6 +1,12 @@ mod attributes; pub use attributes::TestAttributesBuilder; +mod blocks; +pub use blocks::{TestRpcBlock, test_l2_block, test_rpc_block}; + +mod deposits; +pub use deposits::l1_info_deposit; + mod rpc; pub use rpc::{RpcMock, test_engine_client}; diff --git a/rust/kona/crates/node/engine/src/test_utils/rpc.rs b/rust/kona/crates/node/engine/src/test_utils/rpc.rs index be93a9816fe..39f23f14de2 100644 --- a/rust/kona/crates/node/engine/src/test_utils/rpc.rs +++ b/rust/kona/crates/node/engine/src/test_utils/rpc.rs @@ -1,10 +1,13 @@ //! Scripted RPC expectations over Alloy's mock transport. -use crate::EngineClient; +use crate::{EngineClient, test_utils::TestRpcBlock}; use alloy_json_rpc::{ErrorPayload, RequestPacket, ResponsePacket}; use alloy_network::Network; use alloy_provider::RootProvider; use alloy_rpc_client::RpcClient; +use alloy_rpc_types_engine::{ + ForkchoiceState, ForkchoiceUpdated, PayloadStatus, PayloadStatusEnum, +}; use alloy_transport::{ TransportError, TransportErrorKind, TransportFut, mock::{Asserter, MockResponse, MockTransport}, @@ -56,6 +59,30 @@ impl RpcMock { ); } + /// Expects a full-block `eth_getBlockByNumber` call for `number` and answers it with `block`, + /// or with null for `None`. + pub fn expect_block_by_number(&self, number: u64, block: Option) { + self.expect_params( + "eth_getBlockByNumber", + serde_json::json!([format!("{number:#x}"), true]), + block, + ); + } + + /// Expects an `engine_forkchoiceUpdatedV3` call for `forkchoice` without payload attributes and + /// answers it with `status`. + pub fn expect_forkchoice_updated( + &self, + forkchoice: ForkchoiceState, + status: PayloadStatusEnum, + ) { + self.expect_params( + "engine_forkchoiceUpdatedV3", + serde_json::json!([forkchoice, null]), + ForkchoiceUpdated::new(PayloadStatus::from_status(status)), + ); + } + /// Expects a call and delays its response, for recovery tests using Tokio's paused clock. pub fn expect_with_delay( &self, diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs index 44fe763d969..9962dd9e388 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs @@ -2,10 +2,11 @@ use crate::{ DerivationActorRequest, DerivationEngineClient, NodeActor, actors::derivation::{DerivationDelegateProvider, DerivationError}, }; +use alloy_eips::BlockNumHash; use alloy_primitives::BlockHash; use async_trait::async_trait; use kona_derive::{ChainProvider, Signal}; -use kona_engine::FinalizeBlockId; +use kona_engine::{EngineState, FinalizeBlockId}; use kona_protocol::{BlockInfo, L2BlockInfo, SyncStatus}; use thiserror::Error; use tokio::{ @@ -42,6 +43,10 @@ where /// Publishes the delegate's `current_l1`: the L1 block that its derivation has reached. derivation_origin_tx: watch::Sender>, + /// The engine's state as it publishes it. Delegate statuses are checked against the + /// finalized head the engine applied, not the last one forwarded to it, which the engine may + /// have skipped. + engine_state_rx: watch::Receiver, /// The engine's L2 safe head, according to updates from the Engine. engine_l2_safe_head: L2BlockInfo, /// Whether the engine sync has completed. This will only ever go from false -> true. @@ -64,6 +69,7 @@ where derivation_delegate_provider: DelegateProvider, l1_provider: L1Provider, derivation_origin_tx: watch::Sender>, + engine_state_rx: watch::Receiver, ) -> Self { let mut delegated_derivation_ticker = time::interval(Self::DERIVATION_DELEGATE_POLL_INTERVAL); @@ -74,6 +80,7 @@ where derivation_delegate_provider, l1_provider, derivation_origin_tx, + engine_state_rx, engine_l2_safe_head: L2BlockInfo::default(), has_engine_sync_completed: false, delegated_derivation_ticker, @@ -149,9 +156,59 @@ where Ok(()) } - /// Verifies that the L1 info reported by the derivation delegate - /// are consistent with canonical L1 chain. + /// The id of an established finalized head. The default head means finality has not been + /// established yet, so any finalized block from the delegate may initialize it. + fn established_finalized_l2(finalized_head: L2BlockInfo) -> Option { + (finalized_head != L2BlockInfo::default()).then(|| finalized_head.block_info.id()) + } + + /// Rejects a finalized L2 block behind, or at the same height as but different from, the + /// engine's. Either means the delegate lags or follows another chain, so none of its status + /// applies: its safe head would move the engine back or onto that chain too. + /// + /// Only a fork at the engine's finalized height is caught here. A chain that forked below it + /// and reports a higher finalized block passes, and the execution layer rejects the forkchoice + /// update that would move onto it. + fn validate_finalized_l2_progress( + &self, + reported: BlockNumHash, + ) -> Result<(), DerivationDelegationError> { + let engine_finalized_head = self.engine_state_rx.borrow().sync_state.finalized_head(); + let Some(engine) = Self::established_finalized_l2(engine_finalized_head) else { + return Ok(()); + }; + if reported.number < engine.number { + return Err(DerivationDelegationError::FinalizedL2Regressed { engine, reported }); + } + if reported.number == engine.number && reported.hash != engine.hash { + return Err(DerivationDelegationError::FinalizedL2Conflict { engine, reported }); + } + Ok(()) + } + + /// Verifies that the derivation delegate's finalized L2 block is not above its safe L2 block + /// and does not move back from the engine's, and that its L1 info is consistent with the + /// canonical L1 chain. async fn validate_sync_status(&mut self, v: &SyncStatus) -> bool { + // A finalized block above the safe head is an inconsistent snapshot; the engine would only + // refuse to finalize it. + if v.finalized_l2.block_info.number > v.safe_l2.block_info.number { + let err = DerivationDelegationError::FinalizedAheadOfSafe { + finalized: v.finalized_l2.block_info.number, + safe: v.safe_l2.block_info.number, + }; + warn!(target: "derivation", error = %err, "Inconsistent sync status from delegate"); + return false; + } + if let Err(err) = self.validate_finalized_l2_progress(v.finalized_l2.block_info.id()) { + warn!( + target: "derivation", + error = %err, + "Finalized L2 inconsistency detected at sync status from delegate" + ); + return false; + } + let checks = [ ("L1 Origin of Safe L2", v.safe_l2.l1_origin.number, v.safe_l2.l1_origin.hash), ( @@ -249,6 +306,18 @@ where #[derive(Error, Debug)] enum DerivationDelegationError { + /// The delegate's finalized L2 block is above its safe L2 block. + #[error("Finalized L2 {finalized} is ahead of safe L2 {safe}")] + FinalizedAheadOfSafe { finalized: u64, safe: u64 }, + + /// The delegate's finalized L2 block is a different block at the engine's finalized height. + #[error("Finalized L2 {reported:?} conflicts with the engine's finalized L2 {engine:?}")] + FinalizedL2Conflict { engine: BlockNumHash, reported: BlockNumHash }, + + /// The delegate's finalized L2 block is behind the engine's. + #[error("Finalized L2 {reported:?} is behind the engine's finalized L2 {engine:?}")] + FinalizedL2Regressed { engine: BlockNumHash, reported: BlockNumHash }, + /// The L1 provider returned an error (network, RPC, etc.) #[error("L1 provider error: {0}")] L1Provider(String), diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/mod.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/mod.rs index e5460eceb11..5431e58aa93 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/mod.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/mod.rs @@ -5,3 +5,6 @@ mod client; pub use client::{ DerivationDelegateClient, DerivationDelegateClientError, DerivationDelegateProvider, }; + +#[cfg(test)] +mod tests; diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs new file mode 100644 index 00000000000..2468e203727 --- /dev/null +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs @@ -0,0 +1,283 @@ +use super::{DelegateDerivationActor, DerivationDelegateClientError, DerivationDelegateProvider}; +use crate::{DerivationActorRequest, EngineActorRequest, NodeActor, QueuedDerivationEngineClient}; +use alloy_eips::BlockNumHash; +use alloy_primitives::B256; +use async_trait::async_trait; +use kona_derive::test_utils::TestChainProvider; +use kona_engine::{ + ConsolidateInput, EngineState, EngineSyncState, EngineSyncStateUpdate, FinalizeBlockId, +}; +use kona_protocol::{BlockInfo, L2BlockInfo, SyncStatus}; +use rstest::rstest; +use std::{collections::VecDeque, sync::Mutex}; +use tokio::sync::{mpsc, watch}; + +const SAFE_ORIGIN: u64 = 1; +const FINALIZED_ORIGIN: u64 = 2; +const CURRENT_L1: u64 = 3; + +type SyncStatusReply = Result; + +/// Answers each sync status poll with the next scripted reply. +#[derive(Debug)] +struct ScriptedDelegate(Mutex>); + +#[async_trait] +impl DerivationDelegateProvider for ScriptedDelegate { + async fn fetch_sync_status(&self) -> SyncStatusReply { + self.0.lock().unwrap().pop_front().expect("unexpected sync status poll") + } +} + +/// A delegate actor over a canonical L1 chain, with the engine requests it sends and the +/// `current_l1` it publishes. +struct Harness { + actor: + DelegateDerivationActor, + current_l1: watch::Receiver>, + derivation_requests: mpsc::Sender, + engine_requests: mpsc::Receiver, + engine_state: watch::Sender, +} + +impl Harness { + fn new(replies: impl IntoIterator) -> Self { + let mut l1 = TestChainProvider::default(); + for number in [SAFE_ORIGIN, FINALIZED_ORIGIN, CURRENT_L1] { + l1.insert_block(number, canonical_l1_block(number)); + } + let (derivation_requests, derivation_request_rx) = mpsc::channel(8); + let (engine_request_tx, engine_requests) = mpsc::channel(8); + let (current_l1_tx, current_l1) = watch::channel(None); + let (engine_state, engine_state_rx) = watch::channel(EngineState::default()); + let actor = DelegateDerivationActor::new( + QueuedDerivationEngineClient::new(engine_request_tx), + derivation_request_rx, + ScriptedDelegate(Mutex::new(replies.into_iter().collect())), + l1, + current_l1_tx, + engine_state_rx, + ); + Self { actor, current_l1, derivation_requests, engine_requests, engine_state } + } + + async fn complete_engine_sync(&mut self) { + self.complete_engine_sync_with(EngineSyncState::default()).await; + } + + /// Completes the engine sync with `sync_state` as the engine's labels, which the engine + /// publishes before it reports the completion. + async fn complete_engine_sync_with(&mut self, sync_state: EngineSyncState) { + self.publish_engine_sync_state(sync_state); + self.derivation_requests + .send(DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new( + sync_state.local_safe_head(), + ))) + .await + .unwrap(); + self.actor.step().await.unwrap(); + } + + /// Publishes the engine's labels as the engine does after each task it runs. + fn publish_engine_sync_state(&self, sync_state: EngineSyncState) { + self.engine_state.send_modify(|state| state.sync_state = sync_state); + } + + /// The engine requests sent since the last call, in order. + fn forwarded(&mut self) -> Vec { + std::iter::from_fn(|| self.engine_requests.try_recv().ok()).map(Forwarded::from).collect() + } +} + +/// The parts of an engine request that the delegate actor decides. +#[derive(Debug, PartialEq, Eq)] +enum Forwarded { + Finalized(FinalizeBlockId), + Other(String), + Safe(L2BlockInfo), +} + +impl From for Forwarded { + fn from(request: EngineActorRequest) -> Self { + match request { + EngineActorRequest::ProcessFinalizedL2Block(finalized) => Self::Finalized(*finalized), + EngineActorRequest::ProcessSafeL2Signal(ConsolidateInput::BlockInfo(safe)) => { + Self::Safe(safe) + } + other => Self::Other(format!("{other:?}")), + } + } +} + +/// What one poll of `status` forwards: the safe head, then the finalized block pinned by hash. +fn forwarded_for(status: &SyncStatus) -> Vec { + vec![ + Forwarded::Safe(status.safe_l2), + Forwarded::Finalized(FinalizeBlockId::ByHash(status.finalized_l2.block_info.id())), + ] +} + +/// What one poll forwards when the status is accepted, and nothing when it is rejected. +fn forwarded_if(accepted: bool, status: &SyncStatus) -> Vec { + if accepted { forwarded_for(status) } else { Vec::new() } +} + +fn canonical_l1_block(number: u64) -> BlockInfo { + BlockInfo { hash: B256::with_last_byte(number as u8), number, ..Default::default() } +} + +fn l2_block(number: u64, l1_origin: u64) -> L2BlockInfo { + L2BlockInfo { + block_info: BlockInfo { + hash: B256::with_last_byte(0x80 | number as u8), + number, + ..Default::default() + }, + l1_origin: canonical_l1_block(l1_origin).id(), + seq_num: 0, + } +} + +/// A sync status whose safe origin, finalized origin and `current_l1` are all canonical. +fn delegate_status() -> SyncStatus { + SyncStatus { + current_l1: canonical_l1_block(CURRENT_L1), + current_l1_finalized: BlockInfo::default(), + finalized_l1: BlockInfo::default(), + finalized_l2: l2_block(15, FINALIZED_ORIGIN), + head_l1: BlockInfo::default(), + local_safe_l2: l2_block(20, SAFE_ORIGIN), + safe_l1: BlockInfo::default(), + safe_l2: l2_block(20, SAFE_ORIGIN), + unsafe_l2: l2_block(25, SAFE_ORIGIN), + } +} + +fn non_canonical(id: BlockNumHash) -> BlockNumHash { + BlockNumHash { hash: B256::repeat_byte(0xee), ..id } +} + +/// Returns `status` with its finalized L2 block replaced by `finalized_l2`. +fn with_finalized_l2(finalized_l2: L2BlockInfo) -> SyncStatus { + SyncStatus { finalized_l2, ..delegate_status() } +} + +/// An engine sync state whose finalized head is `finalized_l2`. +fn engine_finalized_at(finalized_l2: L2BlockInfo) -> EngineSyncState { + EngineSyncState::default().apply_update(EngineSyncStateUpdate { + finalized_head: Some(finalized_l2), + ..Default::default() + }) +} + +fn conflicting(block: L2BlockInfo) -> L2BlockInfo { + L2BlockInfo { + block_info: BlockInfo { + hash: non_canonical(block.block_info.id()).hash, + ..block.block_info + }, + ..block + } +} + +/// The engine's finalized head when its sync completes is the baseline, so a delegate that starts +/// behind it, or on another chain at its height, is rejected whole from the first poll on. +#[rstest] +#[case::advancing(l2_block(16, FINALIZED_ORIGIN), true)] +#[case::unchanged(l2_block(15, FINALIZED_ORIGIN), true)] +#[case::regressed(l2_block(14, FINALIZED_ORIGIN), false)] +#[case::conflicting(conflicting(l2_block(15, FINALIZED_ORIGIN)), false)] +#[tokio::test(start_paused = true)] +async fn first_poll_rejects_status_behind_engine_finality( + #[case] finalized_l2: L2BlockInfo, + #[case] accepted: bool, +) { + let status = with_finalized_l2(finalized_l2); + let mut harness = Harness::new([Ok(status.clone())]); + harness.complete_engine_sync_with(engine_finalized_at(l2_block(15, FINALIZED_ORIGIN))).await; + + harness.actor.step().await.unwrap(); + + assert_eq!(harness.forwarded(), forwarded_if(accepted, &status)); + assert_eq!(harness.current_l1.borrow().is_some(), accepted); +} + +/// Once the engine applies a forwarded finalized block, a status whose finalized block is behind +/// it, or is a different block at its height, is rejected whole: a lagging or diverged delegate's +/// safe head must not reach the engine either, and its `current_l1` is not published. +#[rstest] +#[case::advancing(l2_block(16, FINALIZED_ORIGIN), true)] +#[case::unchanged(l2_block(15, FINALIZED_ORIGIN), true)] +#[case::regressed(l2_block(14, FINALIZED_ORIGIN), false)] +#[case::conflicting(conflicting(l2_block(15, FINALIZED_ORIGIN)), false)] +#[tokio::test(start_paused = true)] +async fn poll_rejects_status_whose_finality_moves_back( + #[case] next_finalized_l2: L2BlockInfo, + #[case] accepted: bool, +) { + let next_status = with_finalized_l2(next_finalized_l2); + let mut harness = Harness::new([Ok(delegate_status()), Ok(next_status.clone())]); + harness.complete_engine_sync().await; + harness.actor.step().await.unwrap(); + assert_eq!(harness.forwarded(), forwarded_for(&delegate_status())); + harness.publish_engine_sync_state(engine_finalized_at(delegate_status().finalized_l2)); + harness.current_l1.borrow_and_update(); + + harness.actor.step().await.unwrap(); + + assert_eq!(harness.forwarded(), forwarded_if(accepted, &next_status)); + assert_eq!(harness.current_l1.has_changed().unwrap(), accepted); +} + +/// A finalized block the engine skipped, such as one on another chain, does not become the +/// baseline: the canonical block at its height, or below it but above the engine's finalized +/// head, is still accepted, so the delegate's safe head keeps reaching the engine. +#[rstest] +#[case::canonical_at_skipped_height(l2_block(17, FINALIZED_ORIGIN))] +#[case::canonical_below_skipped_height(l2_block(16, FINALIZED_ORIGIN))] +#[tokio::test(start_paused = true)] +async fn poll_accepts_status_after_engine_skipped_forwarded_finality( + #[case] canonical_finalized_l2: L2BlockInfo, +) { + let skipped_status = with_finalized_l2(conflicting(l2_block(17, FINALIZED_ORIGIN))); + let canonical_status = with_finalized_l2(canonical_finalized_l2); + let mut harness = Harness::new([Ok(skipped_status.clone()), Ok(canonical_status.clone())]); + harness.complete_engine_sync_with(engine_finalized_at(l2_block(15, FINALIZED_ORIGIN))).await; + harness.actor.step().await.unwrap(); + assert_eq!(harness.forwarded(), forwarded_for(&skipped_status)); + + harness.actor.step().await.unwrap(); + + assert_eq!(harness.forwarded(), forwarded_for(&canonical_status)); +} + +/// Before the engine establishes finality, any finalized block from the delegate may initialize +/// it, including one at genesis height. +#[tokio::test(start_paused = true)] +async fn first_poll_accepts_finality_before_the_engine_establishes_it() { + let status = with_finalized_l2(l2_block(0, FINALIZED_ORIGIN)); + let mut harness = Harness::new([Ok(status.clone())]); + harness.complete_engine_sync().await; + + harness.actor.step().await.unwrap(); + + assert_eq!(harness.forwarded(), forwarded_for(&status)); +} + +/// A status whose finalized block is above its safe block is inconsistent and rejected whole. +#[rstest] +#[case::at_safe_head(l2_block(20, FINALIZED_ORIGIN), true)] +#[case::ahead_of_safe_head(l2_block(21, FINALIZED_ORIGIN), false)] +#[tokio::test(start_paused = true)] +async fn poll_rejects_status_whose_finalized_is_ahead_of_its_safe( + #[case] finalized_l2: L2BlockInfo, + #[case] accepted: bool, +) { + let status = with_finalized_l2(finalized_l2); + let mut harness = Harness::new([Ok(status.clone())]); + harness.complete_engine_sync().await; + + harness.actor.step().await.unwrap(); + + assert_eq!(harness.forwarded(), forwarded_if(accepted, &status)); +} diff --git a/rust/kona/crates/node/service/src/actors/engine/actor.rs b/rust/kona/crates/node/service/src/actors/engine/actor.rs index c9affd16523..f09047db8a2 100644 --- a/rust/kona/crates/node/service/src/actors/engine/actor.rs +++ b/rust/kona/crates/node/service/src/actors/engine/actor.rs @@ -5,8 +5,8 @@ use async_trait::async_trait; use kona_derive::{ResetSignal, Signal}; use kona_engine::{ BuildTask, CanonicalizeTask, ConsolidateTask, Engine, EngineActorRequest, EngineClient, - EngineTask, EngineTaskError, EngineTaskErrorSeverity, FinalizeTask, ImportedBlockSink, - InsertTask, SealTask, + EngineTask, EngineTaskError, EngineTaskErrorSeverity, EngineTaskErrors, FinalizeTask, + ImportedBlockSink, InsertTask, SealTask, }; use kona_genesis::RollupConfig; use kona_protocol::L2BlockInfo; @@ -34,6 +34,9 @@ where el_sync_complete: bool, /// The last local-safe head sent to the derivation actor. last_engine_safe_head_sent: L2BlockInfo, + /// The error of the last skipped engine task, so an identical repeat logs no new warning + /// until a reset changes the state the task was judged against. + last_skipped_task_error: Option, /// A channel to use to relay the current unsafe head. /// ## Note /// This is `Some` when the node is in sequencer mode, and `None` when the node is in validator @@ -81,6 +84,7 @@ where el_sync_complete: false, engine, last_engine_safe_head_sent: L2BlockInfo::default(), + last_skipped_task_error: None, rollup: config, unsafe_head_tx, inbound_request_rx, @@ -92,6 +96,18 @@ where } } + /// Logs a skipped task at `warn!` when its error differs from the last skip, and at `debug!` + /// while the identical error repeats, so a target resubmitted on every poll warns once. + fn log_skipped_task(&mut self, err: &EngineTaskErrors) { + let message = err.to_string(); + if self.last_skipped_task_error.as_ref() == Some(&message) { + debug!(target: "engine", %err, "Skipped the same engine task again"); + return; + } + warn!(target: "engine", %err, "Skipped an engine task that no longer applies"); + self.last_skipped_task_error = Some(message); + } + fn defer_retry(&mut self) { self.retry_at = Instant::now() + self.retry_delay; self.retry_delay = (self.retry_delay * 2).min(Duration::from_secs(5)); @@ -100,6 +116,7 @@ where fn begin_reset(&mut self) { if !self.reset_pending { self.engine.clear(); + self.last_skipped_task_error = None; self.reset_pending = true; self.retry_at = Instant::now(); } @@ -210,6 +227,7 @@ where self.defer_retry(); return Ok(()); } + EngineTaskErrorSeverity::Skip => self.log_skipped_task(&err), } } } diff --git a/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs b/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs index 49e30d84f94..0b4bc8f58ff 100644 --- a/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs +++ b/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs @@ -1,15 +1,13 @@ use super::*; use crate::QueuedEngineDerivationClient; -use alloy_primitives::Sealed; use alloy_rpc_types_engine::{ ExecutionPayloadV1, ForkchoiceUpdated, PayloadStatus, PayloadStatusEnum, }; use kona_engine::{ - NoopBlockSink, - test_utils::{TestEngineStateBuilder, test_block_info, test_engine_client}, + ConsolidateInput, EngineSyncStateUpdate, FinalizeBlockId, NoopBlockSink, + test_utils::{TestEngineStateBuilder, l1_info_deposit, test_block_info, test_engine_client}, }; -use kona_protocol::L1BlockInfoBedrock; -use op_alloy_consensus::{OpBlock, OpTxEnvelope, TxDeposit}; +use op_alloy_consensus::OpBlock; use op_alloy_rpc_types_engine::OpExecutionPayloadEnvelope; /// Exercise the real actor request/drain loop with forty consecutive unsafe blocks, without @@ -39,10 +37,7 @@ async fn healthy_unsafe_imports_keep_engine_queue_bounded() { block.header.number = number; block.header.timestamp = number * 2; block.header.parent_hash = expected_heads.last().unwrap().hash(); - block.body.transactions.push(OpTxEnvelope::Deposit(Sealed::new(TxDeposit { - input: L1BlockInfoBedrock::default().encode_calldata(), - ..Default::default() - }))); + block.body.transactions.push(l1_info_deposit()); let payload = ExecutionPayloadV1::from_block_slow(&block); let imported: OpBlock = OpExecutionPayloadEnvelope::V1(payload.clone()).try_into_block().unwrap(); @@ -87,3 +82,105 @@ async fn healthy_unsafe_imports_keep_engine_queue_bounded() { l1.assert_finished(); l2.assert_finished(); } + +/// While the execution layer catches up to a delegated safe head ahead of the local unsafe head, +/// each delegate poll repeats the same target. Every poll sends it, after a `SYNCING` answer and +/// after a `VALID` one, since an execution layer that restarts can lose an applied forkchoice. +#[tokio::test(start_paused = true)] +async fn delegated_safe_head_is_sent_on_every_poll() { + let config = Arc::new(RollupConfig::default()); + let (client, l1, l2) = test_engine_client(config.clone()); + let state = TestEngineStateBuilder::new() + .with_unsafe_head(test_block_info(20)) + .with_safe_head(test_block_info(10)) + .with_finalized_head(test_block_info(5)) + .build(); + let delegated_safe_head = test_block_info(30); + let target = state + .sync_state + .apply_update(EngineSyncStateUpdate { + local_safe_head: Some(delegated_safe_head), + safe_head: Some(delegated_safe_head), + unsafe_head: Some(delegated_safe_head), + ..Default::default() + }) + .create_forkchoice_state(); + for status in [PayloadStatusEnum::Syncing, PayloadStatusEnum::Valid, PayloadStatusEnum::Valid] { + l2.expect_forkchoice_updated(target, status); + } + let (state_tx, state_rx) = watch::channel(state); + let (requests, request_rx) = mpsc::channel(8); + let (derivation_tx, _derivation_rx) = mpsc::channel(8); + let mut actor = EngineActor::new( + Arc::new(client), + config, + QueuedEngineDerivationClient { derivation_actor_request_tx: derivation_tx }, + Engine::new(state, state_tx), + None, + request_rx, + Arc::new(NoopBlockSink), + ); + + // Three polls report the same target. Each step drains the previous poll's task and then + // accepts the next poll. + for _ in 0..3 { + requests + .send(EngineActorRequest::ProcessSafeL2Signal(ConsolidateInput::BlockInfo( + delegated_safe_head, + ))) + .await + .unwrap(); + time::timeout(Duration::from_secs(1), actor.step()).await.unwrap().unwrap(); + } + drop(requests); + assert!(matches!(actor.step().await, Err(EngineError::ChannelClosed))); + + assert!(actor.engine.is_empty()); + assert_eq!(state_rx.borrow().sync_state.safe_head(), delegated_safe_head); + l1.assert_finished(); + l2.assert_finished(); +} + +/// A finalization the engine skips leaves the actor running, an identical repeat keeps the same +/// skip in memory, and a reset forgets it. +#[tokio::test(start_paused = true)] +async fn skipped_finalization_keeps_the_actor_running_until_a_reset_forgets_it() { + let config = Arc::new(RollupConfig::default()); + let (client, l1, l2) = test_engine_client(config.clone()); + let state = TestEngineStateBuilder::new() + .with_unsafe_head(test_block_info(30)) + .with_safe_head(test_block_info(20)) + .with_finalized_head(test_block_info(10)) + .build(); + let (state_tx, _state_rx) = watch::channel(state); + let (requests, request_rx) = mpsc::channel(8); + let (derivation_tx, _derivation_rx) = mpsc::channel(8); + let mut actor = EngineActor::new( + Arc::new(client), + config, + QueuedEngineDerivationClient { derivation_actor_request_tx: derivation_tx }, + Engine::new(state, state_tx), + None, + request_rx, + Arc::new(NoopBlockSink), + ); + let stale_finalization = + || EngineActorRequest::ProcessFinalizedL2Block(Box::new(FinalizeBlockId::ByNumber(5))); + + // Each step drains the previous request's task, then accepts the next request. + requests.send(stale_finalization()).await.unwrap(); + time::timeout(Duration::from_secs(1), actor.step()).await.unwrap().unwrap(); + requests.send(stale_finalization()).await.unwrap(); + time::timeout(Duration::from_secs(1), actor.step()).await.unwrap().unwrap(); + let first_skip = actor.last_skipped_task_error.clone(); + assert!(first_skip.is_some(), "the skipped finalization is remembered"); + + requests.send(stale_finalization()).await.unwrap(); + time::timeout(Duration::from_secs(1), actor.step()).await.unwrap().unwrap(); + assert_eq!(actor.last_skipped_task_error, first_skip); + + actor.begin_reset(); + assert_eq!(actor.last_skipped_task_error, None); + l1.assert_finished(); + l2.assert_finished(); +} diff --git a/rust/kona/crates/node/service/src/service/node/rollup.rs b/rust/kona/crates/node/service/src/service/node/rollup.rs index d1b5fb0a12e..ed509b22101 100644 --- a/rust/kona/crates/node/service/src/service/node/rollup.rs +++ b/rust/kona/crates/node/service/src/service/node/rollup.rs @@ -289,6 +289,7 @@ impl RollupNode { engine_actor_request_tx: mpsc::Sender, derivation_actor_request_rx: mpsc::Receiver, derivation_origin_tx: watch::Sender>, + engine_state_rx: watch::Receiver, l1_head: watch::Receiver>, ) -> Result { if let Some(provider) = self.derivation_delegate_provider.clone() { @@ -303,6 +304,7 @@ impl RollupNode { provider, l1_provider, derivation_origin_tx, + engine_state_rx, )))) } else { let pipeline = self @@ -621,6 +623,7 @@ impl RollupNode { engine_actor_request_tx.clone(), derivation_actor_request_rx, derivation_origin_tx, + engine_state_rx.clone(), l1_head_updates_rx.clone(), ) .await?;