diff --git a/rust/kona/crates/node/engine/src/task_queue/core.rs b/rust/kona/crates/node/engine/src/task_queue/core.rs index 2167623aaa2..2628f214302 100644 --- a/rust/kona/crates/node/engine/src/task_queue/core.rs +++ b/rust/kona/crates/node/engine/src/task_queue/core.rs @@ -144,7 +144,7 @@ impl Engine { /// Attempts to drain the queue by executing all [`EngineTask`]s in-order. If any task returns /// an error along the way, the error is returned and the task is not popped from the queue (in - /// case it must be retried), unless it is a flush error. + /// case it must be retried), unless it is a flush or skip error. pub async fn drain(&mut self) -> Result<(), EngineTaskErrors> { // Drain tasks in order of priority, halting on errors for a retry to be attempted. loop { @@ -155,9 +155,13 @@ impl Engine { let result = task.execute(&mut self.state).await; // A flush reports that the task replaced its block with a deposits-only block, so the - // task is done; retrying it would replace that block again. + // task is done; retrying it would replace that block again. A skip reports a task that + // no longer applies, so retrying it cannot succeed either. if let Err(err) = &result && - err.severity() != EngineTaskErrorSeverity::Flush + !matches!( + err.severity(), + EngineTaskErrorSeverity::Flush | EngineTaskErrorSeverity::Skip + ) { return result; } @@ -197,7 +201,8 @@ impl EngineResetError { mod recovery_tests { use super::*; use crate::{ - ConsolidateInput, ConsolidateTask, InsertTask, NoopBlockSink, + ConsolidateInput, ConsolidateTask, FinalizeBlockId, FinalizeTask, InsertTask, + NoopBlockSink, test_utils::{ TestAttributesBuilder, TestEngineStateBuilder, test_block_info, test_engine_client, }, @@ -380,4 +385,30 @@ mod recovery_tests { l1.assert_finished(); l2.assert_finished(); } + + // A skipped task is dropped instead of being kept for a retry, so it cannot hold the queue. + #[tokio::test] + async fn skipped_task_is_dropped_from_the_queue() { + let config = Arc::new(RollupConfig::default()); + let (client, l1, l2) = test_engine_client(config.clone()); + let state = TestEngineStateBuilder::new() + .with_unsafe_head(test_block_info(20)) + .with_finalized_head(test_block_info(10)) + .build(); + let (state_tx, _state_rx) = tokio::sync::watch::channel(state); + let mut engine = Engine::new(state, state_tx); + engine.enqueue(EngineTask::Finalize(Box::new(FinalizeTask::new( + Arc::new(client), + config, + FinalizeBlockId::ByNumber(5), + )))); + + let error = engine.drain().await.unwrap_err(); + + assert_eq!(error.severity(), EngineTaskErrorSeverity::Skip, "got {error:?}"); + assert!(engine.is_empty()); + assert_eq!(engine.state().sync_state, state.sync_state); + l1.assert_finished(); + l2.assert_finished(); + } } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/error.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/error.rs index c8444c2595e..51a7f2c1dda 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/error.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/error.rs @@ -3,6 +3,7 @@ use crate::{ EngineTaskError, SynchronizeTaskError, task_queue::tasks::task::EngineTaskErrorSeverity, }; +use alloy_eips::BlockNumHash; use alloy_transport::{RpcError, TransportErrorKind}; use kona_protocol::FromBlockError; use thiserror::Error; @@ -10,33 +11,69 @@ use thiserror::Error; /// An error that occurs when running the [`crate::FinalizeTask`]. #[derive(Debug, Error)] pub enum FinalizeTaskError { - /// The block is not safe, and therefore cannot be finalized. - #[error("Attempted to finalize a block that is not yet safe")] - BlockNotSafe, + /// The target is below the finalized head, and finality never moves backward. + #[error("Finalization target {target} is behind the finalized head {finalized}")] + BehindFinalizedHead { + /// The number of the finalized head. + finalized: u64, + /// The number of the finalization target. + target: u64, + }, /// The block to finalize was not found. #[error("The block to finalize was not found: Number {0}")] BlockNotFound(u64), + /// The block is not safe, and therefore cannot be finalized. + #[error("Attempted to finalize a block that is not yet safe")] + BlockNotSafe, + /// The engine returned a block at a different height than the one requested. + #[error("Requested block {requested} to finalize, but the engine returned block {returned}")] + BlockNumberMismatch { + /// The requested block number. + requested: u64, + /// The number of the block the engine returned. + returned: u64, + }, + /// The target is a different block at the finalized head's height. + #[error("Finalization target {target:?} conflicts with the finalized head {finalized:?}")] + ConflictsWithFinalizedHead { + /// The finalized head. + finalized: BlockNumHash, + /// The finalization target. + target: BlockNumHash, + }, + /// The forkchoice update call to finalize the block failed. + #[error(transparent)] + ForkchoiceUpdateFailed(#[from] SynchronizeTaskError), /// An error occurred while transforming the RPC block into [`L2BlockInfo`]. /// /// [`L2BlockInfo`]: kona_protocol::L2BlockInfo #[error(transparent)] FromBlock(#[from] FromBlockError), + /// The pinned target is not the engine's canonical block at its height. + #[error("Finalization target {target:?} is not canonical; the engine has {canonical:?}")] + NotCanonical { + /// The engine's canonical block at the target's height. + canonical: BlockNumHash, + /// The finalization target. + target: BlockNumHash, + }, /// A temporary RPC failure. #[error(transparent)] TransportError(#[from] RpcError), - /// The forkchoice update call to finalize the block failed. - #[error(transparent)] - ForkchoiceUpdateFailed(#[from] SynchronizeTaskError), } impl EngineTaskError for FinalizeTaskError { fn severity(&self) -> EngineTaskErrorSeverity { match self { - Self::BlockNotSafe | Self::BlockNotFound(_) | Self::FromBlock(_) => { - EngineTaskErrorSeverity::Critical - } - Self::TransportError(_) => EngineTaskErrorSeverity::Temporary, + Self::BehindFinalizedHead { .. } | + Self::ConflictsWithFinalizedHead { .. } | + Self::NotCanonical { .. } => EngineTaskErrorSeverity::Skip, + Self::BlockNotFound(_) | + Self::BlockNotSafe | + Self::BlockNumberMismatch { .. } | + Self::FromBlock(_) => EngineTaskErrorSeverity::Critical, Self::ForkchoiceUpdateFailed(inner) => inner.severity(), + Self::TransportError(_) => EngineTaskErrorSeverity::Temporary, } } } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/id.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/id.rs index ab5a92e4ddb..c7c12fd6e73 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/id.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/id.rs @@ -10,9 +10,9 @@ use alloy_eips::BlockNumHash; /// authoritative source for the block at a given height (e.g. local L1-finality driven by /// `L2Finalizer`). The number is sufficient because there is only one chain to consult. /// - [`FinalizeBlockId::ByHash`] is used when an upstream supplies `(number, hash)` and the engine -/// must finalize the specific block identified by hash. If the engine does not have that hash, -/// the task must error rather than silently finalize the engine's canonical block at the same -/// height. +/// must finalize the specific block identified by hash. If the engine's canonical block at that +/// height has a different hash, the task skips the target rather than finalize the engine's block +/// in its place. /// /// EL finalization is irreversible, so finalizing the wrong block is unrecoverable. Forcing every /// caller to pick a variant prevents accidental hash-loss at the boundary. @@ -20,8 +20,8 @@ use alloy_eips::BlockNumHash; pub enum FinalizeBlockId { /// Finalize whatever block the engine has at the given L2 block number. ByNumber(u64), - /// Finalize the block matching the given `(number, hash)` pair. The task fails if the engine - /// does not have that hash. + /// Finalize the block matching the given `(number, hash)` pair, only once it is the engine's + /// canonical block at that height. ByHash(BlockNumHash), } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs index 3461d7d860f..dac3404750a 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs @@ -4,7 +4,7 @@ use crate::{ EngineClient, EngineState, ExecutableEngineTask, FinalizeBlockId, FinalizeTaskError, state::EngineSyncStateUpdate, task_queue::synchronize, }; -use alloy_eips::BlockId; +use alloy_eips::BlockNumHash; use async_trait::async_trait; use derive_more::Constructor; use kona_genesis::RollupConfig; @@ -13,6 +13,10 @@ use std::sync::Arc; /// The [`FinalizeTask`] fetches the [`L2BlockInfo`] identified by `block_id`, updates the /// [`EngineState`], and dispatches a forkchoice update to finalize the block. +/// +/// Finality only moves forward. A target behind the finalized head, a different block at the +/// finalized height, or a pinned hash that is not the engine's canonical block fails with an error +/// that skips the task and leaves the engine state unchanged. #[derive(Debug, Clone, Constructor)] pub struct FinalizeTask { /// The engine client. @@ -23,6 +27,61 @@ pub struct FinalizeTask { pub block_id: FinalizeBlockId, } +impl FinalizeTask { + /// Whether `target` is the finalized head itself; a different block at its height is an error. + fn is_already_finalized( + finalized_head: &L2BlockInfo, + target: BlockNumHash, + ) -> Result { + let finalized = finalized_head.block_info.id(); + if target == finalized { + debug!(target: "engine", ?target, "Target is already finalized"); + return Ok(true); + } + // The default head means finality has not been established yet, so any target, including + // the configured genesis, may initialize it. + if *finalized_head != L2BlockInfo::default() && target.number == finalized.number { + return Err(FinalizeTaskError::ConflictsWithFinalizedHead { finalized, target }); + } + Ok(false) + } + + /// Fetches the engine's canonical block at the target's height. + /// + /// A pinned hash must be that block: a forkchoice update cannot finalize a block off the + /// canonical chain, and finalizing the canonical block instead would finalize a block the + /// caller never named. The block's identity is checked before it is decoded, so a mismatch is + /// reported as such rather than as a decoding failure. + async fn fetch_canonical_block_info(&self) -> Result { + let requested = self.block_id.number(); + debug!(target: "engine", block_id = ?self.block_id, "Fetching the block to finalize"); + let block = self + .client + .get_l2_block(requested.into()) + .full() + .await + .map_err(FinalizeTaskError::TransportError)? + .ok_or(FinalizeTaskError::BlockNotFound(requested))?; + let canonical = BlockNumHash { hash: block.header.hash, number: block.header.number }; + debug!(target: "engine", ?canonical, "Fetched the block to finalize"); + + if canonical.number != requested { + return Err(FinalizeTaskError::BlockNumberMismatch { + requested, + returned: canonical.number, + }); + } + if let FinalizeBlockId::ByHash(target) = self.block_id && + target.hash != canonical.hash + { + return Err(FinalizeTaskError::NotCanonical { canonical, target }); + } + + L2BlockInfo::from_block_and_genesis(&block.into_consensus(), &self.cfg.genesis) + .map_err(FinalizeTaskError::FromBlock) + } +} + #[async_trait] impl ExecutableEngineTask for FinalizeTask { type Output = (); @@ -30,40 +89,35 @@ impl ExecutableEngineTask for FinalizeTask { type Error = FinalizeTaskError; async fn execute(&self, state: &mut EngineState) -> Result<(), FinalizeTaskError> { - let block_number = self.block_id.number(); + let finalized_head = state.sync_state.finalized_head(); + let target_number = self.block_id.number(); + + // A lagging source can report an older finalized block, or one from another chain. A + // pinned hash is enough to detect either without asking the execution layer. + if target_number < finalized_head.block_info.number { + return Err(FinalizeTaskError::BehindFinalizedHead { + finalized: finalized_head.block_info.number, + target: target_number, + }); + } + if let FinalizeBlockId::ByHash(target) = self.block_id && + Self::is_already_finalized(&finalized_head, target)? + { + return Ok(()); + } // Sanity check that the block that is being finalized is at least safe. - if state.sync_state.safe_head().block_info.number < block_number { + if state.sync_state.safe_head().block_info.number < target_number { return Err(FinalizeTaskError::BlockNotSafe); } - // Look up by hash when the caller pinned a specific hash (delegated polling supplies - // `(number, hash)`); otherwise look up by number. Finalization is irreversible, so a stale - // hash must produce a hard error, not silently finalize whatever the engine has at the - // same height. - let lookup: BlockId = match self.block_id { - FinalizeBlockId::ByHash(id) => id.hash.into(), - FinalizeBlockId::ByNumber(n) => n.into(), - }; - let block = self - .client - .get_l2_block(lookup) - .full() - .await - .map_err(FinalizeTaskError::TransportError)? - .ok_or(FinalizeTaskError::BlockNotFound(block_number))? - .into_consensus(); - let block_info = L2BlockInfo::from_block_and_genesis(&block, &self.client.cfg().genesis) - .map_err(FinalizeTaskError::FromBlock)?; - - // For ByHash, also assert the returned block's height matches the caller's claim. The - // engine should never serve a block at a different height for a given hash, but a height - // mismatch indicates either an upstream protocol bug or a misuse of the API and must - // never lead to silent finalization. - if let FinalizeBlockId::ByHash(id) = self.block_id && - block_info.block_info.number != id.number + let block_info = self.fetch_canonical_block_info().await?; + + // A by-number target only learns its hash from the lookup. + if matches!(self.block_id, FinalizeBlockId::ByNumber(_)) && + Self::is_already_finalized(&finalized_head, block_info.block_info.id())? { - return Err(FinalizeTaskError::BlockNotFound(id.number)); + return Ok(()); } // Dispatch a forkchoice update. diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs index 08c4f153e37..fb273f99831 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs @@ -1,62 +1,263 @@ use crate::{ + EngineState, EngineSyncState, EngineSyncStateUpdate, EngineTaskError, EngineTaskErrorSeverity, ExecutableEngineTask, FinalizeBlockId, FinalizeTask, FinalizeTaskError, - test_utils::{TestEngineStateBuilder, test_engine_client}, + test_utils::{ + RpcMock, TestEngineStateBuilder, TestRpcBlock as RpcBlock, test_block_info, + test_engine_client, test_l2_block, test_rpc_block, + }, }; +use alloy_consensus::Header; use alloy_eips::BlockNumHash; -use alloy_primitives::b256; -use kona_genesis::RollupConfig; -use kona_protocol::{BlockInfo, L2BlockInfo}; +use alloy_primitives::B256; +use alloy_rpc_types_engine::PayloadStatusEnum; +use alloy_rpc_types_eth::{BlockTransactions, Header as RpcHeader}; +use kona_genesis::{ChainGenesis, RollupConfig}; +use kona_protocol::L2BlockInfo; +use rstest::rstest; use std::sync::Arc; -/// When the engine receives a `ByHash` finalize request for a block hash it doesn't have, -/// [`FinalizeTask`] must fail with [`FinalizeTaskError::BlockNotFound`] rather than silently -/// finalize whatever it happens to have at the same height. -/// -/// Assert the exact by-hash RPC lookup, so a by-number lookup cannot silently finalize a -/// different canonical block at the same height. -#[tokio::test] -async fn finalize_task_by_hash_errors_when_engine_lacks_hash() { - const N: u64 = 10; - let hash_a = b256!("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); - let hash_b = b256!("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"); +const STALE: u64 = 5; +const FINALIZED: u64 = 10; +const ADVANCED: u64 = 15; +const SAFE: u64 = 20; +const UNSAFE: u64 = 30; + +/// The L2 genesis block, which carries no L1-info deposit. +fn genesis_block() -> RpcBlock { + RpcBlock::new(RpcHeader::new(Header::default()), BlockTransactions::Full(vec![])) +} + +fn rollup_config() -> Arc { + let genesis = BlockNumHash { hash: genesis_block().header.hash, number: 0 }; + Arc::new(RollupConfig { + genesis: ChainGenesis { l2: genesis, ..Default::default() }, + ..Default::default() + }) +} + +/// An L2 block at `number`. Blocks built from different `fork` values share a height but not a +/// hash. +fn l2_block(number: u64, fork: u8) -> RpcBlock { + test_rpc_block(&test_l2_block(number, B256::ZERO, fork)) +} + +fn canonical(number: u64) -> RpcBlock { + l2_block(number, 0) +} + +fn conflicting(number: u64) -> RpcBlock { + l2_block(number, 1) +} + +fn l2_block_info(block: &RpcBlock) -> L2BlockInfo { + L2BlockInfo::from_block_and_genesis(&block.clone().into_consensus(), &rollup_config().genesis) + .unwrap() +} + +fn by_hash(block: &RpcBlock) -> FinalizeBlockId { + FinalizeBlockId::ByHash(l2_block_info(block).block_info.id()) +} + +fn by_number(block: &RpcBlock) -> FinalizeBlockId { + FinalizeBlockId::ByNumber(block.header.number) +} - let cfg = Arc::new(RollupConfig::default()); +fn established() -> L2BlockInfo { + l2_block_info(&canonical(FINALIZED)) +} + +fn uninitialized() -> L2BlockInfo { + L2BlockInfo::default() +} + +fn engine_state(finalized_head: L2BlockInfo) -> EngineState { + TestEngineStateBuilder::new() + .with_unsafe_head(test_block_info(UNSAFE)) + .with_safe_head(test_block_info(SAFE)) + .with_finalized_head(finalized_head) + .build() +} + +fn with_finalized_head(state: &EngineState, finalized_head: L2BlockInfo) -> EngineSyncState { + state.sync_state.apply_update(EngineSyncStateUpdate { + finalized_head: Some(finalized_head), + ..Default::default() + }) +} + +async fn finalize( + target: FinalizeBlockId, + state: &mut EngineState, + configure_engine: impl FnOnce(&RpcMock), +) -> Result<(), FinalizeTaskError> { + let cfg = rollup_config(); let (engine_client, l1, l2) = test_engine_client(cfg.clone()); - l2.expect_params( - "eth_getBlockByHash", - serde_json::json!([hash_b, true]), - Option::>::None, - ); - - // Place the safe head at N so the sanity check (`safe_head.number >= block_id.number`) passes - // and `execute()` reaches the lookup we care about. - let safe_head = L2BlockInfo { - block_info: BlockInfo { - number: N, - hash: hash_a, - parent_hash: Default::default(), - timestamp: N * 2, - }, - l1_origin: BlockNumHash::default(), - seq_num: 0, - }; - let mut state = - TestEngineStateBuilder::new().with_unsafe_head(safe_head).with_safe_head(safe_head).build(); - - let task = FinalizeTask::new( - Arc::new(engine_client), - cfg, - FinalizeBlockId::ByHash(BlockNumHash { number: N, hash: hash_b }), - ); - - let result = task.execute(&mut state).await; - - assert!( - matches!(result, Err(FinalizeTaskError::BlockNotFound(n)) if n == N), - "expected BlockNotFound({N}) — got {result:?}. The by-hash lookup must fail loudly when \ - the engine lacks the requested hash; instead, the task either succeeded (finalizing the \ - wrong block) or surfaced a different error." - ); + configure_engine(&l2); + + let result = FinalizeTask::new(Arc::new(engine_client), cfg, target).execute(state).await; + l1.assert_finished(); l2.assert_finished(); + result +} + +const fn is_behind(err: &FinalizeTaskError) -> bool { + matches!(err, FinalizeTaskError::BehindFinalizedHead { finalized: FINALIZED, target: STALE }) +} + +const fn is_conflict(err: &FinalizeTaskError) -> bool { + matches!(err, FinalizeTaskError::ConflictsWithFinalizedHead { .. }) +} + +const fn is_not_canonical(err: &FinalizeTaskError) -> bool { + matches!(err, FinalizeTaskError::NotCanonical { .. }) +} + +const fn is_number_mismatch(err: &FinalizeTaskError) -> bool { + matches!(err, FinalizeTaskError::BlockNumberMismatch { requested: ADVANCED, .. }) +} + +const fn is_block_not_found(err: &FinalizeTaskError) -> bool { + matches!(err, FinalizeTaskError::BlockNotFound(ADVANCED)) +} + +/// Targets behind the finalized head, or a pinned hash that conflicts with it, are skipped before +/// asking the execution layer for the block: the test engine expects no request at all. +#[rstest] +#[case::stale_by_hash(by_hash(&canonical(STALE)), is_behind)] +#[case::stale_by_number(by_number(&canonical(STALE)), is_behind)] +#[case::conflicting_by_hash(by_hash(&conflicting(FINALIZED)), is_conflict)] +#[tokio::test] +async fn finalize_task_skips_target_without_lookup( + #[case] target: FinalizeBlockId, + #[case] is_expected_error: fn(&FinalizeTaskError) -> bool, +) { + let mut state = engine_state(established()); + let original_state = state; + + let err = finalize(target, &mut state, |_| {}).await.unwrap_err(); + + assert!(is_expected_error(&err), "{err:?}"); + assert_eq!(err.severity(), EngineTaskErrorSeverity::Skip); + assert_eq!(state, original_state); +} + +/// A target above the safe head is refused before asking the execution layer for the block: the +/// test engine expects no request at all. +#[tokio::test] +async fn finalize_task_refuses_target_above_safe_head_without_lookup() { + let mut state = engine_state(established()); + let original_state = state; + + let err = finalize(by_hash(&canonical(SAFE + 5)), &mut state, |_| {}).await.unwrap_err(); + + assert!(matches!(err, FinalizeTaskError::BlockNotSafe), "{err:?}"); + assert_eq!(err.severity(), EngineTaskErrorSeverity::Critical); + assert_eq!(state, original_state); +} + +/// A target whose block differs from the engine's canonical block is skipped. A pinned hash is +/// never replaced by the canonical block at its height, even above the finalized head, where a +/// delegate on another fork names a block the engine does not consider canonical. +#[rstest] +#[case::conflicting_by_number(by_number(&conflicting(FINALIZED)), conflicting(FINALIZED), is_conflict)] +#[case::other_fork_by_hash(by_hash(&conflicting(ADVANCED)), canonical(ADVANCED), is_not_canonical)] +#[tokio::test] +async fn finalize_task_skips_target_off_the_canonical_chain( + #[case] target: FinalizeBlockId, + #[case] served_block: RpcBlock, + #[case] is_expected_error: fn(&FinalizeTaskError) -> bool, +) { + let mut state = engine_state(established()); + let original_state = state; + + let err = finalize(target, &mut state, |l2| { + l2.expect_block_by_number(target.number(), Some(served_block)) + }) + .await + .unwrap_err(); + + assert!(is_expected_error(&err), "{err:?}"); + assert_eq!(err.severity(), EngineTaskErrorSeverity::Skip); + assert_eq!(state, original_state); +} + +/// A canonical target above the finalized head becomes the finalized head through one forkchoice +/// update. The canonical block at the finalized height is already finalized, so it needs no update. +#[rstest] +#[case::advancing_by_hash(established(), by_hash, canonical(ADVANCED), 1)] +#[case::advancing_by_number(established(), by_number, canonical(ADVANCED), 1)] +#[case::identical_by_number(established(), by_number, canonical(FINALIZED), 0)] +#[case::uninitialized_adopts_genesis(uninitialized(), by_hash, genesis_block(), 1)] +#[tokio::test] +async fn finalize_task_finalizes_canonical_target( + #[case] finalized_head: L2BlockInfo, + #[case] identify: fn(&RpcBlock) -> FinalizeBlockId, + #[case] served_block: RpcBlock, + #[case] forkchoice_updates: usize, +) { + let mut state = engine_state(finalized_head); + let target = identify(&served_block); + let expected_sync_state = with_finalized_head(&state, l2_block_info(&served_block)); + + finalize(target, &mut state, |l2| { + l2.expect_block_by_number(target.number(), Some(served_block)); + for _ in 0..forkchoice_updates { + l2.expect_forkchoice_updated( + expected_sync_state.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + } + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected_sync_state); +} + +/// A target pinned to the finalized head is already finalized, so it needs no lookup. +#[tokio::test] +async fn finalize_task_skips_lookup_for_finalized_head() { + let mut state = engine_state(established()); + let original_state = state; + + finalize(by_hash(&canonical(FINALIZED)), &mut state, |_| {}).await.unwrap(); + + assert_eq!(state, original_state); +} + +/// The engine serving no block, or a block at a height other than the requested one, stops the +/// node instead of finalizing anything. A misplaced block could otherwise bypass the guards against +/// moving finality backward. +#[rstest] +#[case::missing_by_number(FinalizeBlockId::ByNumber(ADVANCED), None, is_block_not_found)] +#[case::missing_by_hash(by_hash(&canonical(ADVANCED)), None, is_block_not_found)] +#[case::stale_block_by_number( + /* target */ FinalizeBlockId::ByNumber(ADVANCED), + /* served_block */ Some(canonical(STALE)), + /* is_expected_error */ is_number_mismatch +)] +#[case::higher_block_by_number( + /* target */ FinalizeBlockId::ByNumber(ADVANCED), + /* served_block */ Some(canonical(SAFE)), + /* is_expected_error */ is_number_mismatch +)] +#[case::stale_block_by_hash(by_hash(&canonical(ADVANCED)), Some(canonical(STALE)), is_number_mismatch)] +#[tokio::test] +async fn finalize_task_stops_on_unexpected_lookup_result( + #[case] target: FinalizeBlockId, + #[case] served_block: Option, + #[case] is_expected_error: fn(&FinalizeTaskError) -> bool, +) { + let mut state = engine_state(established()); + let original_state = state; + + let err = + finalize(target, &mut state, |l2| l2.expect_block_by_number(target.number(), served_block)) + .await + .unwrap_err(); + + assert!(is_expected_error(&err), "{err:?}"); + assert_eq!(err.severity(), EngineTaskErrorSeverity::Critical); + assert_eq!(state, original_state); } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/task.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/task.rs index ee976a02201..6b3a3d1b4cb 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/task.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/task.rs @@ -32,6 +32,10 @@ pub enum EngineTaskErrorSeverity { /// The error indicates that the engine should be flushed. #[display("flush")] Flush, + /// The task no longer applies to the engine state, so it is discarded instead of retried and + /// the engine keeps running. + #[display("skip")] + Skip, } /// The interface for an engine task error. diff --git a/rust/kona/crates/node/engine/src/test_utils/blocks.rs b/rust/kona/crates/node/engine/src/test_utils/blocks.rs new file mode 100644 index 00000000000..434d3bd01eb --- /dev/null +++ b/rust/kona/crates/node/engine/src/test_utils/blocks.rs @@ -0,0 +1,40 @@ +//! L2 blocks, and their RPC form as the execution layer serves them. + +use super::l1_info_deposit; +use alloy_consensus::transaction::Recovered; +use alloy_primitives::{Address, B256, Bytes}; +use alloy_rpc_types_eth::{BlockTransactions, Header as RpcHeader}; +use op_alloy_consensus::OpBlock; + +/// A full L2 block as the execution layer serves it over RPC. +pub type TestRpcBlock = alloy_rpc_types_eth::Block; + +/// An L2 block at `number` on `parent_hash` whose only transaction is the L1-info deposit, so it +/// converts into an `L2BlockInfo` without being the configured genesis. Blocks with different +/// `extra_data` share a height but not a hash. +pub fn test_l2_block(number: u64, parent_hash: B256, extra_data: u8) -> OpBlock { + let mut block = OpBlock::default(); + block.header.extra_data = Bytes::from(vec![extra_data]); + block.header.number = number; + block.header.parent_hash = parent_hash; + block.header.timestamp = number * 2; + block.body.transactions.push(l1_info_deposit()); + block +} + +/// `block` as the execution layer serves it over RPC. +pub fn test_rpc_block(block: &OpBlock) -> TestRpcBlock { + let transactions = block + .body + .transactions + .iter() + .cloned() + .map(|transaction| { + op_alloy_rpc_types::Transaction::from_transaction( + Recovered::new_unchecked(transaction, Address::ZERO), + Default::default(), + ) + }) + .collect(); + TestRpcBlock::new(RpcHeader::new(block.header.clone()), BlockTransactions::Full(transactions)) +} diff --git a/rust/kona/crates/node/engine/src/test_utils/mod.rs b/rust/kona/crates/node/engine/src/test_utils/mod.rs index e5949f23225..0a60403a0d6 100644 --- a/rust/kona/crates/node/engine/src/test_utils/mod.rs +++ b/rust/kona/crates/node/engine/src/test_utils/mod.rs @@ -1,6 +1,9 @@ mod attributes; pub use attributes::TestAttributesBuilder; +mod blocks; +pub use blocks::{TestRpcBlock, test_l2_block, test_rpc_block}; + mod deposits; pub use deposits::l1_info_deposit; diff --git a/rust/kona/crates/node/engine/src/test_utils/rpc.rs b/rust/kona/crates/node/engine/src/test_utils/rpc.rs index 1cfc3995356..39f23f14de2 100644 --- a/rust/kona/crates/node/engine/src/test_utils/rpc.rs +++ b/rust/kona/crates/node/engine/src/test_utils/rpc.rs @@ -1,6 +1,6 @@ //! Scripted RPC expectations over Alloy's mock transport. -use crate::EngineClient; +use crate::{EngineClient, test_utils::TestRpcBlock}; use alloy_json_rpc::{ErrorPayload, RequestPacket, ResponsePacket}; use alloy_network::Network; use alloy_provider::RootProvider; @@ -59,6 +59,16 @@ impl RpcMock { ); } + /// Expects a full-block `eth_getBlockByNumber` call for `number` and answers it with `block`, + /// or with null for `None`. + pub fn expect_block_by_number(&self, number: u64, block: Option) { + self.expect_params( + "eth_getBlockByNumber", + serde_json::json!([format!("{number:#x}"), true]), + block, + ); + } + /// Expects an `engine_forkchoiceUpdatedV3` call for `forkchoice` without payload attributes and /// answers it with `status`. pub fn expect_forkchoice_updated( diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs index 44fe763d969..9962dd9e388 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs @@ -2,10 +2,11 @@ use crate::{ DerivationActorRequest, DerivationEngineClient, NodeActor, actors::derivation::{DerivationDelegateProvider, DerivationError}, }; +use alloy_eips::BlockNumHash; use alloy_primitives::BlockHash; use async_trait::async_trait; use kona_derive::{ChainProvider, Signal}; -use kona_engine::FinalizeBlockId; +use kona_engine::{EngineState, FinalizeBlockId}; use kona_protocol::{BlockInfo, L2BlockInfo, SyncStatus}; use thiserror::Error; use tokio::{ @@ -42,6 +43,10 @@ where /// Publishes the delegate's `current_l1`: the L1 block that its derivation has reached. derivation_origin_tx: watch::Sender>, + /// The engine's state as it publishes it. Delegate statuses are checked against the + /// finalized head the engine applied, not the last one forwarded to it, which the engine may + /// have skipped. + engine_state_rx: watch::Receiver, /// The engine's L2 safe head, according to updates from the Engine. engine_l2_safe_head: L2BlockInfo, /// Whether the engine sync has completed. This will only ever go from false -> true. @@ -64,6 +69,7 @@ where derivation_delegate_provider: DelegateProvider, l1_provider: L1Provider, derivation_origin_tx: watch::Sender>, + engine_state_rx: watch::Receiver, ) -> Self { let mut delegated_derivation_ticker = time::interval(Self::DERIVATION_DELEGATE_POLL_INTERVAL); @@ -74,6 +80,7 @@ where derivation_delegate_provider, l1_provider, derivation_origin_tx, + engine_state_rx, engine_l2_safe_head: L2BlockInfo::default(), has_engine_sync_completed: false, delegated_derivation_ticker, @@ -149,9 +156,59 @@ where Ok(()) } - /// Verifies that the L1 info reported by the derivation delegate - /// are consistent with canonical L1 chain. + /// The id of an established finalized head. The default head means finality has not been + /// established yet, so any finalized block from the delegate may initialize it. + fn established_finalized_l2(finalized_head: L2BlockInfo) -> Option { + (finalized_head != L2BlockInfo::default()).then(|| finalized_head.block_info.id()) + } + + /// Rejects a finalized L2 block behind, or at the same height as but different from, the + /// engine's. Either means the delegate lags or follows another chain, so none of its status + /// applies: its safe head would move the engine back or onto that chain too. + /// + /// Only a fork at the engine's finalized height is caught here. A chain that forked below it + /// and reports a higher finalized block passes, and the execution layer rejects the forkchoice + /// update that would move onto it. + fn validate_finalized_l2_progress( + &self, + reported: BlockNumHash, + ) -> Result<(), DerivationDelegationError> { + let engine_finalized_head = self.engine_state_rx.borrow().sync_state.finalized_head(); + let Some(engine) = Self::established_finalized_l2(engine_finalized_head) else { + return Ok(()); + }; + if reported.number < engine.number { + return Err(DerivationDelegationError::FinalizedL2Regressed { engine, reported }); + } + if reported.number == engine.number && reported.hash != engine.hash { + return Err(DerivationDelegationError::FinalizedL2Conflict { engine, reported }); + } + Ok(()) + } + + /// Verifies that the derivation delegate's finalized L2 block is not above its safe L2 block + /// and does not move back from the engine's, and that its L1 info is consistent with the + /// canonical L1 chain. async fn validate_sync_status(&mut self, v: &SyncStatus) -> bool { + // A finalized block above the safe head is an inconsistent snapshot; the engine would only + // refuse to finalize it. + if v.finalized_l2.block_info.number > v.safe_l2.block_info.number { + let err = DerivationDelegationError::FinalizedAheadOfSafe { + finalized: v.finalized_l2.block_info.number, + safe: v.safe_l2.block_info.number, + }; + warn!(target: "derivation", error = %err, "Inconsistent sync status from delegate"); + return false; + } + if let Err(err) = self.validate_finalized_l2_progress(v.finalized_l2.block_info.id()) { + warn!( + target: "derivation", + error = %err, + "Finalized L2 inconsistency detected at sync status from delegate" + ); + return false; + } + let checks = [ ("L1 Origin of Safe L2", v.safe_l2.l1_origin.number, v.safe_l2.l1_origin.hash), ( @@ -249,6 +306,18 @@ where #[derive(Error, Debug)] enum DerivationDelegationError { + /// The delegate's finalized L2 block is above its safe L2 block. + #[error("Finalized L2 {finalized} is ahead of safe L2 {safe}")] + FinalizedAheadOfSafe { finalized: u64, safe: u64 }, + + /// The delegate's finalized L2 block is a different block at the engine's finalized height. + #[error("Finalized L2 {reported:?} conflicts with the engine's finalized L2 {engine:?}")] + FinalizedL2Conflict { engine: BlockNumHash, reported: BlockNumHash }, + + /// The delegate's finalized L2 block is behind the engine's. + #[error("Finalized L2 {reported:?} is behind the engine's finalized L2 {engine:?}")] + FinalizedL2Regressed { engine: BlockNumHash, reported: BlockNumHash }, + /// The L1 provider returned an error (network, RPC, etc.) #[error("L1 provider error: {0}")] L1Provider(String), diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/mod.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/mod.rs index e5460eceb11..5431e58aa93 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/mod.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/mod.rs @@ -5,3 +5,6 @@ mod client; pub use client::{ DerivationDelegateClient, DerivationDelegateClientError, DerivationDelegateProvider, }; + +#[cfg(test)] +mod tests; diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs new file mode 100644 index 00000000000..2468e203727 --- /dev/null +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs @@ -0,0 +1,283 @@ +use super::{DelegateDerivationActor, DerivationDelegateClientError, DerivationDelegateProvider}; +use crate::{DerivationActorRequest, EngineActorRequest, NodeActor, QueuedDerivationEngineClient}; +use alloy_eips::BlockNumHash; +use alloy_primitives::B256; +use async_trait::async_trait; +use kona_derive::test_utils::TestChainProvider; +use kona_engine::{ + ConsolidateInput, EngineState, EngineSyncState, EngineSyncStateUpdate, FinalizeBlockId, +}; +use kona_protocol::{BlockInfo, L2BlockInfo, SyncStatus}; +use rstest::rstest; +use std::{collections::VecDeque, sync::Mutex}; +use tokio::sync::{mpsc, watch}; + +const SAFE_ORIGIN: u64 = 1; +const FINALIZED_ORIGIN: u64 = 2; +const CURRENT_L1: u64 = 3; + +type SyncStatusReply = Result; + +/// Answers each sync status poll with the next scripted reply. +#[derive(Debug)] +struct ScriptedDelegate(Mutex>); + +#[async_trait] +impl DerivationDelegateProvider for ScriptedDelegate { + async fn fetch_sync_status(&self) -> SyncStatusReply { + self.0.lock().unwrap().pop_front().expect("unexpected sync status poll") + } +} + +/// A delegate actor over a canonical L1 chain, with the engine requests it sends and the +/// `current_l1` it publishes. +struct Harness { + actor: + DelegateDerivationActor, + current_l1: watch::Receiver>, + derivation_requests: mpsc::Sender, + engine_requests: mpsc::Receiver, + engine_state: watch::Sender, +} + +impl Harness { + fn new(replies: impl IntoIterator) -> Self { + let mut l1 = TestChainProvider::default(); + for number in [SAFE_ORIGIN, FINALIZED_ORIGIN, CURRENT_L1] { + l1.insert_block(number, canonical_l1_block(number)); + } + let (derivation_requests, derivation_request_rx) = mpsc::channel(8); + let (engine_request_tx, engine_requests) = mpsc::channel(8); + let (current_l1_tx, current_l1) = watch::channel(None); + let (engine_state, engine_state_rx) = watch::channel(EngineState::default()); + let actor = DelegateDerivationActor::new( + QueuedDerivationEngineClient::new(engine_request_tx), + derivation_request_rx, + ScriptedDelegate(Mutex::new(replies.into_iter().collect())), + l1, + current_l1_tx, + engine_state_rx, + ); + Self { actor, current_l1, derivation_requests, engine_requests, engine_state } + } + + async fn complete_engine_sync(&mut self) { + self.complete_engine_sync_with(EngineSyncState::default()).await; + } + + /// Completes the engine sync with `sync_state` as the engine's labels, which the engine + /// publishes before it reports the completion. + async fn complete_engine_sync_with(&mut self, sync_state: EngineSyncState) { + self.publish_engine_sync_state(sync_state); + self.derivation_requests + .send(DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new( + sync_state.local_safe_head(), + ))) + .await + .unwrap(); + self.actor.step().await.unwrap(); + } + + /// Publishes the engine's labels as the engine does after each task it runs. + fn publish_engine_sync_state(&self, sync_state: EngineSyncState) { + self.engine_state.send_modify(|state| state.sync_state = sync_state); + } + + /// The engine requests sent since the last call, in order. + fn forwarded(&mut self) -> Vec { + std::iter::from_fn(|| self.engine_requests.try_recv().ok()).map(Forwarded::from).collect() + } +} + +/// The parts of an engine request that the delegate actor decides. +#[derive(Debug, PartialEq, Eq)] +enum Forwarded { + Finalized(FinalizeBlockId), + Other(String), + Safe(L2BlockInfo), +} + +impl From for Forwarded { + fn from(request: EngineActorRequest) -> Self { + match request { + EngineActorRequest::ProcessFinalizedL2Block(finalized) => Self::Finalized(*finalized), + EngineActorRequest::ProcessSafeL2Signal(ConsolidateInput::BlockInfo(safe)) => { + Self::Safe(safe) + } + other => Self::Other(format!("{other:?}")), + } + } +} + +/// What one poll of `status` forwards: the safe head, then the finalized block pinned by hash. +fn forwarded_for(status: &SyncStatus) -> Vec { + vec![ + Forwarded::Safe(status.safe_l2), + Forwarded::Finalized(FinalizeBlockId::ByHash(status.finalized_l2.block_info.id())), + ] +} + +/// What one poll forwards when the status is accepted, and nothing when it is rejected. +fn forwarded_if(accepted: bool, status: &SyncStatus) -> Vec { + if accepted { forwarded_for(status) } else { Vec::new() } +} + +fn canonical_l1_block(number: u64) -> BlockInfo { + BlockInfo { hash: B256::with_last_byte(number as u8), number, ..Default::default() } +} + +fn l2_block(number: u64, l1_origin: u64) -> L2BlockInfo { + L2BlockInfo { + block_info: BlockInfo { + hash: B256::with_last_byte(0x80 | number as u8), + number, + ..Default::default() + }, + l1_origin: canonical_l1_block(l1_origin).id(), + seq_num: 0, + } +} + +/// A sync status whose safe origin, finalized origin and `current_l1` are all canonical. +fn delegate_status() -> SyncStatus { + SyncStatus { + current_l1: canonical_l1_block(CURRENT_L1), + current_l1_finalized: BlockInfo::default(), + finalized_l1: BlockInfo::default(), + finalized_l2: l2_block(15, FINALIZED_ORIGIN), + head_l1: BlockInfo::default(), + local_safe_l2: l2_block(20, SAFE_ORIGIN), + safe_l1: BlockInfo::default(), + safe_l2: l2_block(20, SAFE_ORIGIN), + unsafe_l2: l2_block(25, SAFE_ORIGIN), + } +} + +fn non_canonical(id: BlockNumHash) -> BlockNumHash { + BlockNumHash { hash: B256::repeat_byte(0xee), ..id } +} + +/// Returns `status` with its finalized L2 block replaced by `finalized_l2`. +fn with_finalized_l2(finalized_l2: L2BlockInfo) -> SyncStatus { + SyncStatus { finalized_l2, ..delegate_status() } +} + +/// An engine sync state whose finalized head is `finalized_l2`. +fn engine_finalized_at(finalized_l2: L2BlockInfo) -> EngineSyncState { + EngineSyncState::default().apply_update(EngineSyncStateUpdate { + finalized_head: Some(finalized_l2), + ..Default::default() + }) +} + +fn conflicting(block: L2BlockInfo) -> L2BlockInfo { + L2BlockInfo { + block_info: BlockInfo { + hash: non_canonical(block.block_info.id()).hash, + ..block.block_info + }, + ..block + } +} + +/// The engine's finalized head when its sync completes is the baseline, so a delegate that starts +/// behind it, or on another chain at its height, is rejected whole from the first poll on. +#[rstest] +#[case::advancing(l2_block(16, FINALIZED_ORIGIN), true)] +#[case::unchanged(l2_block(15, FINALIZED_ORIGIN), true)] +#[case::regressed(l2_block(14, FINALIZED_ORIGIN), false)] +#[case::conflicting(conflicting(l2_block(15, FINALIZED_ORIGIN)), false)] +#[tokio::test(start_paused = true)] +async fn first_poll_rejects_status_behind_engine_finality( + #[case] finalized_l2: L2BlockInfo, + #[case] accepted: bool, +) { + let status = with_finalized_l2(finalized_l2); + let mut harness = Harness::new([Ok(status.clone())]); + harness.complete_engine_sync_with(engine_finalized_at(l2_block(15, FINALIZED_ORIGIN))).await; + + harness.actor.step().await.unwrap(); + + assert_eq!(harness.forwarded(), forwarded_if(accepted, &status)); + assert_eq!(harness.current_l1.borrow().is_some(), accepted); +} + +/// Once the engine applies a forwarded finalized block, a status whose finalized block is behind +/// it, or is a different block at its height, is rejected whole: a lagging or diverged delegate's +/// safe head must not reach the engine either, and its `current_l1` is not published. +#[rstest] +#[case::advancing(l2_block(16, FINALIZED_ORIGIN), true)] +#[case::unchanged(l2_block(15, FINALIZED_ORIGIN), true)] +#[case::regressed(l2_block(14, FINALIZED_ORIGIN), false)] +#[case::conflicting(conflicting(l2_block(15, FINALIZED_ORIGIN)), false)] +#[tokio::test(start_paused = true)] +async fn poll_rejects_status_whose_finality_moves_back( + #[case] next_finalized_l2: L2BlockInfo, + #[case] accepted: bool, +) { + let next_status = with_finalized_l2(next_finalized_l2); + let mut harness = Harness::new([Ok(delegate_status()), Ok(next_status.clone())]); + harness.complete_engine_sync().await; + harness.actor.step().await.unwrap(); + assert_eq!(harness.forwarded(), forwarded_for(&delegate_status())); + harness.publish_engine_sync_state(engine_finalized_at(delegate_status().finalized_l2)); + harness.current_l1.borrow_and_update(); + + harness.actor.step().await.unwrap(); + + assert_eq!(harness.forwarded(), forwarded_if(accepted, &next_status)); + assert_eq!(harness.current_l1.has_changed().unwrap(), accepted); +} + +/// A finalized block the engine skipped, such as one on another chain, does not become the +/// baseline: the canonical block at its height, or below it but above the engine's finalized +/// head, is still accepted, so the delegate's safe head keeps reaching the engine. +#[rstest] +#[case::canonical_at_skipped_height(l2_block(17, FINALIZED_ORIGIN))] +#[case::canonical_below_skipped_height(l2_block(16, FINALIZED_ORIGIN))] +#[tokio::test(start_paused = true)] +async fn poll_accepts_status_after_engine_skipped_forwarded_finality( + #[case] canonical_finalized_l2: L2BlockInfo, +) { + let skipped_status = with_finalized_l2(conflicting(l2_block(17, FINALIZED_ORIGIN))); + let canonical_status = with_finalized_l2(canonical_finalized_l2); + let mut harness = Harness::new([Ok(skipped_status.clone()), Ok(canonical_status.clone())]); + harness.complete_engine_sync_with(engine_finalized_at(l2_block(15, FINALIZED_ORIGIN))).await; + harness.actor.step().await.unwrap(); + assert_eq!(harness.forwarded(), forwarded_for(&skipped_status)); + + harness.actor.step().await.unwrap(); + + assert_eq!(harness.forwarded(), forwarded_for(&canonical_status)); +} + +/// Before the engine establishes finality, any finalized block from the delegate may initialize +/// it, including one at genesis height. +#[tokio::test(start_paused = true)] +async fn first_poll_accepts_finality_before_the_engine_establishes_it() { + let status = with_finalized_l2(l2_block(0, FINALIZED_ORIGIN)); + let mut harness = Harness::new([Ok(status.clone())]); + harness.complete_engine_sync().await; + + harness.actor.step().await.unwrap(); + + assert_eq!(harness.forwarded(), forwarded_for(&status)); +} + +/// A status whose finalized block is above its safe block is inconsistent and rejected whole. +#[rstest] +#[case::at_safe_head(l2_block(20, FINALIZED_ORIGIN), true)] +#[case::ahead_of_safe_head(l2_block(21, FINALIZED_ORIGIN), false)] +#[tokio::test(start_paused = true)] +async fn poll_rejects_status_whose_finalized_is_ahead_of_its_safe( + #[case] finalized_l2: L2BlockInfo, + #[case] accepted: bool, +) { + let status = with_finalized_l2(finalized_l2); + let mut harness = Harness::new([Ok(status.clone())]); + harness.complete_engine_sync().await; + + harness.actor.step().await.unwrap(); + + assert_eq!(harness.forwarded(), forwarded_if(accepted, &status)); +} diff --git a/rust/kona/crates/node/service/src/actors/engine/actor.rs b/rust/kona/crates/node/service/src/actors/engine/actor.rs index c9affd16523..f09047db8a2 100644 --- a/rust/kona/crates/node/service/src/actors/engine/actor.rs +++ b/rust/kona/crates/node/service/src/actors/engine/actor.rs @@ -5,8 +5,8 @@ use async_trait::async_trait; use kona_derive::{ResetSignal, Signal}; use kona_engine::{ BuildTask, CanonicalizeTask, ConsolidateTask, Engine, EngineActorRequest, EngineClient, - EngineTask, EngineTaskError, EngineTaskErrorSeverity, FinalizeTask, ImportedBlockSink, - InsertTask, SealTask, + EngineTask, EngineTaskError, EngineTaskErrorSeverity, EngineTaskErrors, FinalizeTask, + ImportedBlockSink, InsertTask, SealTask, }; use kona_genesis::RollupConfig; use kona_protocol::L2BlockInfo; @@ -34,6 +34,9 @@ where el_sync_complete: bool, /// The last local-safe head sent to the derivation actor. last_engine_safe_head_sent: L2BlockInfo, + /// The error of the last skipped engine task, so an identical repeat logs no new warning + /// until a reset changes the state the task was judged against. + last_skipped_task_error: Option, /// A channel to use to relay the current unsafe head. /// ## Note /// This is `Some` when the node is in sequencer mode, and `None` when the node is in validator @@ -81,6 +84,7 @@ where el_sync_complete: false, engine, last_engine_safe_head_sent: L2BlockInfo::default(), + last_skipped_task_error: None, rollup: config, unsafe_head_tx, inbound_request_rx, @@ -92,6 +96,18 @@ where } } + /// Logs a skipped task at `warn!` when its error differs from the last skip, and at `debug!` + /// while the identical error repeats, so a target resubmitted on every poll warns once. + fn log_skipped_task(&mut self, err: &EngineTaskErrors) { + let message = err.to_string(); + if self.last_skipped_task_error.as_ref() == Some(&message) { + debug!(target: "engine", %err, "Skipped the same engine task again"); + return; + } + warn!(target: "engine", %err, "Skipped an engine task that no longer applies"); + self.last_skipped_task_error = Some(message); + } + fn defer_retry(&mut self) { self.retry_at = Instant::now() + self.retry_delay; self.retry_delay = (self.retry_delay * 2).min(Duration::from_secs(5)); @@ -100,6 +116,7 @@ where fn begin_reset(&mut self) { if !self.reset_pending { self.engine.clear(); + self.last_skipped_task_error = None; self.reset_pending = true; self.retry_at = Instant::now(); } @@ -210,6 +227,7 @@ where self.defer_retry(); return Ok(()); } + EngineTaskErrorSeverity::Skip => self.log_skipped_task(&err), } } } diff --git a/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs b/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs index 3dc21d249e1..0b4bc8f58ff 100644 --- a/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs +++ b/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs @@ -4,7 +4,7 @@ use alloy_rpc_types_engine::{ ExecutionPayloadV1, ForkchoiceUpdated, PayloadStatus, PayloadStatusEnum, }; use kona_engine::{ - ConsolidateInput, EngineSyncStateUpdate, NoopBlockSink, + ConsolidateInput, EngineSyncStateUpdate, FinalizeBlockId, NoopBlockSink, test_utils::{TestEngineStateBuilder, l1_info_deposit, test_block_info, test_engine_client}, }; use op_alloy_consensus::OpBlock; @@ -140,3 +140,47 @@ async fn delegated_safe_head_is_sent_on_every_poll() { l1.assert_finished(); l2.assert_finished(); } + +/// A finalization the engine skips leaves the actor running, an identical repeat keeps the same +/// skip in memory, and a reset forgets it. +#[tokio::test(start_paused = true)] +async fn skipped_finalization_keeps_the_actor_running_until_a_reset_forgets_it() { + let config = Arc::new(RollupConfig::default()); + let (client, l1, l2) = test_engine_client(config.clone()); + let state = TestEngineStateBuilder::new() + .with_unsafe_head(test_block_info(30)) + .with_safe_head(test_block_info(20)) + .with_finalized_head(test_block_info(10)) + .build(); + let (state_tx, _state_rx) = watch::channel(state); + let (requests, request_rx) = mpsc::channel(8); + let (derivation_tx, _derivation_rx) = mpsc::channel(8); + let mut actor = EngineActor::new( + Arc::new(client), + config, + QueuedEngineDerivationClient { derivation_actor_request_tx: derivation_tx }, + Engine::new(state, state_tx), + None, + request_rx, + Arc::new(NoopBlockSink), + ); + let stale_finalization = + || EngineActorRequest::ProcessFinalizedL2Block(Box::new(FinalizeBlockId::ByNumber(5))); + + // Each step drains the previous request's task, then accepts the next request. + requests.send(stale_finalization()).await.unwrap(); + time::timeout(Duration::from_secs(1), actor.step()).await.unwrap().unwrap(); + requests.send(stale_finalization()).await.unwrap(); + time::timeout(Duration::from_secs(1), actor.step()).await.unwrap().unwrap(); + let first_skip = actor.last_skipped_task_error.clone(); + assert!(first_skip.is_some(), "the skipped finalization is remembered"); + + requests.send(stale_finalization()).await.unwrap(); + time::timeout(Duration::from_secs(1), actor.step()).await.unwrap().unwrap(); + assert_eq!(actor.last_skipped_task_error, first_skip); + + actor.begin_reset(); + assert_eq!(actor.last_skipped_task_error, None); + l1.assert_finished(); + l2.assert_finished(); +} diff --git a/rust/kona/crates/node/service/src/service/node/rollup.rs b/rust/kona/crates/node/service/src/service/node/rollup.rs index d1b5fb0a12e..ed509b22101 100644 --- a/rust/kona/crates/node/service/src/service/node/rollup.rs +++ b/rust/kona/crates/node/service/src/service/node/rollup.rs @@ -289,6 +289,7 @@ impl RollupNode { engine_actor_request_tx: mpsc::Sender, derivation_actor_request_rx: mpsc::Receiver, derivation_origin_tx: watch::Sender>, + engine_state_rx: watch::Receiver, l1_head: watch::Receiver>, ) -> Result { if let Some(provider) = self.derivation_delegate_provider.clone() { @@ -303,6 +304,7 @@ impl RollupNode { provider, l1_provider, derivation_origin_tx, + engine_state_rx, )))) } else { let pipeline = self @@ -621,6 +623,7 @@ impl RollupNode { engine_actor_request_tx.clone(), derivation_actor_request_rx, derivation_origin_tx, + engine_state_rx.clone(), l1_head_updates_rx.clone(), ) .await?;