From 3ca85833f58d2b22adcea5be86104d3f386c6f07 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Fri, 6 Nov 2026 20:36:49 -0800 Subject: [PATCH 1/5] kona-engine: prevent delegated finalized-head regression A lagging delegate could move finality backward, and a delegate on another fork could name a finalized block that is not the execution layer's canonical block. The first is irreversible. The second either stopped the node or failed every forkchoice update and reset the engine on each poll. The finalize task now looks the target up by number and finalizes a pinned hash only when it is the canonical block at that height. A target behind the finalized head, a different block at the finalized height, or a pinned hash that is not canonical fails with an error of the new `Skip` severity: the engine drops the task, counts it as a failure, and the engine actor warns once per distinct error until a reset. A target equal to the finalized head needs no lookup or forkchoice update, and each finality guard runs once per kind of target. A block served at another height is a distinct `BlockNumberMismatch`. The delegate actor also rejects a whole sync status whose finalized L2 block is behind, or conflicts with, the engine's, so a lagging or diverged delegate's safe head does not reach the engine either. The baseline is the finalized head the engine applied, read from the state it publishes, not the last target forwarded to it: a target the engine skipped must not make the canonical block at its height look like a conflict, which would stop the delegate's safe head too. BREAKING CHANGE: `EngineTaskErrorSeverity` has a new `Skip` variant, and `FinalizeTaskError` has new variants `BehindFinalizedHead`, `BlockNumberMismatch`, `ConflictsWithFinalizedHead` and `NotCanonical`; exhaustive matches must handle them. A block served at another height is now `BlockNumberMismatch` instead of `BlockNotFound`. `DerivationActorRequest::ProcessEngineSyncCompletionRequest` carries an `EngineSyncState`, and `EngineDerivationClient::notify_sync_completed` takes one. `DelegateDerivationActor::new` takes the engine's state receiver. Co-Authored-By: Claude Opus 5.5 Co-Authored-By: Claude Fable 5.1 --- .../crates/node/engine/src/task_queue/core.rs | 39 ++- .../src/task_queue/tasks/finalize/error.rs | 57 +++- .../src/task_queue/tasks/finalize/id.rs | 10 +- .../src/task_queue/tasks/finalize/task.rs | 134 ++++++-- .../task_queue/tasks/finalize/task_test.rs | 289 ++++++++++++++---- .../node/engine/src/task_queue/tasks/task.rs | 4 + .../node/engine/src/test_utils/blocks.rs | 40 +++ .../crates/node/engine/src/test_utils/mod.rs | 3 + .../crates/node/engine/src/test_utils/rpc.rs | 12 +- .../src/actors/derivation/actor/core.rs | 3 +- .../src/actors/derivation/actor/tests.rs | 29 +- .../actors/derivation/actor/tests/database.rs | 2 +- .../derivation/actor/tests/op_node_parity.rs | 2 +- .../src/actors/derivation/delegated/actor.rs | 60 +++- .../src/actors/derivation/delegated/mod.rs | 3 + .../src/actors/derivation/delegated/tests.rs | 250 +++++++++++++++ .../service/src/actors/derivation/request.rs | 7 +- .../node/service/src/actors/engine/actor.rs | 24 +- .../node/service/src/actors/engine/client.rs | 16 +- .../node/service/src/actors/engine/tests.rs | 2 +- .../node/service/src/service/node/rollup.rs | 3 + 21 files changed, 855 insertions(+), 134 deletions(-) create mode 100644 rust/kona/crates/node/engine/src/test_utils/blocks.rs create mode 100644 rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs diff --git a/rust/kona/crates/node/engine/src/task_queue/core.rs b/rust/kona/crates/node/engine/src/task_queue/core.rs index acc3fa27d40..e9d9dcb4f1c 100644 --- a/rust/kona/crates/node/engine/src/task_queue/core.rs +++ b/rust/kona/crates/node/engine/src/task_queue/core.rs @@ -145,7 +145,7 @@ impl Engine { /// Attempts to drain the queue by executing all [`EngineTask`]s in-order. If any task returns /// an error along the way, the error is returned and the task is not popped from the queue (in - /// case it must be retried), unless it is a flush error. + /// case it must be retried), unless it is a flush or skip error. pub async fn drain(&mut self) -> Result<(), EngineTaskErrors> { // Drain tasks in order of priority, halting on errors for a retry to be attempted. loop { @@ -156,9 +156,13 @@ impl Engine { let result = task.execute(&mut self.state).await; // A flush reports that the task replaced its block with a deposits-only block, so the - // task is done; retrying it would replace that block again. + // task is done; retrying it would replace that block again. A skip reports a task that + // no longer applies, so retrying it cannot succeed either. if let Err(err) = &result && - err.severity() != EngineTaskErrorSeverity::Flush + !matches!( + err.severity(), + EngineTaskErrorSeverity::Flush | EngineTaskErrorSeverity::Skip + ) { return result; } @@ -198,7 +202,8 @@ impl EngineResetError { mod recovery_tests { use super::*; use crate::{ - ConsolidateInput, ConsolidateTask, InsertTask, NoopBlockSink, + ConsolidateInput, ConsolidateTask, FinalizeBlockId, FinalizeTask, InsertTask, + NoopBlockSink, test_utils::{ TestAttributesBuilder, TestEngineStateBuilder, test_block_info, test_engine_client, }, @@ -381,4 +386,30 @@ mod recovery_tests { l1.assert_finished(); l2.assert_finished(); } + + // A skipped task is dropped instead of being kept for a retry, so it cannot hold the queue. + #[tokio::test] + async fn skipped_task_is_dropped_from_the_queue() { + let config = Arc::new(RollupConfig::default()); + let (client, l1, l2) = test_engine_client(config.clone()); + let state = TestEngineStateBuilder::new() + .with_unsafe_head(test_block_info(20)) + .with_finalized_head(test_block_info(10)) + .build(); + let (state_tx, _state_rx) = tokio::sync::watch::channel(state); + let mut engine = Engine::new(state, state_tx); + engine.enqueue(EngineTask::Finalize(Box::new(FinalizeTask::new( + Arc::new(client), + config, + FinalizeBlockId::ByNumber(5), + )))); + + let error = engine.drain().await.unwrap_err(); + + assert_eq!(error.severity(), EngineTaskErrorSeverity::Skip, "got {error:?}"); + assert!(engine.is_empty()); + assert_eq!(engine.state().sync_state, state.sync_state); + l1.assert_finished(); + l2.assert_finished(); + } } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/error.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/error.rs index c8444c2595e..51a7f2c1dda 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/error.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/error.rs @@ -3,6 +3,7 @@ use crate::{ EngineTaskError, SynchronizeTaskError, task_queue::tasks::task::EngineTaskErrorSeverity, }; +use alloy_eips::BlockNumHash; use alloy_transport::{RpcError, TransportErrorKind}; use kona_protocol::FromBlockError; use thiserror::Error; @@ -10,33 +11,69 @@ use thiserror::Error; /// An error that occurs when running the [`crate::FinalizeTask`]. #[derive(Debug, Error)] pub enum FinalizeTaskError { - /// The block is not safe, and therefore cannot be finalized. - #[error("Attempted to finalize a block that is not yet safe")] - BlockNotSafe, + /// The target is below the finalized head, and finality never moves backward. + #[error("Finalization target {target} is behind the finalized head {finalized}")] + BehindFinalizedHead { + /// The number of the finalized head. + finalized: u64, + /// The number of the finalization target. + target: u64, + }, /// The block to finalize was not found. #[error("The block to finalize was not found: Number {0}")] BlockNotFound(u64), + /// The block is not safe, and therefore cannot be finalized. + #[error("Attempted to finalize a block that is not yet safe")] + BlockNotSafe, + /// The engine returned a block at a different height than the one requested. + #[error("Requested block {requested} to finalize, but the engine returned block {returned}")] + BlockNumberMismatch { + /// The requested block number. + requested: u64, + /// The number of the block the engine returned. + returned: u64, + }, + /// The target is a different block at the finalized head's height. + #[error("Finalization target {target:?} conflicts with the finalized head {finalized:?}")] + ConflictsWithFinalizedHead { + /// The finalized head. + finalized: BlockNumHash, + /// The finalization target. + target: BlockNumHash, + }, + /// The forkchoice update call to finalize the block failed. + #[error(transparent)] + ForkchoiceUpdateFailed(#[from] SynchronizeTaskError), /// An error occurred while transforming the RPC block into [`L2BlockInfo`]. /// /// [`L2BlockInfo`]: kona_protocol::L2BlockInfo #[error(transparent)] FromBlock(#[from] FromBlockError), + /// The pinned target is not the engine's canonical block at its height. + #[error("Finalization target {target:?} is not canonical; the engine has {canonical:?}")] + NotCanonical { + /// The engine's canonical block at the target's height. + canonical: BlockNumHash, + /// The finalization target. + target: BlockNumHash, + }, /// A temporary RPC failure. #[error(transparent)] TransportError(#[from] RpcError), - /// The forkchoice update call to finalize the block failed. - #[error(transparent)] - ForkchoiceUpdateFailed(#[from] SynchronizeTaskError), } impl EngineTaskError for FinalizeTaskError { fn severity(&self) -> EngineTaskErrorSeverity { match self { - Self::BlockNotSafe | Self::BlockNotFound(_) | Self::FromBlock(_) => { - EngineTaskErrorSeverity::Critical - } - Self::TransportError(_) => EngineTaskErrorSeverity::Temporary, + Self::BehindFinalizedHead { .. } | + Self::ConflictsWithFinalizedHead { .. } | + Self::NotCanonical { .. } => EngineTaskErrorSeverity::Skip, + Self::BlockNotFound(_) | + Self::BlockNotSafe | + Self::BlockNumberMismatch { .. } | + Self::FromBlock(_) => EngineTaskErrorSeverity::Critical, Self::ForkchoiceUpdateFailed(inner) => inner.severity(), + Self::TransportError(_) => EngineTaskErrorSeverity::Temporary, } } } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/id.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/id.rs index ab5a92e4ddb..c7c12fd6e73 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/id.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/id.rs @@ -10,9 +10,9 @@ use alloy_eips::BlockNumHash; /// authoritative source for the block at a given height (e.g. local L1-finality driven by /// `L2Finalizer`). The number is sufficient because there is only one chain to consult. /// - [`FinalizeBlockId::ByHash`] is used when an upstream supplies `(number, hash)` and the engine -/// must finalize the specific block identified by hash. If the engine does not have that hash, -/// the task must error rather than silently finalize the engine's canonical block at the same -/// height. +/// must finalize the specific block identified by hash. If the engine's canonical block at that +/// height has a different hash, the task skips the target rather than finalize the engine's block +/// in its place. /// /// EL finalization is irreversible, so finalizing the wrong block is unrecoverable. Forcing every /// caller to pick a variant prevents accidental hash-loss at the boundary. @@ -20,8 +20,8 @@ use alloy_eips::BlockNumHash; pub enum FinalizeBlockId { /// Finalize whatever block the engine has at the given L2 block number. ByNumber(u64), - /// Finalize the block matching the given `(number, hash)` pair. The task fails if the engine - /// does not have that hash. + /// Finalize the block matching the given `(number, hash)` pair, only once it is the engine's + /// canonical block at that height. ByHash(BlockNumHash), } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs index d00bb74b624..a01d43c1608 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs @@ -5,7 +5,7 @@ use crate::{ state::EngineSyncStateUpdate, task_queue::{ForkchoiceDelivery, synchronize}, }; -use alloy_eips::BlockId; +use alloy_eips::BlockNumHash; use async_trait::async_trait; use derive_more::Constructor; use kona_genesis::RollupConfig; @@ -14,6 +14,10 @@ use std::sync::Arc; /// The [`FinalizeTask`] fetches the [`L2BlockInfo`] identified by `block_id`, updates the /// [`EngineState`], and dispatches a forkchoice update to finalize the block. +/// +/// Finality only moves forward. A target behind the finalized head, a different block at the +/// finalized height, or a pinned hash that is not the engine's canonical block fails with an error +/// that skips the task and leaves the engine state unchanged. #[derive(Debug, Clone, Constructor)] pub struct FinalizeTask { /// The engine client. @@ -24,6 +28,88 @@ pub struct FinalizeTask { pub block_id: FinalizeBlockId, } +impl FinalizeTask { + /// Rejects a target that would move finality backward. + const fn ensure_not_behind_finalized_head( + finalized_head: &L2BlockInfo, + target_number: u64, + ) -> Result<(), FinalizeTaskError> { + if target_number < finalized_head.block_info.number { + return Err(FinalizeTaskError::BehindFinalizedHead { + finalized: finalized_head.block_info.number, + target: target_number, + }); + } + Ok(()) + } + + /// Rejects a different block at the finalized head's height. + /// + /// The default head means finality has not been established yet, so any target, including the + /// configured genesis, may initialize it. + fn ensure_no_conflict_with_finalized_head( + finalized_head: &L2BlockInfo, + target: BlockNumHash, + ) -> Result<(), FinalizeTaskError> { + let finalized = finalized_head.block_info.id(); + if *finalized_head != L2BlockInfo::default() && + target.number == finalized.number && + target.hash != finalized.hash + { + return Err(FinalizeTaskError::ConflictsWithFinalizedHead { finalized, target }); + } + Ok(()) + } + + /// Whether `target` is the finalized head itself; a different block at its height is an error. + fn is_already_finalized( + finalized_head: &L2BlockInfo, + target: BlockNumHash, + ) -> Result { + if target == finalized_head.block_info.id() { + debug!(target: "engine", ?target, "Target is already finalized"); + return Ok(true); + } + Self::ensure_no_conflict_with_finalized_head(finalized_head, target)?; + Ok(false) + } + + /// Fetches the engine's canonical block at the target's height. + /// + /// A pinned hash must be that block: a forkchoice update cannot finalize a block off the + /// canonical chain, and finalizing the canonical block instead would finalize a block the + /// caller never named. The block's identity is checked before it is decoded, so a mismatch is + /// reported as such rather than as a decoding failure. + async fn fetch_canonical_block_info(&self) -> Result { + let requested = self.block_id.number(); + debug!(target: "engine", block_id = ?self.block_id, "Fetching the block to finalize"); + let block = self + .client + .get_l2_block(requested.into()) + .full() + .await + .map_err(FinalizeTaskError::TransportError)? + .ok_or(FinalizeTaskError::BlockNotFound(requested))?; + let canonical = BlockNumHash { hash: block.header.hash, number: block.header.number }; + debug!(target: "engine", ?canonical, "Fetched the block to finalize"); + + if canonical.number != requested { + return Err(FinalizeTaskError::BlockNumberMismatch { + requested, + returned: canonical.number, + }); + } + if let FinalizeBlockId::ByHash(target) = self.block_id && + target.hash != canonical.hash + { + return Err(FinalizeTaskError::NotCanonical { canonical, target }); + } + + L2BlockInfo::from_block_and_genesis(&block.into_consensus(), &self.cfg.genesis) + .map_err(FinalizeTaskError::FromBlock) + } +} + #[async_trait] impl ExecutableEngineTask for FinalizeTask { type Output = (); @@ -31,40 +117,30 @@ impl ExecutableEngineTask for FinalizeTask { type Error = FinalizeTaskError; async fn execute(&self, state: &mut EngineState) -> Result<(), FinalizeTaskError> { - let block_number = self.block_id.number(); + let finalized_head = state.sync_state.finalized_head(); + let target_number = self.block_id.number(); + + // A lagging source can report an older finalized block, or one from another chain. A + // pinned hash is enough to detect either without asking the execution layer. + Self::ensure_not_behind_finalized_head(&finalized_head, target_number)?; + if let FinalizeBlockId::ByHash(target) = self.block_id && + Self::is_already_finalized(&finalized_head, target)? + { + return Ok(()); + } // Sanity check that the block that is being finalized is at least safe. - if state.sync_state.safe_head().block_info.number < block_number { + if state.sync_state.safe_head().block_info.number < target_number { return Err(FinalizeTaskError::BlockNotSafe); } - // Look up by hash when the caller pinned a specific hash (delegated polling supplies - // `(number, hash)`); otherwise look up by number. Finalization is irreversible, so a stale - // hash must produce a hard error, not silently finalize whatever the engine has at the - // same height. - let lookup: BlockId = match self.block_id { - FinalizeBlockId::ByHash(id) => id.hash.into(), - FinalizeBlockId::ByNumber(n) => n.into(), - }; - let block = self - .client - .get_l2_block(lookup) - .full() - .await - .map_err(FinalizeTaskError::TransportError)? - .ok_or(FinalizeTaskError::BlockNotFound(block_number))? - .into_consensus(); - let block_info = L2BlockInfo::from_block_and_genesis(&block, &self.client.cfg().genesis) - .map_err(FinalizeTaskError::FromBlock)?; - - // For ByHash, also assert the returned block's height matches the caller's claim. The - // engine should never serve a block at a different height for a given hash, but a height - // mismatch indicates either an upstream protocol bug or a misuse of the API and must - // never lead to silent finalization. - if let FinalizeBlockId::ByHash(id) = self.block_id && - block_info.block_info.number != id.number + let block_info = self.fetch_canonical_block_info().await?; + + // A by-number target only learns its hash from the lookup. + if matches!(self.block_id, FinalizeBlockId::ByNumber(_)) && + Self::is_already_finalized(&finalized_head, block_info.block_info.id())? { - return Err(FinalizeTaskError::BlockNotFound(id.number)); + return Ok(()); } // Dispatch a forkchoice update. diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs index 08c4f153e37..fc2243dcb6c 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs @@ -1,62 +1,249 @@ use crate::{ + EngineState, EngineSyncState, EngineSyncStateUpdate, EngineTaskError, EngineTaskErrorSeverity, ExecutableEngineTask, FinalizeBlockId, FinalizeTask, FinalizeTaskError, - test_utils::{TestEngineStateBuilder, test_engine_client}, + test_utils::{ + RpcMock, TestEngineStateBuilder, TestRpcBlock as RpcBlock, test_block_info, + test_engine_client, test_l2_block, test_rpc_block, + }, }; +use alloy_consensus::Header; use alloy_eips::BlockNumHash; -use alloy_primitives::b256; -use kona_genesis::RollupConfig; -use kona_protocol::{BlockInfo, L2BlockInfo}; +use alloy_primitives::B256; +use alloy_rpc_types_engine::PayloadStatusEnum; +use alloy_rpc_types_eth::{BlockTransactions, Header as RpcHeader}; +use kona_genesis::{ChainGenesis, RollupConfig}; +use kona_protocol::L2BlockInfo; +use rstest::rstest; use std::sync::Arc; -/// When the engine receives a `ByHash` finalize request for a block hash it doesn't have, -/// [`FinalizeTask`] must fail with [`FinalizeTaskError::BlockNotFound`] rather than silently -/// finalize whatever it happens to have at the same height. -/// -/// Assert the exact by-hash RPC lookup, so a by-number lookup cannot silently finalize a -/// different canonical block at the same height. -#[tokio::test] -async fn finalize_task_by_hash_errors_when_engine_lacks_hash() { - const N: u64 = 10; - let hash_a = b256!("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); - let hash_b = b256!("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"); +const STALE: u64 = 5; +const FINALIZED: u64 = 10; +const ADVANCED: u64 = 15; +const SAFE: u64 = 20; +const UNSAFE: u64 = 30; + +/// The L2 genesis block, which carries no L1-info deposit. +fn genesis_block() -> RpcBlock { + RpcBlock::new(RpcHeader::new(Header::default()), BlockTransactions::Full(vec![])) +} + +fn rollup_config() -> Arc { + let genesis = BlockNumHash { hash: genesis_block().header.hash, number: 0 }; + Arc::new(RollupConfig { + genesis: ChainGenesis { l2: genesis, ..Default::default() }, + ..Default::default() + }) +} + +/// An L2 block at `number`. Blocks built from different `fork` values share a height but not a +/// hash. +fn l2_block(number: u64, fork: u8) -> RpcBlock { + test_rpc_block(&test_l2_block(number, B256::ZERO, fork)) +} + +fn canonical(number: u64) -> RpcBlock { + l2_block(number, 0) +} + +fn conflicting(number: u64) -> RpcBlock { + l2_block(number, 1) +} + +fn l2_block_info(block: &RpcBlock) -> L2BlockInfo { + L2BlockInfo::from_block_and_genesis(&block.clone().into_consensus(), &rollup_config().genesis) + .unwrap() +} + +fn by_hash(block: &RpcBlock) -> FinalizeBlockId { + FinalizeBlockId::ByHash(l2_block_info(block).block_info.id()) +} - let cfg = Arc::new(RollupConfig::default()); +fn by_number(block: &RpcBlock) -> FinalizeBlockId { + FinalizeBlockId::ByNumber(block.header.number) +} + +fn established() -> L2BlockInfo { + l2_block_info(&canonical(FINALIZED)) +} + +fn uninitialized() -> L2BlockInfo { + L2BlockInfo::default() +} + +fn engine_state(finalized_head: L2BlockInfo) -> EngineState { + TestEngineStateBuilder::new() + .with_unsafe_head(test_block_info(UNSAFE)) + .with_safe_head(test_block_info(SAFE)) + .with_finalized_head(finalized_head) + .build() +} + +fn with_finalized_head(state: &EngineState, finalized_head: L2BlockInfo) -> EngineSyncState { + state.sync_state.apply_update(EngineSyncStateUpdate { + finalized_head: Some(finalized_head), + ..Default::default() + }) +} + +async fn finalize( + target: FinalizeBlockId, + state: &mut EngineState, + configure_engine: impl FnOnce(&RpcMock), +) -> Result<(), FinalizeTaskError> { + let cfg = rollup_config(); let (engine_client, l1, l2) = test_engine_client(cfg.clone()); - l2.expect_params( - "eth_getBlockByHash", - serde_json::json!([hash_b, true]), - Option::>::None, - ); - - // Place the safe head at N so the sanity check (`safe_head.number >= block_id.number`) passes - // and `execute()` reaches the lookup we care about. - let safe_head = L2BlockInfo { - block_info: BlockInfo { - number: N, - hash: hash_a, - parent_hash: Default::default(), - timestamp: N * 2, - }, - l1_origin: BlockNumHash::default(), - seq_num: 0, - }; - let mut state = - TestEngineStateBuilder::new().with_unsafe_head(safe_head).with_safe_head(safe_head).build(); - - let task = FinalizeTask::new( - Arc::new(engine_client), - cfg, - FinalizeBlockId::ByHash(BlockNumHash { number: N, hash: hash_b }), - ); - - let result = task.execute(&mut state).await; - - assert!( - matches!(result, Err(FinalizeTaskError::BlockNotFound(n)) if n == N), - "expected BlockNotFound({N}) — got {result:?}. The by-hash lookup must fail loudly when \ - the engine lacks the requested hash; instead, the task either succeeded (finalizing the \ - wrong block) or surfaced a different error." - ); + configure_engine(&l2); + + let result = FinalizeTask::new(Arc::new(engine_client), cfg, target).execute(state).await; + l1.assert_finished(); l2.assert_finished(); + result +} + +const fn is_behind(err: &FinalizeTaskError) -> bool { + matches!(err, FinalizeTaskError::BehindFinalizedHead { finalized: FINALIZED, target: STALE }) +} + +const fn is_conflict(err: &FinalizeTaskError) -> bool { + matches!(err, FinalizeTaskError::ConflictsWithFinalizedHead { .. }) +} + +const fn is_not_canonical(err: &FinalizeTaskError) -> bool { + matches!(err, FinalizeTaskError::NotCanonical { .. }) +} + +const fn is_number_mismatch(err: &FinalizeTaskError) -> bool { + matches!(err, FinalizeTaskError::BlockNumberMismatch { requested: ADVANCED, .. }) +} + +const fn is_block_not_found(err: &FinalizeTaskError) -> bool { + matches!(err, FinalizeTaskError::BlockNotFound(ADVANCED)) +} + +/// Targets behind the finalized head, or a pinned hash that conflicts with it, are skipped before +/// asking the execution layer for the block: the test engine expects no request at all. +#[rstest] +#[case::stale_by_hash(by_hash(&canonical(STALE)), is_behind)] +#[case::stale_by_number(by_number(&canonical(STALE)), is_behind)] +#[case::conflicting_by_hash(by_hash(&conflicting(FINALIZED)), is_conflict)] +#[tokio::test] +async fn finalize_task_skips_target_without_lookup( + #[case] target: FinalizeBlockId, + #[case] is_expected_error: fn(&FinalizeTaskError) -> bool, +) { + let mut state = engine_state(established()); + let original_state = state; + + let err = finalize(target, &mut state, |_| {}).await.unwrap_err(); + + assert!(is_expected_error(&err), "{err:?}"); + assert_eq!(err.severity(), EngineTaskErrorSeverity::Skip); + assert_eq!(state, original_state); +} + +/// A target whose block differs from the engine's canonical block is skipped. A pinned hash is +/// never replaced by the canonical block at its height, even above the finalized head, where a +/// delegate on another fork names a block the engine does not consider canonical. +#[rstest] +#[case::conflicting_by_number(by_number(&conflicting(FINALIZED)), conflicting(FINALIZED), is_conflict)] +#[case::other_fork_by_hash(by_hash(&conflicting(ADVANCED)), canonical(ADVANCED), is_not_canonical)] +#[tokio::test] +async fn finalize_task_skips_target_off_the_canonical_chain( + #[case] target: FinalizeBlockId, + #[case] served_block: RpcBlock, + #[case] is_expected_error: fn(&FinalizeTaskError) -> bool, +) { + let mut state = engine_state(established()); + let original_state = state; + + let err = finalize(target, &mut state, |l2| { + l2.expect_block_by_number(target.number(), Some(served_block)) + }) + .await + .unwrap_err(); + + assert!(is_expected_error(&err), "{err:?}"); + assert_eq!(err.severity(), EngineTaskErrorSeverity::Skip); + assert_eq!(state, original_state); +} + +/// A canonical target above the finalized head becomes the finalized head through one forkchoice +/// update. The canonical block at the finalized height is already finalized, so it needs no update. +#[rstest] +#[case::advancing_by_hash(established(), by_hash, canonical(ADVANCED), 1)] +#[case::advancing_by_number(established(), by_number, canonical(ADVANCED), 1)] +#[case::identical_by_number(established(), by_number, canonical(FINALIZED), 0)] +#[case::uninitialized_adopts_genesis(uninitialized(), by_hash, genesis_block(), 1)] +#[tokio::test] +async fn finalize_task_finalizes_canonical_target( + #[case] finalized_head: L2BlockInfo, + #[case] identify: fn(&RpcBlock) -> FinalizeBlockId, + #[case] served_block: RpcBlock, + #[case] forkchoice_updates: usize, +) { + let mut state = engine_state(finalized_head); + let target = identify(&served_block); + let expected_sync_state = with_finalized_head(&state, l2_block_info(&served_block)); + + finalize(target, &mut state, |l2| { + l2.expect_block_by_number(target.number(), Some(served_block)); + for _ in 0..forkchoice_updates { + l2.expect_forkchoice_updated( + expected_sync_state.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + } + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected_sync_state); +} + +/// A target pinned to the finalized head is already finalized, so it needs no lookup. +#[tokio::test] +async fn finalize_task_skips_lookup_for_finalized_head() { + let mut state = engine_state(established()); + let original_state = state; + + finalize(by_hash(&canonical(FINALIZED)), &mut state, |_| {}).await.unwrap(); + + assert_eq!(state, original_state); +} + +/// The engine serving no block, or a block at a height other than the requested one, stops the +/// node instead of finalizing anything. A misplaced block could otherwise bypass the guards against +/// moving finality backward. +#[rstest] +#[case::missing_by_number(FinalizeBlockId::ByNumber(ADVANCED), None, is_block_not_found)] +#[case::missing_by_hash(by_hash(&canonical(ADVANCED)), None, is_block_not_found)] +#[case::stale_block_by_number( + /* target */ FinalizeBlockId::ByNumber(ADVANCED), + /* served_block */ Some(canonical(STALE)), + /* is_expected_error */ is_number_mismatch +)] +#[case::higher_block_by_number( + /* target */ FinalizeBlockId::ByNumber(ADVANCED), + /* served_block */ Some(canonical(SAFE)), + /* is_expected_error */ is_number_mismatch +)] +#[case::stale_block_by_hash(by_hash(&canonical(ADVANCED)), Some(canonical(STALE)), is_number_mismatch)] +#[tokio::test] +async fn finalize_task_stops_on_unexpected_lookup_result( + #[case] target: FinalizeBlockId, + #[case] served_block: Option, + #[case] is_expected_error: fn(&FinalizeTaskError) -> bool, +) { + let mut state = engine_state(established()); + let original_state = state; + + let err = + finalize(target, &mut state, |l2| l2.expect_block_by_number(target.number(), served_block)) + .await + .unwrap_err(); + + assert!(is_expected_error(&err), "{err:?}"); + assert_eq!(err.severity(), EngineTaskErrorSeverity::Critical); + assert_eq!(state, original_state); } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/task.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/task.rs index ee976a02201..6b3a3d1b4cb 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/task.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/task.rs @@ -32,6 +32,10 @@ pub enum EngineTaskErrorSeverity { /// The error indicates that the engine should be flushed. #[display("flush")] Flush, + /// The task no longer applies to the engine state, so it is discarded instead of retried and + /// the engine keeps running. + #[display("skip")] + Skip, } /// The interface for an engine task error. diff --git a/rust/kona/crates/node/engine/src/test_utils/blocks.rs b/rust/kona/crates/node/engine/src/test_utils/blocks.rs new file mode 100644 index 00000000000..434d3bd01eb --- /dev/null +++ b/rust/kona/crates/node/engine/src/test_utils/blocks.rs @@ -0,0 +1,40 @@ +//! L2 blocks, and their RPC form as the execution layer serves them. + +use super::l1_info_deposit; +use alloy_consensus::transaction::Recovered; +use alloy_primitives::{Address, B256, Bytes}; +use alloy_rpc_types_eth::{BlockTransactions, Header as RpcHeader}; +use op_alloy_consensus::OpBlock; + +/// A full L2 block as the execution layer serves it over RPC. +pub type TestRpcBlock = alloy_rpc_types_eth::Block; + +/// An L2 block at `number` on `parent_hash` whose only transaction is the L1-info deposit, so it +/// converts into an `L2BlockInfo` without being the configured genesis. Blocks with different +/// `extra_data` share a height but not a hash. +pub fn test_l2_block(number: u64, parent_hash: B256, extra_data: u8) -> OpBlock { + let mut block = OpBlock::default(); + block.header.extra_data = Bytes::from(vec![extra_data]); + block.header.number = number; + block.header.parent_hash = parent_hash; + block.header.timestamp = number * 2; + block.body.transactions.push(l1_info_deposit()); + block +} + +/// `block` as the execution layer serves it over RPC. +pub fn test_rpc_block(block: &OpBlock) -> TestRpcBlock { + let transactions = block + .body + .transactions + .iter() + .cloned() + .map(|transaction| { + op_alloy_rpc_types::Transaction::from_transaction( + Recovered::new_unchecked(transaction, Address::ZERO), + Default::default(), + ) + }) + .collect(); + TestRpcBlock::new(RpcHeader::new(block.header.clone()), BlockTransactions::Full(transactions)) +} diff --git a/rust/kona/crates/node/engine/src/test_utils/mod.rs b/rust/kona/crates/node/engine/src/test_utils/mod.rs index e5949f23225..0a60403a0d6 100644 --- a/rust/kona/crates/node/engine/src/test_utils/mod.rs +++ b/rust/kona/crates/node/engine/src/test_utils/mod.rs @@ -1,6 +1,9 @@ mod attributes; pub use attributes::TestAttributesBuilder; +mod blocks; +pub use blocks::{TestRpcBlock, test_l2_block, test_rpc_block}; + mod deposits; pub use deposits::l1_info_deposit; diff --git a/rust/kona/crates/node/engine/src/test_utils/rpc.rs b/rust/kona/crates/node/engine/src/test_utils/rpc.rs index 1cfc3995356..39f23f14de2 100644 --- a/rust/kona/crates/node/engine/src/test_utils/rpc.rs +++ b/rust/kona/crates/node/engine/src/test_utils/rpc.rs @@ -1,6 +1,6 @@ //! Scripted RPC expectations over Alloy's mock transport. -use crate::EngineClient; +use crate::{EngineClient, test_utils::TestRpcBlock}; use alloy_json_rpc::{ErrorPayload, RequestPacket, ResponsePacket}; use alloy_network::Network; use alloy_provider::RootProvider; @@ -59,6 +59,16 @@ impl RpcMock { ); } + /// Expects a full-block `eth_getBlockByNumber` call for `number` and answers it with `block`, + /// or with null for `None`. + pub fn expect_block_by_number(&self, number: u64, block: Option) { + self.expect_params( + "eth_getBlockByNumber", + serde_json::json!([format!("{number:#x}"), true]), + block, + ); + } + /// Expects an `engine_forkchoiceUpdatedV3` call for `forkchoice` without payload attributes and /// answers it with `status`. pub fn expect_forkchoice_updated( diff --git a/rust/kona/crates/node/service/src/actors/derivation/actor/core.rs b/rust/kona/crates/node/service/src/actors/derivation/actor/core.rs index e3a5ea1ba4d..830b7ce4eaf 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/actor/core.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/actor/core.rs @@ -369,8 +369,9 @@ where self.attempt_derivation().await?; } - DerivationActorRequest::ProcessEngineSyncCompletionRequest(safe_head) => { + DerivationActorRequest::ProcessEngineSyncCompletionRequest(sync_state) => { info!(target: "derivation", "Engine finished syncing, starting derivation."); + let safe_head = Box::new(sync_state.local_safe_head()); self.derivation_state_machine .update(&DerivationStateUpdate::ELSyncCompleted(safe_head.clone()))?; self.truncate_safe_db(*safe_head, "EL sync completion")?; diff --git a/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs b/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs index 5bd4f9cfc00..01eb187e119 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs @@ -10,7 +10,7 @@ use kona_derive::{ ActivationSignal, Pipeline, PipelineError, PipelineErrorKind, PipelineResult, ResetError, Signal, SignalReceiver, StepResult, }; -use kona_engine::FinalizeBlockId; +use kona_engine::{EngineSyncState, EngineSyncStateUpdate, FinalizeBlockId}; use kona_genesis::{HardForkConfig, RollupConfig, SystemConfig}; use kona_protocol::{BlockInfo, L2BlockInfo, OpAttributesWithParent}; use kona_safedb::{SafeDatabase, SafeDb, SafeDbError, SafeHeadRecord}; @@ -41,12 +41,7 @@ async fn reset_adopts_safe_head_and_resumes_atomically(#[case] previous_head: u6 l1_block(0).id(), watch::channel(None).0, ); - request_tx - .send(DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(l2_block( - previous_head, - )))) - .await - .unwrap(); + request_tx.send(engine_sync_completed_at(l2_block(previous_head))).await.unwrap(); actor.step().await.unwrap(); request_tx @@ -267,10 +262,7 @@ async fn test_origin_is_published(#[case] step_result: StepResult) { actor.step().await.unwrap(); assert_eq!(derivation_origin_rx.borrow().map(|origin| origin.id()), Some(safe_head.l1_origin)); - request_tx - .send(DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(safe_head))) - .await - .unwrap(); + request_tx.send(engine_sync_completed_at(safe_head)).await.unwrap(); actor.step().await.unwrap(); assert_eq!(derivation_origin_rx.borrow().unwrap().number, 11); @@ -465,6 +457,16 @@ fn l1_block(number: u64) -> BlockInfo { BlockInfo { hash: B256::with_last_byte(number as u8), number, ..Default::default() } } +/// The engine reporting that its sync completed with `safe_head` as its safe heads. +fn engine_sync_completed_at(safe_head: L2BlockInfo) -> DerivationActorRequest { + let sync_state = EngineSyncState::default().apply_update(EngineSyncStateUpdate { + local_safe_head: Some(safe_head), + safe_head: Some(safe_head), + ..Default::default() + }); + DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(sync_state)) +} + fn l2_block(number: u64) -> L2BlockInfo { L2BlockInfo { block_info: BlockInfo { @@ -535,10 +537,7 @@ impl RecordingActor { } async fn complete_el_sync_at(&mut self, head: u64) { - self.handle(DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new( - l2_block(head), - ))) - .await; + self.handle(engine_sync_completed_at(l2_block(head))).await; } async fn confirm_safe_head(&mut self, head: u64) { diff --git a/rust/kona/crates/node/service/src/actors/derivation/actor/tests/database.rs b/rust/kona/crates/node/service/src/actors/derivation/actor/tests/database.rs index 699bc1e6933..eeee778a78a 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/actor/tests/database.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/actor/tests/database.rs @@ -135,7 +135,7 @@ fn engine_reset_request() -> DerivationActorRequest { } fn sync_completion_request() -> DerivationActorRequest { - DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(l2_block(RESUME_HEAD))) + engine_sync_completed_at(l2_block(RESUME_HEAD)) } #[rstest] diff --git a/rust/kona/crates/node/service/src/actors/derivation/actor/tests/op_node_parity.rs b/rust/kona/crates/node/service/src/actors/derivation/actor/tests/op_node_parity.rs index 104544f2522..0df6cef81c1 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/actor/tests/op_node_parity.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/actor/tests/op_node_parity.rs @@ -16,7 +16,7 @@ fn l1_head_update(number: u64) -> DerivationActorRequest { } fn el_sync_completed(head: u64) -> DerivationActorRequest { - DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(l2_block(head))) + engine_sync_completed_at(l2_block(head)) } /// A reset whose pipeline signal fails once with a temporary provider error neither stops the diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs index 44fe763d969..7001d2f71d6 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs @@ -2,10 +2,11 @@ use crate::{ DerivationActorRequest, DerivationEngineClient, NodeActor, actors::derivation::{DerivationDelegateProvider, DerivationError}, }; +use alloy_eips::BlockNumHash; use alloy_primitives::BlockHash; use async_trait::async_trait; use kona_derive::{ChainProvider, Signal}; -use kona_engine::FinalizeBlockId; +use kona_engine::{EngineState, FinalizeBlockId}; use kona_protocol::{BlockInfo, L2BlockInfo, SyncStatus}; use thiserror::Error; use tokio::{ @@ -42,6 +43,10 @@ where /// Publishes the delegate's `current_l1`: the L1 block that its derivation has reached. derivation_origin_tx: watch::Sender>, + /// The engine's state as it publishes it. Delegate statuses are checked against the + /// finalized head the engine applied, not the last one forwarded to it, which the engine may + /// have skipped. + engine_state_rx: watch::Receiver, /// The engine's L2 safe head, according to updates from the Engine. engine_l2_safe_head: L2BlockInfo, /// Whether the engine sync has completed. This will only ever go from false -> true. @@ -64,6 +69,7 @@ where derivation_delegate_provider: DelegateProvider, l1_provider: L1Provider, derivation_origin_tx: watch::Sender>, + engine_state_rx: watch::Receiver, ) -> Self { let mut delegated_derivation_ticker = time::interval(Self::DERIVATION_DELEGATE_POLL_INTERVAL); @@ -74,6 +80,7 @@ where derivation_delegate_provider, l1_provider, derivation_origin_tx, + engine_state_rx, engine_l2_safe_head: L2BlockInfo::default(), has_engine_sync_completed: false, delegated_derivation_ticker, @@ -149,9 +156,44 @@ where Ok(()) } - /// Verifies that the L1 info reported by the derivation delegate - /// are consistent with canonical L1 chain. + /// The id of an established finalized head. The default head means finality has not been + /// established yet, so any finalized block from the delegate may initialize it. + fn established_finalized_l2(finalized_head: L2BlockInfo) -> Option { + (finalized_head != L2BlockInfo::default()).then(|| finalized_head.block_info.id()) + } + + /// Rejects a finalized L2 block behind, or at the same height as but different from, the + /// engine's. Either means the delegate lags or follows another chain, so none of its status + /// applies: its safe head would move the engine back or onto that chain too. + fn validate_finalized_l2_progress( + &self, + reported: BlockNumHash, + ) -> Result<(), DerivationDelegationError> { + let engine_finalized_head = self.engine_state_rx.borrow().sync_state.finalized_head(); + let Some(engine) = Self::established_finalized_l2(engine_finalized_head) else { + return Ok(()); + }; + if reported.number < engine.number { + return Err(DerivationDelegationError::FinalizedL2Regressed { engine, reported }); + } + if reported.number == engine.number && reported.hash != engine.hash { + return Err(DerivationDelegationError::FinalizedL2Conflict { engine, reported }); + } + Ok(()) + } + + /// Verifies that the finalized L2 block reported by the derivation delegate does not move back + /// from the engine's, and that its L1 info is consistent with the canonical L1 chain. async fn validate_sync_status(&mut self, v: &SyncStatus) -> bool { + if let Err(err) = self.validate_finalized_l2_progress(v.finalized_l2.block_info.id()) { + warn!( + target: "derivation", + error = %err, + "Finalized L2 inconsistency detected at sync status from delegate" + ); + return false; + } + let checks = [ ("L1 Origin of Safe L2", v.safe_l2.l1_origin.number, v.safe_l2.l1_origin.hash), ( @@ -227,9 +269,9 @@ where debug!(target: "derivation", safe_head = ?*safe_head, "Received safe head from engine."); self.engine_l2_safe_head = *safe_head; } - DerivationActorRequest::ProcessEngineSyncCompletionRequest(safe_head) => { + DerivationActorRequest::ProcessEngineSyncCompletionRequest(sync_state) => { info!(target: "derivation", "Engine finished syncing, starting derivation."); - self.engine_l2_safe_head = *safe_head; + self.engine_l2_safe_head = sync_state.local_safe_head(); self.has_engine_sync_completed = true; } DerivationActorRequest::ProcessEngineSignalRequest(signal) => match *signal { @@ -249,6 +291,14 @@ where #[derive(Error, Debug)] enum DerivationDelegationError { + /// The delegate's finalized L2 block is a different block at the engine's finalized height. + #[error("Finalized L2 {reported:?} conflicts with the engine's finalized L2 {engine:?}")] + FinalizedL2Conflict { engine: BlockNumHash, reported: BlockNumHash }, + + /// The delegate's finalized L2 block is behind the engine's. + #[error("Finalized L2 {reported:?} is behind the engine's finalized L2 {engine:?}")] + FinalizedL2Regressed { engine: BlockNumHash, reported: BlockNumHash }, + /// The L1 provider returned an error (network, RPC, etc.) #[error("L1 provider error: {0}")] L1Provider(String), diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/mod.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/mod.rs index e5460eceb11..5431e58aa93 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/mod.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/mod.rs @@ -5,3 +5,6 @@ mod client; pub use client::{ DerivationDelegateClient, DerivationDelegateClientError, DerivationDelegateProvider, }; + +#[cfg(test)] +mod tests; diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs new file mode 100644 index 00000000000..8f976aa256c --- /dev/null +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs @@ -0,0 +1,250 @@ +use super::{DelegateDerivationActor, DerivationDelegateClientError, DerivationDelegateProvider}; +use crate::{DerivationActorRequest, EngineActorRequest, NodeActor, QueuedDerivationEngineClient}; +use alloy_eips::BlockNumHash; +use alloy_primitives::B256; +use async_trait::async_trait; +use kona_derive::test_utils::TestChainProvider; +use kona_engine::{ + ConsolidateInput, EngineState, EngineSyncState, EngineSyncStateUpdate, FinalizeBlockId, +}; +use kona_protocol::{BlockInfo, L2BlockInfo, SyncStatus}; +use rstest::rstest; +use std::{collections::VecDeque, sync::Mutex}; +use tokio::sync::{mpsc, watch}; + +const SAFE_ORIGIN: u64 = 1; +const FINALIZED_ORIGIN: u64 = 2; +const CURRENT_L1: u64 = 3; + +type SyncStatusReply = Result; + +/// Answers each sync status poll with the next scripted reply. +#[derive(Debug)] +struct ScriptedDelegate(Mutex>); + +#[async_trait] +impl DerivationDelegateProvider for ScriptedDelegate { + async fn fetch_sync_status(&self) -> SyncStatusReply { + self.0.lock().unwrap().pop_front().expect("unexpected sync status poll") + } +} + +/// A delegate actor over a canonical L1 chain, with the engine requests it sends and the +/// `current_l1` it publishes. +struct Harness { + actor: + DelegateDerivationActor, + current_l1: watch::Receiver>, + derivation_requests: mpsc::Sender, + engine_requests: mpsc::Receiver, + engine_state: watch::Sender, +} + +impl Harness { + fn new(replies: impl IntoIterator) -> Self { + let mut l1 = TestChainProvider::default(); + for number in [SAFE_ORIGIN, FINALIZED_ORIGIN, CURRENT_L1] { + l1.insert_block(number, canonical_l1_block(number)); + } + let (derivation_requests, derivation_request_rx) = mpsc::channel(8); + let (engine_request_tx, engine_requests) = mpsc::channel(8); + let (current_l1_tx, current_l1) = watch::channel(None); + let (engine_state, engine_state_rx) = watch::channel(EngineState::default()); + let actor = DelegateDerivationActor::new( + QueuedDerivationEngineClient::new(engine_request_tx), + derivation_request_rx, + ScriptedDelegate(Mutex::new(replies.into_iter().collect())), + l1, + current_l1_tx, + engine_state_rx, + ); + Self { actor, current_l1, derivation_requests, engine_requests, engine_state } + } + + async fn complete_engine_sync(&mut self) { + self.complete_engine_sync_with(EngineSyncState::default()).await; + } + + /// Completes the engine sync with `sync_state` as the engine's labels, which the engine + /// publishes before it reports the completion. + async fn complete_engine_sync_with(&mut self, sync_state: EngineSyncState) { + self.publish_engine_sync_state(sync_state); + self.derivation_requests + .send(DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(sync_state))) + .await + .unwrap(); + self.actor.step().await.unwrap(); + } + + /// Publishes the engine's labels as the engine does after each task it runs. + fn publish_engine_sync_state(&self, sync_state: EngineSyncState) { + self.engine_state.send_modify(|state| state.sync_state = sync_state); + } + + /// The engine requests sent since the last call, in order. + fn forwarded(&mut self) -> Vec { + std::iter::from_fn(|| self.engine_requests.try_recv().ok()).map(Forwarded::from).collect() + } +} + +/// The parts of an engine request that the delegate actor decides. +#[derive(Debug, PartialEq, Eq)] +enum Forwarded { + Finalized(FinalizeBlockId), + Other(String), + Safe(L2BlockInfo), +} + +impl From for Forwarded { + fn from(request: EngineActorRequest) -> Self { + match request { + EngineActorRequest::ProcessFinalizedL2Block(finalized) => Self::Finalized(*finalized), + EngineActorRequest::ProcessSafeL2Signal(ConsolidateInput::BlockInfo(safe)) => { + Self::Safe(safe) + } + other => Self::Other(format!("{other:?}")), + } + } +} + +/// What one poll of `status` forwards: the safe head, then the finalized block pinned by hash. +fn forwarded_for(status: &SyncStatus) -> Vec { + vec![ + Forwarded::Safe(status.safe_l2), + Forwarded::Finalized(FinalizeBlockId::ByHash(status.finalized_l2.block_info.id())), + ] +} + +/// What one poll forwards when the status is accepted, and nothing when it is rejected. +fn forwarded_if(accepted: bool, status: &SyncStatus) -> Vec { + if accepted { forwarded_for(status) } else { Vec::new() } +} + +fn canonical_l1_block(number: u64) -> BlockInfo { + BlockInfo { hash: B256::with_last_byte(number as u8), number, ..Default::default() } +} + +fn l2_block(number: u64, l1_origin: u64) -> L2BlockInfo { + L2BlockInfo { + block_info: BlockInfo { + hash: B256::with_last_byte(0x80 | number as u8), + number, + ..Default::default() + }, + l1_origin: canonical_l1_block(l1_origin).id(), + seq_num: 0, + } +} + +/// A sync status whose safe origin, finalized origin and `current_l1` are all canonical. +fn delegate_status() -> SyncStatus { + SyncStatus { + current_l1: canonical_l1_block(CURRENT_L1), + current_l1_finalized: BlockInfo::default(), + finalized_l1: BlockInfo::default(), + finalized_l2: l2_block(15, FINALIZED_ORIGIN), + head_l1: BlockInfo::default(), + local_safe_l2: l2_block(20, SAFE_ORIGIN), + safe_l1: BlockInfo::default(), + safe_l2: l2_block(20, SAFE_ORIGIN), + unsafe_l2: l2_block(25, SAFE_ORIGIN), + } +} + +fn non_canonical(id: BlockNumHash) -> BlockNumHash { + BlockNumHash { hash: B256::repeat_byte(0xee), ..id } +} + +/// Returns `status` with its finalized L2 block replaced by `finalized_l2`. +fn with_finalized_l2(finalized_l2: L2BlockInfo) -> SyncStatus { + SyncStatus { finalized_l2, ..delegate_status() } +} + +/// An engine sync state whose finalized head is `finalized_l2`. +fn engine_finalized_at(finalized_l2: L2BlockInfo) -> EngineSyncState { + EngineSyncState::default().apply_update(EngineSyncStateUpdate { + finalized_head: Some(finalized_l2), + ..Default::default() + }) +} + +fn conflicting(block: L2BlockInfo) -> L2BlockInfo { + L2BlockInfo { + block_info: BlockInfo { + hash: non_canonical(block.block_info.id()).hash, + ..block.block_info + }, + ..block + } +} + +/// The engine's finalized head when its sync completes is the baseline, so a delegate that starts +/// behind it, or on another chain at its height, is rejected whole from the first poll on. +#[rstest] +#[case::advancing(l2_block(16, FINALIZED_ORIGIN), true)] +#[case::unchanged(l2_block(15, FINALIZED_ORIGIN), true)] +#[case::regressed(l2_block(14, FINALIZED_ORIGIN), false)] +#[case::conflicting(conflicting(l2_block(15, FINALIZED_ORIGIN)), false)] +#[tokio::test(start_paused = true)] +async fn first_poll_rejects_status_behind_engine_finality( + #[case] finalized_l2: L2BlockInfo, + #[case] accepted: bool, +) { + let status = with_finalized_l2(finalized_l2); + let mut harness = Harness::new([Ok(status.clone())]); + harness.complete_engine_sync_with(engine_finalized_at(l2_block(15, FINALIZED_ORIGIN))).await; + + harness.actor.step().await.unwrap(); + + assert_eq!(harness.forwarded(), forwarded_if(accepted, &status)); + assert_eq!(harness.current_l1.borrow().is_some(), accepted); +} + +/// Once the engine applies a forwarded finalized block, a status whose finalized block is behind +/// it, or is a different block at its height, is rejected whole: a lagging or diverged delegate's +/// safe head must not reach the engine either, and its `current_l1` is not published. +#[rstest] +#[case::advancing(l2_block(16, FINALIZED_ORIGIN), true)] +#[case::unchanged(l2_block(15, FINALIZED_ORIGIN), true)] +#[case::regressed(l2_block(14, FINALIZED_ORIGIN), false)] +#[case::conflicting(conflicting(l2_block(15, FINALIZED_ORIGIN)), false)] +#[tokio::test(start_paused = true)] +async fn poll_rejects_status_whose_finality_moves_back( + #[case] next_finalized_l2: L2BlockInfo, + #[case] accepted: bool, +) { + let next_status = with_finalized_l2(next_finalized_l2); + let mut harness = Harness::new([Ok(delegate_status()), Ok(next_status.clone())]); + harness.complete_engine_sync().await; + harness.actor.step().await.unwrap(); + assert_eq!(harness.forwarded(), forwarded_for(&delegate_status())); + harness.publish_engine_sync_state(engine_finalized_at(delegate_status().finalized_l2)); + harness.current_l1.borrow_and_update(); + + harness.actor.step().await.unwrap(); + + assert_eq!(harness.forwarded(), forwarded_if(accepted, &next_status)); + assert_eq!(harness.current_l1.has_changed().unwrap(), accepted); +} + +/// A finalized block the engine skipped, such as one on another chain, does not become the +/// baseline: the canonical block at its height, or below it but above the engine's finalized +/// head, is still accepted, so the delegate's safe head keeps reaching the engine. +#[rstest] +#[case::canonical_at_skipped_height(l2_block(17, FINALIZED_ORIGIN))] +#[case::canonical_below_skipped_height(l2_block(16, FINALIZED_ORIGIN))] +#[tokio::test(start_paused = true)] +async fn poll_accepts_status_after_engine_skipped_forwarded_finality( + #[case] canonical_finalized_l2: L2BlockInfo, +) { + let skipped_status = with_finalized_l2(conflicting(l2_block(17, FINALIZED_ORIGIN))); + let canonical_status = with_finalized_l2(canonical_finalized_l2); + let mut harness = Harness::new([Ok(skipped_status.clone()), Ok(canonical_status.clone())]); + harness.complete_engine_sync_with(engine_finalized_at(l2_block(15, FINALIZED_ORIGIN))).await; + harness.actor.step().await.unwrap(); + assert_eq!(harness.forwarded(), forwarded_for(&skipped_status)); + + harness.actor.step().await.unwrap(); + + assert_eq!(harness.forwarded(), forwarded_for(&canonical_status)); +} diff --git a/rust/kona/crates/node/service/src/actors/derivation/request.rs b/rust/kona/crates/node/service/src/actors/derivation/request.rs index 9482eef5582..caa1e2b9ad2 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/request.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/request.rs @@ -1,4 +1,5 @@ use kona_derive::Signal; +use kona_engine::EngineSyncState; use kona_protocol::{BlockInfo, L2BlockInfo}; use thiserror::Error; @@ -21,9 +22,9 @@ pub enum DerivationClientError { /// Inbound requests that the [`crate::DerivationActor`] can process. #[derive(Debug)] pub enum DerivationActorRequest { - /// Request to process the fact that Engine sync has completed, along with the current safe - /// head. - ProcessEngineSyncCompletionRequest(Box), + /// Request to process the fact that Engine sync has completed, along with the engine's sync + /// state at that point. + ProcessEngineSyncCompletionRequest(Box), /// Request to process the provided L2 engine safe head update. ProcessEngineSafeHeadUpdateRequest(Box), /// A request containing a [`Signal`] to the derivation pipeline. diff --git a/rust/kona/crates/node/service/src/actors/engine/actor.rs b/rust/kona/crates/node/service/src/actors/engine/actor.rs index c9affd16523..87619ba2756 100644 --- a/rust/kona/crates/node/service/src/actors/engine/actor.rs +++ b/rust/kona/crates/node/service/src/actors/engine/actor.rs @@ -5,8 +5,8 @@ use async_trait::async_trait; use kona_derive::{ResetSignal, Signal}; use kona_engine::{ BuildTask, CanonicalizeTask, ConsolidateTask, Engine, EngineActorRequest, EngineClient, - EngineTask, EngineTaskError, EngineTaskErrorSeverity, FinalizeTask, ImportedBlockSink, - InsertTask, SealTask, + EngineTask, EngineTaskError, EngineTaskErrorSeverity, EngineTaskErrors, FinalizeTask, + ImportedBlockSink, InsertTask, SealTask, }; use kona_genesis::RollupConfig; use kona_protocol::L2BlockInfo; @@ -34,6 +34,9 @@ where el_sync_complete: bool, /// The last local-safe head sent to the derivation actor. last_engine_safe_head_sent: L2BlockInfo, + /// The error of the last skipped engine task, so an identical repeat logs no new warning + /// until a reset changes the state the task was judged against. + last_skipped_task_error: Option, /// A channel to use to relay the current unsafe head. /// ## Note /// This is `Some` when the node is in sequencer mode, and `None` when the node is in validator @@ -81,6 +84,7 @@ where el_sync_complete: false, engine, last_engine_safe_head_sent: L2BlockInfo::default(), + last_skipped_task_error: None, rollup: config, unsafe_head_tx, inbound_request_rx, @@ -92,6 +96,18 @@ where } } + /// Logs a skipped task at `warn!` when its error differs from the last skip, and at `debug!` + /// while the identical error repeats, so a target resubmitted on every poll warns once. + fn log_skipped_task(&mut self, err: &EngineTaskErrors) { + let message = err.to_string(); + if self.last_skipped_task_error.as_ref() == Some(&message) { + debug!(target: "engine", %err, "Skipped the same engine task again"); + return; + } + warn!(target: "engine", %err, "Skipped an engine task that no longer applies"); + self.last_skipped_task_error = Some(message); + } + fn defer_retry(&mut self) { self.retry_at = Instant::now() + self.retry_delay; self.retry_delay = (self.retry_delay * 2).min(Duration::from_secs(5)); @@ -100,6 +116,7 @@ where fn begin_reset(&mut self) { if !self.reset_pending { self.engine.clear(); + self.last_skipped_task_error = None; self.reset_pending = true; self.retry_at = Instant::now(); } @@ -210,6 +227,7 @@ where self.defer_retry(); return Ok(()); } + EngineTaskErrorSeverity::Skip => self.log_skipped_task(&err), } } } @@ -233,7 +251,7 @@ where self.el_sync_complete = true; self.derivation_client - .notify_sync_completed(self.engine.state().sync_state.local_safe_head()) + .notify_sync_completed(self.engine.state().sync_state) .await .map(|_| Ok(())) .map_err(|e| { diff --git a/rust/kona/crates/node/service/src/actors/engine/client.rs b/rust/kona/crates/node/service/src/actors/engine/client.rs index 25322a5a7f2..518e7b957d0 100644 --- a/rust/kona/crates/node/service/src/actors/engine/client.rs +++ b/rust/kona/crates/node/service/src/actors/engine/client.rs @@ -2,6 +2,7 @@ use crate::{DerivationActorRequest, DerivationClientError, DerivationClientResul use async_trait::async_trait; use derive_more::Constructor; use kona_derive::Signal; +use kona_engine::EngineSyncState; use kona_protocol::L2BlockInfo; use std::fmt::Debug; use tokio::sync::mpsc; @@ -10,9 +11,13 @@ use tokio::sync::mpsc; #[cfg_attr(test, mockall::automock)] #[async_trait] pub trait EngineDerivationClient: Debug + Send + Sync { - /// Notifies the [`crate::DerivationActor`] that engine syncing has completed. + /// Notifies the [`crate::DerivationActor`] that engine syncing has completed, with the engine's + /// sync state at that point. /// Note: Does not wait for the derivation client to process this message. - async fn notify_sync_completed(&self, safe_head: L2BlockInfo) -> DerivationClientResult<()>; + async fn notify_sync_completed( + &self, + sync_state: EngineSyncState, + ) -> DerivationClientResult<()>; /// Sends the new engine `safe_head` to the [`crate::DerivationActor`]. /// Note: Does not wait for the derivation client to process this message. @@ -33,11 +38,14 @@ pub struct QueuedEngineDerivationClient { #[async_trait] impl EngineDerivationClient for QueuedEngineDerivationClient { - async fn notify_sync_completed(&self, safe_head: L2BlockInfo) -> DerivationClientResult<()> { + async fn notify_sync_completed( + &self, + sync_state: EngineSyncState, + ) -> DerivationClientResult<()> { info!(target: "engine", "Sending sync completed to derivation actor"); self.derivation_actor_request_tx - .send(DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(safe_head))) + .send(DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(sync_state))) .await .map_err(|_| { DerivationClientError::RequestError("request channel closed.".to_string()) diff --git a/rust/kona/crates/node/service/src/actors/engine/tests.rs b/rust/kona/crates/node/service/src/actors/engine/tests.rs index b61712d95b5..d000710cbf1 100644 --- a/rust/kona/crates/node/service/src/actors/engine/tests.rs +++ b/rust/kona/crates/node/service/src/actors/engine/tests.rs @@ -312,5 +312,5 @@ async fn derivation_is_handed_the_local_safe_head() { else { panic!("expected the sync completion second"); }; - assert_eq!(*sent, local_safe_head); + assert_eq!(sent.local_safe_head(), local_safe_head); } diff --git a/rust/kona/crates/node/service/src/service/node/rollup.rs b/rust/kona/crates/node/service/src/service/node/rollup.rs index d1b5fb0a12e..ed509b22101 100644 --- a/rust/kona/crates/node/service/src/service/node/rollup.rs +++ b/rust/kona/crates/node/service/src/service/node/rollup.rs @@ -289,6 +289,7 @@ impl RollupNode { engine_actor_request_tx: mpsc::Sender, derivation_actor_request_rx: mpsc::Receiver, derivation_origin_tx: watch::Sender>, + engine_state_rx: watch::Receiver, l1_head: watch::Receiver>, ) -> Result { if let Some(provider) = self.derivation_delegate_provider.clone() { @@ -303,6 +304,7 @@ impl RollupNode { provider, l1_provider, derivation_origin_tx, + engine_state_rx, )))) } else { let pipeline = self @@ -621,6 +623,7 @@ impl RollupNode { engine_actor_request_tx.clone(), derivation_actor_request_rx, derivation_origin_tx, + engine_state_rx.clone(), l1_head_updates_rx.clone(), ) .await?; From e9305c437df6547fbe172cb14d7acafb0082affd Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 12 Nov 2026 14:38:20 +0100 Subject: [PATCH 2/5] kona-engine: inline the finalize task's finality checks `ensure_not_behind_finalized_head` and `ensure_no_conflict_with_finalized_head` each had one caller, so their checks move inline. A new test pins a target above the safe head: it fails with `BlockNotSafe` before any lookup. Co-Authored-By: Claude Opus 5.5 --- .../src/task_queue/tasks/finalize/task.rs | 48 +++++-------------- .../task_queue/tasks/finalize/task_test.rs | 14 ++++++ 2 files changed, 27 insertions(+), 35 deletions(-) diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs index 2099ccd7403..dac3404750a 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task.rs @@ -28,48 +28,21 @@ pub struct FinalizeTask { } impl FinalizeTask { - /// Rejects a target that would move finality backward. - const fn ensure_not_behind_finalized_head( - finalized_head: &L2BlockInfo, - target_number: u64, - ) -> Result<(), FinalizeTaskError> { - if target_number < finalized_head.block_info.number { - return Err(FinalizeTaskError::BehindFinalizedHead { - finalized: finalized_head.block_info.number, - target: target_number, - }); - } - Ok(()) - } - - /// Rejects a different block at the finalized head's height. - /// - /// The default head means finality has not been established yet, so any target, including the - /// configured genesis, may initialize it. - fn ensure_no_conflict_with_finalized_head( - finalized_head: &L2BlockInfo, - target: BlockNumHash, - ) -> Result<(), FinalizeTaskError> { - let finalized = finalized_head.block_info.id(); - if *finalized_head != L2BlockInfo::default() && - target.number == finalized.number && - target.hash != finalized.hash - { - return Err(FinalizeTaskError::ConflictsWithFinalizedHead { finalized, target }); - } - Ok(()) - } - /// Whether `target` is the finalized head itself; a different block at its height is an error. fn is_already_finalized( finalized_head: &L2BlockInfo, target: BlockNumHash, ) -> Result { - if target == finalized_head.block_info.id() { + let finalized = finalized_head.block_info.id(); + if target == finalized { debug!(target: "engine", ?target, "Target is already finalized"); return Ok(true); } - Self::ensure_no_conflict_with_finalized_head(finalized_head, target)?; + // The default head means finality has not been established yet, so any target, including + // the configured genesis, may initialize it. + if *finalized_head != L2BlockInfo::default() && target.number == finalized.number { + return Err(FinalizeTaskError::ConflictsWithFinalizedHead { finalized, target }); + } Ok(false) } @@ -121,7 +94,12 @@ impl ExecutableEngineTask for FinalizeTask { // A lagging source can report an older finalized block, or one from another chain. A // pinned hash is enough to detect either without asking the execution layer. - Self::ensure_not_behind_finalized_head(&finalized_head, target_number)?; + if target_number < finalized_head.block_info.number { + return Err(FinalizeTaskError::BehindFinalizedHead { + finalized: finalized_head.block_info.number, + target: target_number, + }); + } if let FinalizeBlockId::ByHash(target) = self.block_id && Self::is_already_finalized(&finalized_head, target)? { diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs index fc2243dcb6c..fb273f99831 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/finalize/task_test.rs @@ -142,6 +142,20 @@ async fn finalize_task_skips_target_without_lookup( assert_eq!(state, original_state); } +/// A target above the safe head is refused before asking the execution layer for the block: the +/// test engine expects no request at all. +#[tokio::test] +async fn finalize_task_refuses_target_above_safe_head_without_lookup() { + let mut state = engine_state(established()); + let original_state = state; + + let err = finalize(by_hash(&canonical(SAFE + 5)), &mut state, |_| {}).await.unwrap_err(); + + assert!(matches!(err, FinalizeTaskError::BlockNotSafe), "{err:?}"); + assert_eq!(err.severity(), EngineTaskErrorSeverity::Critical); + assert_eq!(state, original_state); +} + /// A target whose block differs from the engine's canonical block is skipped. A pinned hash is /// never replaced by the canonical block at its height, even above the finalized head, where a /// delegate on another fork names a block the engine does not consider canonical. From d92d1fcb5e390b99f4b4456541af403af0bb6a5b Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 12 Nov 2026 14:38:21 +0100 Subject: [PATCH 3/5] kona-node-service: report sync completion with the safe head only `ProcessEngineSyncCompletionRequest` and `notify_sync_completed` carry the local-safe head again instead of the whole `EngineSyncState`. Both consumers only read the local-safe head; the delegate actor takes its finalized baseline from the engine's published state. Co-Authored-By: Claude Opus 5.5 --- .../src/actors/derivation/actor/core.rs | 3 +- .../src/actors/derivation/actor/tests.rs | 29 ++++++++++--------- .../actors/derivation/actor/tests/database.rs | 2 +- .../derivation/actor/tests/op_node_parity.rs | 2 +- .../src/actors/derivation/delegated/actor.rs | 4 +-- .../src/actors/derivation/delegated/tests.rs | 4 ++- .../service/src/actors/derivation/request.rs | 7 ++--- .../node/service/src/actors/engine/actor.rs | 2 +- .../node/service/src/actors/engine/client.rs | 16 +++------- .../node/service/src/actors/engine/tests.rs | 2 +- 10 files changed, 32 insertions(+), 39 deletions(-) diff --git a/rust/kona/crates/node/service/src/actors/derivation/actor/core.rs b/rust/kona/crates/node/service/src/actors/derivation/actor/core.rs index 830b7ce4eaf..e3a5ea1ba4d 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/actor/core.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/actor/core.rs @@ -369,9 +369,8 @@ where self.attempt_derivation().await?; } - DerivationActorRequest::ProcessEngineSyncCompletionRequest(sync_state) => { + DerivationActorRequest::ProcessEngineSyncCompletionRequest(safe_head) => { info!(target: "derivation", "Engine finished syncing, starting derivation."); - let safe_head = Box::new(sync_state.local_safe_head()); self.derivation_state_machine .update(&DerivationStateUpdate::ELSyncCompleted(safe_head.clone()))?; self.truncate_safe_db(*safe_head, "EL sync completion")?; diff --git a/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs b/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs index 01eb187e119..5bd4f9cfc00 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs @@ -10,7 +10,7 @@ use kona_derive::{ ActivationSignal, Pipeline, PipelineError, PipelineErrorKind, PipelineResult, ResetError, Signal, SignalReceiver, StepResult, }; -use kona_engine::{EngineSyncState, EngineSyncStateUpdate, FinalizeBlockId}; +use kona_engine::FinalizeBlockId; use kona_genesis::{HardForkConfig, RollupConfig, SystemConfig}; use kona_protocol::{BlockInfo, L2BlockInfo, OpAttributesWithParent}; use kona_safedb::{SafeDatabase, SafeDb, SafeDbError, SafeHeadRecord}; @@ -41,7 +41,12 @@ async fn reset_adopts_safe_head_and_resumes_atomically(#[case] previous_head: u6 l1_block(0).id(), watch::channel(None).0, ); - request_tx.send(engine_sync_completed_at(l2_block(previous_head))).await.unwrap(); + request_tx + .send(DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(l2_block( + previous_head, + )))) + .await + .unwrap(); actor.step().await.unwrap(); request_tx @@ -262,7 +267,10 @@ async fn test_origin_is_published(#[case] step_result: StepResult) { actor.step().await.unwrap(); assert_eq!(derivation_origin_rx.borrow().map(|origin| origin.id()), Some(safe_head.l1_origin)); - request_tx.send(engine_sync_completed_at(safe_head)).await.unwrap(); + request_tx + .send(DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(safe_head))) + .await + .unwrap(); actor.step().await.unwrap(); assert_eq!(derivation_origin_rx.borrow().unwrap().number, 11); @@ -457,16 +465,6 @@ fn l1_block(number: u64) -> BlockInfo { BlockInfo { hash: B256::with_last_byte(number as u8), number, ..Default::default() } } -/// The engine reporting that its sync completed with `safe_head` as its safe heads. -fn engine_sync_completed_at(safe_head: L2BlockInfo) -> DerivationActorRequest { - let sync_state = EngineSyncState::default().apply_update(EngineSyncStateUpdate { - local_safe_head: Some(safe_head), - safe_head: Some(safe_head), - ..Default::default() - }); - DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(sync_state)) -} - fn l2_block(number: u64) -> L2BlockInfo { L2BlockInfo { block_info: BlockInfo { @@ -537,7 +535,10 @@ impl RecordingActor { } async fn complete_el_sync_at(&mut self, head: u64) { - self.handle(engine_sync_completed_at(l2_block(head))).await; + self.handle(DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new( + l2_block(head), + ))) + .await; } async fn confirm_safe_head(&mut self, head: u64) { diff --git a/rust/kona/crates/node/service/src/actors/derivation/actor/tests/database.rs b/rust/kona/crates/node/service/src/actors/derivation/actor/tests/database.rs index eeee778a78a..699bc1e6933 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/actor/tests/database.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/actor/tests/database.rs @@ -135,7 +135,7 @@ fn engine_reset_request() -> DerivationActorRequest { } fn sync_completion_request() -> DerivationActorRequest { - engine_sync_completed_at(l2_block(RESUME_HEAD)) + DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(l2_block(RESUME_HEAD))) } #[rstest] diff --git a/rust/kona/crates/node/service/src/actors/derivation/actor/tests/op_node_parity.rs b/rust/kona/crates/node/service/src/actors/derivation/actor/tests/op_node_parity.rs index 0df6cef81c1..104544f2522 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/actor/tests/op_node_parity.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/actor/tests/op_node_parity.rs @@ -16,7 +16,7 @@ fn l1_head_update(number: u64) -> DerivationActorRequest { } fn el_sync_completed(head: u64) -> DerivationActorRequest { - engine_sync_completed_at(l2_block(head)) + DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(l2_block(head))) } /// A reset whose pipeline signal fails once with a temporary provider error neither stops the diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs index 7001d2f71d6..c7ec925ecd2 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs @@ -269,9 +269,9 @@ where debug!(target: "derivation", safe_head = ?*safe_head, "Received safe head from engine."); self.engine_l2_safe_head = *safe_head; } - DerivationActorRequest::ProcessEngineSyncCompletionRequest(sync_state) => { + DerivationActorRequest::ProcessEngineSyncCompletionRequest(safe_head) => { info!(target: "derivation", "Engine finished syncing, starting derivation."); - self.engine_l2_safe_head = sync_state.local_safe_head(); + self.engine_l2_safe_head = *safe_head; self.has_engine_sync_completed = true; } DerivationActorRequest::ProcessEngineSignalRequest(signal) => match *signal { diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs index 8f976aa256c..0eda07f0beb 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs @@ -70,7 +70,9 @@ impl Harness { async fn complete_engine_sync_with(&mut self, sync_state: EngineSyncState) { self.publish_engine_sync_state(sync_state); self.derivation_requests - .send(DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(sync_state))) + .send(DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new( + sync_state.local_safe_head(), + ))) .await .unwrap(); self.actor.step().await.unwrap(); diff --git a/rust/kona/crates/node/service/src/actors/derivation/request.rs b/rust/kona/crates/node/service/src/actors/derivation/request.rs index caa1e2b9ad2..9482eef5582 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/request.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/request.rs @@ -1,5 +1,4 @@ use kona_derive::Signal; -use kona_engine::EngineSyncState; use kona_protocol::{BlockInfo, L2BlockInfo}; use thiserror::Error; @@ -22,9 +21,9 @@ pub enum DerivationClientError { /// Inbound requests that the [`crate::DerivationActor`] can process. #[derive(Debug)] pub enum DerivationActorRequest { - /// Request to process the fact that Engine sync has completed, along with the engine's sync - /// state at that point. - ProcessEngineSyncCompletionRequest(Box), + /// Request to process the fact that Engine sync has completed, along with the current safe + /// head. + ProcessEngineSyncCompletionRequest(Box), /// Request to process the provided L2 engine safe head update. ProcessEngineSafeHeadUpdateRequest(Box), /// A request containing a [`Signal`] to the derivation pipeline. diff --git a/rust/kona/crates/node/service/src/actors/engine/actor.rs b/rust/kona/crates/node/service/src/actors/engine/actor.rs index 87619ba2756..f09047db8a2 100644 --- a/rust/kona/crates/node/service/src/actors/engine/actor.rs +++ b/rust/kona/crates/node/service/src/actors/engine/actor.rs @@ -251,7 +251,7 @@ where self.el_sync_complete = true; self.derivation_client - .notify_sync_completed(self.engine.state().sync_state) + .notify_sync_completed(self.engine.state().sync_state.local_safe_head()) .await .map(|_| Ok(())) .map_err(|e| { diff --git a/rust/kona/crates/node/service/src/actors/engine/client.rs b/rust/kona/crates/node/service/src/actors/engine/client.rs index 518e7b957d0..25322a5a7f2 100644 --- a/rust/kona/crates/node/service/src/actors/engine/client.rs +++ b/rust/kona/crates/node/service/src/actors/engine/client.rs @@ -2,7 +2,6 @@ use crate::{DerivationActorRequest, DerivationClientError, DerivationClientResul use async_trait::async_trait; use derive_more::Constructor; use kona_derive::Signal; -use kona_engine::EngineSyncState; use kona_protocol::L2BlockInfo; use std::fmt::Debug; use tokio::sync::mpsc; @@ -11,13 +10,9 @@ use tokio::sync::mpsc; #[cfg_attr(test, mockall::automock)] #[async_trait] pub trait EngineDerivationClient: Debug + Send + Sync { - /// Notifies the [`crate::DerivationActor`] that engine syncing has completed, with the engine's - /// sync state at that point. + /// Notifies the [`crate::DerivationActor`] that engine syncing has completed. /// Note: Does not wait for the derivation client to process this message. - async fn notify_sync_completed( - &self, - sync_state: EngineSyncState, - ) -> DerivationClientResult<()>; + async fn notify_sync_completed(&self, safe_head: L2BlockInfo) -> DerivationClientResult<()>; /// Sends the new engine `safe_head` to the [`crate::DerivationActor`]. /// Note: Does not wait for the derivation client to process this message. @@ -38,14 +33,11 @@ pub struct QueuedEngineDerivationClient { #[async_trait] impl EngineDerivationClient for QueuedEngineDerivationClient { - async fn notify_sync_completed( - &self, - sync_state: EngineSyncState, - ) -> DerivationClientResult<()> { + async fn notify_sync_completed(&self, safe_head: L2BlockInfo) -> DerivationClientResult<()> { info!(target: "engine", "Sending sync completed to derivation actor"); self.derivation_actor_request_tx - .send(DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(sync_state))) + .send(DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(safe_head))) .await .map_err(|_| { DerivationClientError::RequestError("request channel closed.".to_string()) diff --git a/rust/kona/crates/node/service/src/actors/engine/tests.rs b/rust/kona/crates/node/service/src/actors/engine/tests.rs index d000710cbf1..b61712d95b5 100644 --- a/rust/kona/crates/node/service/src/actors/engine/tests.rs +++ b/rust/kona/crates/node/service/src/actors/engine/tests.rs @@ -312,5 +312,5 @@ async fn derivation_is_handed_the_local_safe_head() { else { panic!("expected the sync completion second"); }; - assert_eq!(sent.local_safe_head(), local_safe_head); + assert_eq!(*sent, local_safe_head); } From e2bcf65ec2a0e8794c37936808398f103b90f775 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 12 Nov 2026 14:38:22 +0100 Subject: [PATCH 4/5] kona-node-service: reject a delegate status whose finalized is above its safe Such a snapshot is inconsistent: the engine would refuse to finalize it. op-node's follow mode rejects it the same way. A test also pins that any finalized block, genesis included, is accepted before the engine establishes finality, and the doc on the finality check now states that it only catches a fork at the engine's finalized height. Co-Authored-By: Claude Opus 5.5 --- .../src/actors/derivation/delegated/actor.rs | 23 ++++++++++++-- .../src/actors/derivation/delegated/tests.rs | 31 +++++++++++++++++++ 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs index c7ec925ecd2..9962dd9e388 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs @@ -165,6 +165,10 @@ where /// Rejects a finalized L2 block behind, or at the same height as but different from, the /// engine's. Either means the delegate lags or follows another chain, so none of its status /// applies: its safe head would move the engine back or onto that chain too. + /// + /// Only a fork at the engine's finalized height is caught here. A chain that forked below it + /// and reports a higher finalized block passes, and the execution layer rejects the forkchoice + /// update that would move onto it. fn validate_finalized_l2_progress( &self, reported: BlockNumHash, @@ -182,9 +186,20 @@ where Ok(()) } - /// Verifies that the finalized L2 block reported by the derivation delegate does not move back - /// from the engine's, and that its L1 info is consistent with the canonical L1 chain. + /// Verifies that the derivation delegate's finalized L2 block is not above its safe L2 block + /// and does not move back from the engine's, and that its L1 info is consistent with the + /// canonical L1 chain. async fn validate_sync_status(&mut self, v: &SyncStatus) -> bool { + // A finalized block above the safe head is an inconsistent snapshot; the engine would only + // refuse to finalize it. + if v.finalized_l2.block_info.number > v.safe_l2.block_info.number { + let err = DerivationDelegationError::FinalizedAheadOfSafe { + finalized: v.finalized_l2.block_info.number, + safe: v.safe_l2.block_info.number, + }; + warn!(target: "derivation", error = %err, "Inconsistent sync status from delegate"); + return false; + } if let Err(err) = self.validate_finalized_l2_progress(v.finalized_l2.block_info.id()) { warn!( target: "derivation", @@ -291,6 +306,10 @@ where #[derive(Error, Debug)] enum DerivationDelegationError { + /// The delegate's finalized L2 block is above its safe L2 block. + #[error("Finalized L2 {finalized} is ahead of safe L2 {safe}")] + FinalizedAheadOfSafe { finalized: u64, safe: u64 }, + /// The delegate's finalized L2 block is a different block at the engine's finalized height. #[error("Finalized L2 {reported:?} conflicts with the engine's finalized L2 {engine:?}")] FinalizedL2Conflict { engine: BlockNumHash, reported: BlockNumHash }, diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs index 0eda07f0beb..2468e203727 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs @@ -250,3 +250,34 @@ async fn poll_accepts_status_after_engine_skipped_forwarded_finality( assert_eq!(harness.forwarded(), forwarded_for(&canonical_status)); } + +/// Before the engine establishes finality, any finalized block from the delegate may initialize +/// it, including one at genesis height. +#[tokio::test(start_paused = true)] +async fn first_poll_accepts_finality_before_the_engine_establishes_it() { + let status = with_finalized_l2(l2_block(0, FINALIZED_ORIGIN)); + let mut harness = Harness::new([Ok(status.clone())]); + harness.complete_engine_sync().await; + + harness.actor.step().await.unwrap(); + + assert_eq!(harness.forwarded(), forwarded_for(&status)); +} + +/// A status whose finalized block is above its safe block is inconsistent and rejected whole. +#[rstest] +#[case::at_safe_head(l2_block(20, FINALIZED_ORIGIN), true)] +#[case::ahead_of_safe_head(l2_block(21, FINALIZED_ORIGIN), false)] +#[tokio::test(start_paused = true)] +async fn poll_rejects_status_whose_finalized_is_ahead_of_its_safe( + #[case] finalized_l2: L2BlockInfo, + #[case] accepted: bool, +) { + let status = with_finalized_l2(finalized_l2); + let mut harness = Harness::new([Ok(status.clone())]); + harness.complete_engine_sync().await; + + harness.actor.step().await.unwrap(); + + assert_eq!(harness.forwarded(), forwarded_if(accepted, &status)); +} From 5c7d4925978a6aaadb62fd8d98f8bdd6dcdf6e2c Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 12 Nov 2026 14:38:22 +0100 Subject: [PATCH 5/5] kona-node-service: drive a skipped finalization through the engine actor The test sends stale finalizations through the actor: it keeps running, an identical repeat keeps the same skip in memory, and a reset forgets it. Co-Authored-By: Claude Opus 5.5 --- .../service/src/actors/engine/queue_tests.rs | 46 ++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs b/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs index 3dc21d249e1..0b4bc8f58ff 100644 --- a/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs +++ b/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs @@ -4,7 +4,7 @@ use alloy_rpc_types_engine::{ ExecutionPayloadV1, ForkchoiceUpdated, PayloadStatus, PayloadStatusEnum, }; use kona_engine::{ - ConsolidateInput, EngineSyncStateUpdate, NoopBlockSink, + ConsolidateInput, EngineSyncStateUpdate, FinalizeBlockId, NoopBlockSink, test_utils::{TestEngineStateBuilder, l1_info_deposit, test_block_info, test_engine_client}, }; use op_alloy_consensus::OpBlock; @@ -140,3 +140,47 @@ async fn delegated_safe_head_is_sent_on_every_poll() { l1.assert_finished(); l2.assert_finished(); } + +/// A finalization the engine skips leaves the actor running, an identical repeat keeps the same +/// skip in memory, and a reset forgets it. +#[tokio::test(start_paused = true)] +async fn skipped_finalization_keeps_the_actor_running_until_a_reset_forgets_it() { + let config = Arc::new(RollupConfig::default()); + let (client, l1, l2) = test_engine_client(config.clone()); + let state = TestEngineStateBuilder::new() + .with_unsafe_head(test_block_info(30)) + .with_safe_head(test_block_info(20)) + .with_finalized_head(test_block_info(10)) + .build(); + let (state_tx, _state_rx) = watch::channel(state); + let (requests, request_rx) = mpsc::channel(8); + let (derivation_tx, _derivation_rx) = mpsc::channel(8); + let mut actor = EngineActor::new( + Arc::new(client), + config, + QueuedEngineDerivationClient { derivation_actor_request_tx: derivation_tx }, + Engine::new(state, state_tx), + None, + request_rx, + Arc::new(NoopBlockSink), + ); + let stale_finalization = + || EngineActorRequest::ProcessFinalizedL2Block(Box::new(FinalizeBlockId::ByNumber(5))); + + // Each step drains the previous request's task, then accepts the next request. + requests.send(stale_finalization()).await.unwrap(); + time::timeout(Duration::from_secs(1), actor.step()).await.unwrap().unwrap(); + requests.send(stale_finalization()).await.unwrap(); + time::timeout(Duration::from_secs(1), actor.step()).await.unwrap().unwrap(); + let first_skip = actor.last_skipped_task_error.clone(); + assert!(first_skip.is_some(), "the skipped finalization is remembered"); + + requests.send(stale_finalization()).await.unwrap(); + time::timeout(Duration::from_secs(1), actor.step()).await.unwrap().unwrap(); + assert_eq!(actor.last_skipped_task_error, first_skip); + + actor.begin_reset(); + assert_eq!(actor.last_skipped_task_error, None); + l1.assert_finished(); + l2.assert_finished(); +}