From 536e4f675a9fec332a0b5850702ea7c252579a6e Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 11 Nov 2026 18:16:47 +0100 Subject: [PATCH] kona-engine,kona-node-service: import delegated local-safe and cross-safe heads separately Under interop op-node imports a delegate's local-safe and cross-safe heads separately; kona-node copies the delegated safe head into both, so its forkchoice safe hash can run ahead of cross-chain verification. Kona-node now follows local-safe for the EL lookup, EL sync and reorgs, and moves the safe label to cross-safe only when it advances. Only the unsafe head and established finality count as proof of the chain; other heights are checked against an EL that applied the last forkchoice, and only at or below the unsafe head. A head that does not extend the unsafe chain reorgs directly, moving safe to cross-safe, clamped to finalized, unless the EL holds the current safe label, which a catch-up then keeps; only a conflict with finalized skips the update. While the EL has not applied the last forkchoice, safe only moves forward and nothing reorgs. Inconsistent heads are rejected, and the local-safe L1 origin must be canonical. Before interop both heads are one block, so non-interop chains see no change. A rejected update resets the engine; recovery is #24065. Closes #23911. BREAKING CHANGE: `ConsolidateInput::BlockInfo` is replaced by `Delegated(Box)`, built with `DelegatedSafeHeads::try_new`, which returns `InconsistentSafeHeadsError`; the `From` conversions are removed. Co-Authored-By: Claude Opus 5.5 --- .../kona-node/configuration.mdx | 20 + .../kona-node/design/engine.mdx | 2 +- .../bin/node/src/flags/engine/providers.rs | 3 + rust/kona/crates/node/engine/src/lib.rs | 9 +- .../tasks/consolidate/delegated_safe_heads.rs | 140 ++ .../src/task_queue/tasks/consolidate/error.rs | 3 +- .../src/task_queue/tasks/consolidate/mod.rs | 3 + .../src/task_queue/tasks/consolidate/task.rs | 1625 ++++++++++++----- .../node/engine/src/task_queue/tasks/mod.rs | 5 +- .../engine/src/test_utils/engine_state.rs | 15 +- .../crates/node/engine/src/test_utils/misc.rs | 12 + .../crates/node/engine/src/test_utils/mod.rs | 2 +- .../src/actors/derivation/delegated/actor.rs | 136 +- .../src/actors/derivation/delegated/tests.rs | 54 +- .../service/src/actors/engine/queue_tests.rs | 22 +- .../node/service/src/actors/engine/tests.rs | 9 +- 16 files changed, 1530 insertions(+), 530 deletions(-) create mode 100644 rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/delegated_safe_heads.rs diff --git a/docs/public-docs/node-operators/kona-node/configuration.mdx b/docs/public-docs/node-operators/kona-node/configuration.mdx index 30db30bd372..5fd4ac422d8 100644 --- a/docs/public-docs/node-operators/kona-node/configuration.mdx +++ b/docs/public-docs/node-operators/kona-node/configuration.mdx @@ -250,6 +250,26 @@ The RPC server starts once. If it stops unexpectedly, the node shuts down with a `kona-node` does not run a supervisor RPC server, so there is no `--supervisor.*` flag group. `--interop.dependency-set` is the node's only interop flag. +## Derivation Delegation Arguments + +| Flag | Env | Description | Default | +|------|-----|-------------|---------| +| `--l2-follow-source ` (alias `--l2.follow.source`) | `KONA_NODE_L2_FOLLOW_SOURCE` | RPC URL of an OP Stack consensus client exposing `optimism_syncStatus`. When set, `kona-node` derives nothing itself and follows that client's safe and finalized heads instead. Cannot be combined with `--safedb.path`. | - | + +Like op-node's follow-source mode, `kona-node` imports the source's local-safe and cross-safe +heads separately. The local-safe head is the block the execution client must have, or sync to, +and becomes the local-safe label; the cross-safe head becomes the safe label that forkchoice +updates carry, and finalization never goes above it. The safe label moves back to a lower +cross-safe head only when the source's chain has diverged from the node's, or when the unsafe head +moves to a source block whose ancestry the node cannot check and the execution client does not hold +the current safe label, and never below the finalized head. While the execution client has not +applied kona-node's last forkchoice update, for example while it is still syncing, its canonical +blocks do not count as evidence: a delegated block it lacks, or one that differs from its block, +moves the safe label only forward. +Before interop both heads are the same block. +Under interop, `kona-node` verifies no cross-chain dependencies itself, so its safe label is only +as safe as the source's cross-safe head: the source must verify them. + ## RPC Trust Flag diff --git a/docs/public-docs/node-operators/kona-node/design/engine.mdx b/docs/public-docs/node-operators/kona-node/design/engine.mdx index 129ca885a0b..b5438629c43 100644 --- a/docs/public-docs/node-operators/kona-node/design/engine.mdx +++ b/docs/public-docs/node-operators/kona-node/design/engine.mdx @@ -145,7 +145,7 @@ pub struct ConsolidateTask { pub client: Arc, /// The RollupConfig. pub cfg: Arc, - /// The input for consolidation (either attributes or block info). + /// The input for consolidation (either derived attributes or delegated safe heads). pub input: ConsolidateInput, /// Where to hand the decoded block once the engine has canonicalized it. pub block_sink: Arc, diff --git a/rust/kona/bin/node/src/flags/engine/providers.rs b/rust/kona/bin/node/src/flags/engine/providers.rs index 20b434fb96a..697d18b7a50 100644 --- a/rust/kona/bin/node/src/flags/engine/providers.rs +++ b/rust/kona/bin/node/src/flags/engine/providers.rs @@ -90,6 +90,9 @@ impl Default for L2ClientArgs { #[derive(Clone, Debug, Default, clap::Args)] pub struct DerivationDelegateArgs { /// The source must be an OP Stack L2 CL RPC exposing `optimism_syncStatus`. + /// + /// Its local-safe and cross-safe heads become this node's local-safe and safe labels. Under + /// interop this node verifies no cross-chain dependencies itself, so the source must. #[arg(long, visible_alias = "l2.follow.source", env = "KONA_NODE_L2_FOLLOW_SOURCE")] pub l2_follow_source: Option, } diff --git a/rust/kona/crates/node/engine/src/lib.rs b/rust/kona/crates/node/engine/src/lib.rs index eb26238be30..c6bcc1548bb 100644 --- a/rust/kona/crates/node/engine/src/lib.rs +++ b/rust/kona/crates/node/engine/src/lib.rs @@ -40,10 +40,11 @@ extern crate tracing; mod task_queue; pub use task_queue::{ BuildTask, BuildTaskError, CanonicalizeTask, CanonicalizeTaskError, ConsolidateInput, - ConsolidateTask, ConsolidateTaskError, Engine, EngineBuildError, EngineResetError, EngineTask, - EngineTaskError, EngineTaskErrorSeverity, EngineTaskErrors, ExecutableEngineTask, - FinalizeBlockId, FinalizeTask, FinalizeTaskError, InsertTask, InsertTaskError, SealTask, - SealTaskError, SynchronizeTaskError, + ConsolidateTask, ConsolidateTaskError, DelegatedSafeHeads, Engine, EngineBuildError, + EngineResetError, EngineTask, EngineTaskError, EngineTaskErrorSeverity, EngineTaskErrors, + ExecutableEngineTask, FinalizeBlockId, FinalizeTask, FinalizeTaskError, + InconsistentSafeHeadsError, InsertTask, InsertTaskError, SealTask, SealTaskError, + SynchronizeTaskError, }; mod attributes; diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/delegated_safe_heads.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/delegated_safe_heads.rs new file mode 100644 index 00000000000..200c72b469c --- /dev/null +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/delegated_safe_heads.rs @@ -0,0 +1,140 @@ +//! The safe heads a derivation delegate reports. + +use alloy_eips::BlockNumHash; +use kona_protocol::L2BlockInfo; +use thiserror::Error; + +/// The safe heads a derivation delegate reports, imported as separate labels. +/// +/// Before interop both are the same block. Under interop the cross-safe head can lag the +/// local-safe head while the delegate verifies cross-chain dependencies, and only the cross-safe +/// head may reach the execution layer as its safe block. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DelegatedSafeHeads { + local_safe_head: L2BlockInfo, + safe_head: L2BlockInfo, +} + +/// Safe heads that no consistent delegate reports. +#[derive(Debug, Clone, Copy, Error, PartialEq, Eq)] +pub enum InconsistentSafeHeadsError { + /// The cross-safe head is ahead of the local-safe head. + #[error("Safe head {safe} is ahead of local safe head {local_safe}")] + SafeAheadOfLocalSafe { + /// The number of the delegate's local-safe head. + local_safe: u64, + /// The number of the delegate's cross-safe head. + safe: u64, + }, + /// The cross-safe head is a different block at the local-safe head's height. + #[error("Safe head {safe:?} conflicts with local safe head {local_safe:?}")] + SafeConflictsWithLocalSafe { + /// The delegate's local-safe head. + local_safe: BlockNumHash, + /// The delegate's cross-safe head. + safe: BlockNumHash, + }, +} + +impl DelegatedSafeHeads { + /// Pairs the delegate's local-safe head, derived from L1 with cross-chain dependencies not + /// verified yet, with its cross-safe head, which must be at or below it on the same chain: not + /// ahead of it, and not a different block at its height. + pub fn try_new( + local_safe_head: L2BlockInfo, + safe_head: L2BlockInfo, + ) -> Result { + let local_safe = local_safe_head.block_info.id(); + let safe = safe_head.block_info.id(); + if safe.number > local_safe.number { + return Err(InconsistentSafeHeadsError::SafeAheadOfLocalSafe { + local_safe: local_safe.number, + safe: safe.number, + }); + } + if safe.number == local_safe.number && safe.hash != local_safe.hash { + return Err(InconsistentSafeHeadsError::SafeConflictsWithLocalSafe { + local_safe, + safe, + }); + } + Ok(Self { local_safe_head, safe_head }) + } + + /// The delegate's local-safe head. + pub const fn local_safe_head(&self) -> L2BlockInfo { + self.local_safe_head + } + + /// The delegate's cross-safe head. + pub const fn safe_head(&self) -> L2BlockInfo { + self.safe_head + } + + /// The cross-safe head when it is above `current_safe_head`, and `None` otherwise: a + /// cross-safe head at or below the safe label leaves it where it is. + pub(crate) const fn advancing_safe_head( + &self, + current_safe_head: &L2BlockInfo, + ) -> Option { + if self.safe_head.block_info.number > current_safe_head.block_info.number { + Some(self.safe_head) + } else { + None + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::test_block_info; + use alloy_primitives::B256; + use kona_protocol::BlockInfo; + use rstest::rstest; + + const LOCAL_SAFE_HEAD: L2BlockInfo = L2BlockInfo { + block_info: BlockInfo { + hash: B256::repeat_byte(0x10), + number: 10, + parent_hash: B256::ZERO, + timestamp: 0, + }, + l1_origin: BlockNumHash { hash: B256::ZERO, number: 0 }, + seq_num: 0, + }; + + /// A different block at the height of `block`. + fn conflicting_block(block: L2BlockInfo) -> L2BlockInfo { + L2BlockInfo { + block_info: BlockInfo { hash: !block.block_info.hash, ..block.block_info }, + ..block + } + } + + #[rstest] + #[case::cross_safe_lags(test_block_info(8), true)] + #[case::same_block(LOCAL_SAFE_HEAD, true)] + #[case::conflicting_at_same_height(conflicting_block(LOCAL_SAFE_HEAD), false)] + #[case::cross_safe_ahead(test_block_info(11), false)] + fn try_new_rejects_cross_safe_off_the_local_safe_chain( + #[case] safe_head: L2BlockInfo, + #[case] accepted: bool, + ) { + let heads = DelegatedSafeHeads::try_new(LOCAL_SAFE_HEAD, safe_head); + + assert_eq!(heads.is_ok(), accepted, "got {heads:?}"); + } + + /// Only a cross-safe head strictly above the safe label advances it. + #[rstest] + #[case::above(8, true)] + #[case::same_height(7, false)] + #[case::below(6, false)] + fn advancing_safe_head_requires_a_higher_block(#[case] safe: u64, #[case] advances: bool) { + let heads = + DelegatedSafeHeads::try_new(test_block_info(10), test_block_info(safe)).unwrap(); + + assert_eq!(heads.advancing_safe_head(&test_block_info(7)).is_some(), advances); + } +} diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/error.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/error.rs index 2ce1b187a76..0c9869ab7ae 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/error.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/error.rs @@ -12,8 +12,7 @@ pub enum ConsolidateTaskError { /// A delegated safe head would move the safe labels below finality. #[error("Delegated safe head breaks finality: {0}")] BreaksFinality(#[from] FinalityViolation), - /// Failed to fetch the execution layer's block at the height of a delegated safe head below the - /// safe head. + /// Failed to fetch the execution layer's block at the height of a delegated safe head. #[error("Failed to fetch the block at delegated safe head height {0}")] FailedToFetchDelegatedSafeBlock(u64), /// The unsafe L2 block is missing. diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/mod.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/mod.rs index 21b125bd135..110983b601f 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/mod.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/mod.rs @@ -1,5 +1,8 @@ //! Consolidation Task +mod delegated_safe_heads; +pub use delegated_safe_heads::{DelegatedSafeHeads, InconsistentSafeHeadsError}; + mod error; pub use error::ConsolidateTaskError; diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs index 00f31457729..f9d6c5dd2cc 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs @@ -1,10 +1,12 @@ //! A task to consolidate the engine state. use crate::{ - ConsolidateTaskError, EngineClient, EngineState, ExecutableEngineTask, ImportedBlockSink, + ConsolidateTaskError, DelegatedSafeHeads, EngineClient, EngineState, ExecutableEngineTask, + FinalityViolation, ImportedBlockSink, state::EngineSyncStateUpdate, task_queue::{build_and_import, synchronize, synchronize_unconditionally}, }; +use alloy_primitives::B256; use alloy_rpc_types_eth::Block; use async_trait::async_trait; use kona_genesis::RollupConfig; @@ -12,18 +14,18 @@ use kona_protocol::{L2BlockInfo, OpAttributesWithParent}; use op_alloy_rpc_types::Transaction; use std::sync::Arc; -/// Input for consolidation - either derived attributes or safe L2 block +/// Input for consolidation - either derived attributes or delegated safe heads #[derive(Debug, Clone)] pub enum ConsolidateInput { /// Consolidate based on derived attributes. Attributes(Box), - /// Derivation Delegation: consolidate based on safe L2 block info. - BlockInfo(L2BlockInfo), + /// Derivation Delegation: consolidate onto the safe heads a derivation delegate reports. + Delegated(Box), } -impl From for ConsolidateInput { - fn from(v: L2BlockInfo) -> Self { - Self::BlockInfo(v) +impl From for ConsolidateInput { + fn from(heads: DelegatedSafeHeads) -> Self { + Self::Delegated(Box::new(heads)) } } @@ -38,7 +40,7 @@ impl ConsolidateInput { const fn l2_block_number(&self) -> u64 { match self { Self::Attributes(attributes) => attributes.block_number(), - Self::BlockInfo(info) => info.block_info.number, + Self::Delegated(heads) => heads.local_safe_head().block_info.number, } } @@ -48,7 +50,7 @@ impl ConsolidateInput { Self::Attributes(attributes) => { crate::AttributesMatch::check(cfg, attributes, block).is_match() } - Self::BlockInfo(info) => block.header.hash == info.block_info.hash, + Self::Delegated(heads) => block.header.hash == heads.local_safe_head().block_info.hash, } } @@ -59,14 +61,14 @@ impl ConsolidateInput { } /// The [`ConsolidateTask`] attempts to consolidate the engine state -/// using the specified payload attributes or block info. +/// using the specified derived attributes or delegated safe heads. #[derive(Debug, Clone)] pub struct ConsolidateTask { /// The engine client. pub client: Arc, /// The [`RollupConfig`]. pub cfg: Arc, - /// The input for consolidation (either attributes or block info). + /// The input for consolidation (either derived attributes or delegated safe heads). pub input: ConsolidateInput, /// Where to hand the decoded block once the engine has canonicalized it. pub block_sink: Arc, @@ -100,29 +102,167 @@ impl ConsolidateTask { Ok(()) } - /// This provides symmetric fallback behavior with `build_and_import`. - async fn reconcile_to_safe_head( + /// The hash of the execution layer's canonical block at `number`, or `None` when it has none + /// there. Only the hash is needed, so the block is fetched without its transactions. + async fn el_block_hash(&self, number: u64) -> Result, ConsolidateTaskError> { + debug!(target: "engine", number, "Looking up the execution layer's block hash"); + let block = self.client.get_l2_block(number.into()).await.map_err(|err| { + warn!(target: "engine", ?err, number, "Failed to look up the execution layer's block"); + ConsolidateTaskError::FailedToFetchDelegatedSafeBlock(number) + })?; + let el_hash = block.map(|block| block.header.hash); + debug!(target: "engine", number, ?el_hash, "Looked up the execution layer's block hash"); + Ok(el_hash) + } + + /// The hash of the engine's unsafe or established finalized head at `number`, if one sits at + /// that height. Only these two pin the engine's chain: the unsafe head is the head the + /// execution layer follows, and finality cannot move, while the safe labels can stay on blocks + /// that a delegate's reorg abandoned. + fn chain_pinning_label_hash_at(state: &EngineState, number: u64) -> Option { + let sync_state = &state.sync_state; + Some(sync_state.unsafe_head()) + .filter(|unsafe_head| *unsafe_head != L2BlockInfo::default()) + .map(|unsafe_head| unsafe_head.block_info.id()) + .into_iter() + .chain(sync_state.established_finalized_head()) + .find(|label| label.number == number) + .map(|label| label.hash) + } + + /// The safe label when the cross-safe head cannot be checked against the engine's chain: the + /// cross-safe head only when it respects finality and advances the label. + fn unverified_advancing_safe_head( + state: &EngineState, + heads: &DelegatedSafeHeads, + ) -> Option { + Self::check_cross_safe_finality(state, heads) + .ok() + .and_then(|()| heads.advancing_safe_head(&state.sync_state.safe_head())) + } + + /// The one check of the delegated cross-safe head against finality. + fn check_cross_safe_finality( + state: &EngineState, + heads: &DelegatedSafeHeads, + ) -> Result<(), FinalityViolation> { + state.sync_state.ensure_respects_finality(heads.safe_head().block_info.id()) + } + + /// The safe label when the unsafe head moves onto a chain the current safe label is not proven + /// to be on: the cross-safe head, clamped to the finalized head; a conflict with finality + /// skips. + fn safe_label_off_engine_chain( + state: &EngineState, + heads: &DelegatedSafeHeads, + ) -> Result { + match Self::check_cross_safe_finality(state, heads) { + Ok(()) => Ok(heads.safe_head()), + Err(FinalityViolation::BehindFinalizedHead { .. }) => { + Ok(state.sync_state.finalized_head()) + } + Err(violation @ FinalityViolation::ConflictsWithFinalizedHead { .. }) => { + Err(violation.into()) + } + } + } + + /// The safe label for delegated safe heads whose local-safe head is on the engine's chain: the + /// cross-safe head when it advances the label, or when it replaces, at the label's height, a + /// safe label that a reorg left on abandoned blocks, and `None`, which keeps the label. + async fn safe_label_on_engine_chain( + &self, + state: &EngineState, + heads: &DelegatedSafeHeads, + ) -> Result, ConsolidateTaskError> { + if let Err(violation) = Self::check_cross_safe_finality(state, heads) { + debug!(target: "engine", %violation, ?heads, "Delegated cross-safe head breaks finality; keeping the safe label"); + return Ok(None); + } + let safe = state.sync_state.safe_head(); + let cross_safe = heads.safe_head(); + // Below the safe label the cross-safe head can neither advance nor replace it. + if cross_safe.block_info.number < safe.block_info.number { + return Ok(None); + } + + let is_known_on_engine_chain = cross_safe.block_info.id() == + heads.local_safe_head().block_info.id() || + cross_safe.block_info.id() == safe.block_info.id(); + let chain_hash = if is_known_on_engine_chain { + Some(cross_safe.block_info.hash) + } else { + self.engine_chain_hash_at(state, cross_safe.block_info.number).await? + }; + if chain_hash.is_some_and(|hash| hash != cross_safe.block_info.hash) { + debug!(target: "engine", ?heads, "Delegated cross-safe head is not on the chain of its local-safe head; keeping the safe label"); + return Ok(None); + } + let is_proven_on_engine_chain = chain_hash.is_some(); + let replaces_stale_safe_label = is_proven_on_engine_chain && + cross_safe.block_info.number == safe.block_info.number && + cross_safe.block_info.hash != safe.block_info.hash; + Ok(if replaces_stale_safe_label { + Some(cross_safe) + } else { + heads.advancing_safe_head(&safe) + }) + } + + /// The hash of the engine's chain at `number`: its unsafe or finalized head when one sits + /// there, and otherwise the execution layer's canonical block. `None` when nothing can tell. + async fn engine_chain_hash_at( + &self, + state: &EngineState, + number: u64, + ) -> Result, ConsolidateTaskError> { + if let Some(label_hash) = Self::chain_pinning_label_hash_at(state, number) { + return Ok(Some(label_hash)); + } + self.el_canonical_hash_at(state, number).await + } + + /// The hash of the execution layer's canonical block at `number`, only while that chain is the + /// engine's: the execution layer applied the engine's last forkchoice, and `number` is at or + /// below the unsafe head. Otherwise its canonical blocks may predate a sync, or sit above a + /// rewound unsafe head, and prove nothing, so `None` without a lookup. + async fn el_canonical_hash_at( + &self, + state: &EngineState, + number: u64, + ) -> Result, ConsolidateTaskError> { + let is_el_canonical_chain_current = state.el_applied_last_forkchoice && + number <= state.sync_state.unsafe_head().block_info.number; + if is_el_canonical_chain_current { self.el_block_hash(number).await } else { Ok(None) } + } + + /// Moves the unsafe and local-safe heads to the delegated local-safe head, and the safe label + /// to `safe_head`. + async fn reconcile_to_delegated_safe_heads( &self, state: &mut EngineState, - safe_l2: &L2BlockInfo, + heads: &DelegatedSafeHeads, + safe_head: Option, ) -> Result<(), ConsolidateTaskError> { - warn!( + debug!( target: "engine", - safe_l2 = %safe_l2, + local_safe_l2 = %heads.local_safe_head(), + safe_l2 = %heads.safe_head(), + ?safe_head, "Apply safe head" ); - // We intentionally set the unsafe head to safe_l2 to ensure the engine observes a - // self-consistent head state. This is required to correctly handle reorgs (where unsafe + // We intentionally set the unsafe head to the local-safe head to ensure the engine observes + // a self-consistent head state. This is required to correctly handle reorgs (where unsafe // may be ahead on a non-canonical fork) and to trigger EL sync when the local unsafe head // lags behind the safe head. - self.apply_delegated_safe_head( + self.apply_delegated_safe_heads( state, - safe_l2, + heads, EngineSyncStateUpdate { - local_safe_head: Some(*safe_l2), - safe_head: Some(*safe_l2), - unsafe_head: Some(*safe_l2), + local_safe_head: Some(heads.local_safe_head()), + safe_head, + unsafe_head: Some(heads.local_safe_head()), ..Default::default() }, "Updated safe head via follow safe", @@ -130,32 +270,86 @@ impl ConsolidateTask { .await } - /// Moves the safe labels to a delegated safe head that the execution layer does not have yet, - /// leaving the unsafe head in place. - /// - /// A block missing below the unsafe head means the execution layer is still syncing toward the - /// unsafe head. Resetting would restart that sync from the execution layer's old labels, and - /// moving the unsafe head back would retarget it. Like op-node's follow-source mode, only the - /// safe labels follow the delegate, and the forkchoice update carries them to the execution - /// layer. + /// Reorgs the engine onto delegated safe heads whose chain differs from the engine's, moving + /// the safe label to the cross-safe head even when lower, since the old safe block may be on + /// the abandoned chain, or to the finalized head when the cross-safe head is below it. + async fn reorg_onto_delegated_safe_heads( + &self, + state: &mut EngineState, + heads: &DelegatedSafeHeads, + ) -> Result<(), ConsolidateTaskError> { + let safe_head = Self::safe_label_off_engine_chain(state, heads)?; + warn!( + target: "engine", + local_safe_l2 = %heads.local_safe_head(), + %safe_head, + "Reorging onto the delegate's chain" + ); + self.reconcile_to_delegated_safe_heads(state, heads, Some(safe_head)).await + } + + /// Moves the engine to delegated safe heads at or above the unsafe head. Only the unsafe head + /// itself or its child extends the engine's chain; any other local-safe head has unknown + /// ancestry and reorgs, so the safe label follows it and stays an ancestor of the new head. + async fn extend_onto_delegated_safe_heads( + &self, + state: &mut EngineState, + heads: &DelegatedSafeHeads, + ) -> Result<(), ConsolidateTaskError> { + let unsafe_head = state.sync_state.unsafe_head().block_info; + let local_safe = heads.local_safe_head().block_info; + let is_at_unsafe_height = local_safe.number == unsafe_head.number; + if is_at_unsafe_height && local_safe.hash != unsafe_head.hash { + return self.reorg_onto_delegated_safe_heads(state, heads).await; + } + let is_child_of_unsafe_head = local_safe.number == unsafe_head.number + 1 && + local_safe.parent_hash == unsafe_head.hash; + let is_unproven_extension = !is_at_unsafe_height && !is_child_of_unsafe_head; + let would_lower_safe_label = + heads.safe_head().block_info.number < state.sync_state.safe_head().block_info.number; + if is_unproven_extension && + !(would_lower_safe_label && self.is_safe_label_on_el_chain(state).await?) + { + return self.reorg_onto_delegated_safe_heads(state, heads).await; + } + + let safe_head = self.safe_label_on_engine_chain(state, heads).await?; + self.reconcile_to_delegated_safe_heads(state, heads, safe_head).await + } + + /// Whether the execution layer's canonical chain holds the engine's safe label. A local-safe + /// head several blocks above the unsafe head has ancestry the engine cannot check, but a + /// delegate catching up builds on the safe label the execution layer holds, so that label stays + /// rather than dropping to a lower cross-safe head. + async fn is_safe_label_on_el_chain( + &self, + state: &EngineState, + ) -> Result { + let safe = state.sync_state.safe_head().block_info; + Ok(self.el_canonical_hash_at(state, safe.number).await? == Some(safe.hash)) + } + + /// Moves the safe labels to a delegated local-safe head that the syncing execution layer does + /// not have yet, keeping the unsafe head as its sync target; the cross-safe head only advances. async fn follow_safe_head_during_el_sync( &self, state: &mut EngineState, - safe_l2: &L2BlockInfo, + heads: &DelegatedSafeHeads, ) -> Result<(), ConsolidateTaskError> { warn!( target: "engine", - safe_l2 = %safe_l2, + local_safe_l2 = %heads.local_safe_head(), unsafe_number = state.sync_state.unsafe_head().block_info.number, "Execution layer does not have the delegated safe head yet; keeping its sync target" ); - self.apply_delegated_safe_head( + let safe_head = Self::unverified_advancing_safe_head(state, heads); + self.apply_delegated_safe_heads( state, - safe_l2, + heads, EngineSyncStateUpdate { - local_safe_head: Some(*safe_l2), - safe_head: Some(*safe_l2), + local_safe_head: Some(heads.local_safe_head()), + safe_head, ..Default::default() }, "Updated safe head while the execution layer syncs", @@ -163,97 +357,101 @@ impl ConsolidateTask { .await } - /// Applies `update`, whose labels follow the delegated `safe_l2`, and always sends its + /// Applies `update`, whose labels follow the delegated `heads`, and always sends its /// forkchoice update: the execution layer may have answered `SYNCING` to the same labels /// before, or lost them in a restart. - async fn apply_delegated_safe_head( + async fn apply_delegated_safe_heads( &self, state: &mut EngineState, - safe_l2: &L2BlockInfo, + heads: &DelegatedSafeHeads, update: EngineSyncStateUpdate, outcome: &'static str, ) -> Result<(), ConsolidateTaskError> { - synchronize_unconditionally(self.client.as_ref(), state, update) - .await - .map_err(|e| { - warn!(target: "engine", ?e, safe_l2 = %safe_l2, "Applying the delegated safe head failed"); - e - })?; + synchronize_unconditionally(self.client.as_ref(), state, update).await.map_err(|e| { + warn!(target: "engine", ?e, ?heads, "Applying the delegated safe heads failed"); + e + })?; info!( target: "engine", - hash = %safe_l2.block_info.hash, - number = safe_l2.block_info.number, + local_safe_hash = %heads.local_safe_head().block_info.hash, + local_safe_number = heads.local_safe_head().block_info.number, + safe_hash = %state.sync_state.safe_head().block_info.hash, + safe_number = state.sync_state.safe_head().block_info.number, "{outcome}" ); Ok(()) } - /// Follows a delegated safe head below the safe head. Only a different block at its height in - /// a synced execution layer proves the delegate diverged and reorgs the engine onto it; the - /// same block (a lagging delegate) or a missing one leaves the labels. After a delegate L1 - /// reorg whose fork point is above the head it reports, the labels stay on the abandoned - /// blocks until it reports a head above that point. - async fn follow_delegated_safe_head_behind_safe_head( + /// Follows delegated safe heads below the engine's local-safe head. Only a different block at + /// that height in a synced execution layer proves divergence and reorgs; otherwise the + /// cross-safe head may only move the safe label. After a delegate L1 reorg whose fork point is + /// above its reported head, the labels stay on the abandoned blocks until it passes that point. + async fn follow_lagging_delegated_safe_heads( &self, state: &mut EngineState, - safe_l2: &L2BlockInfo, + heads: &DelegatedSafeHeads, ) -> Result<(), ConsolidateTaskError> { - // Until the execution layer applies the engine's forkchoice, its canonical block at that - // height may predate the engine's own labels, so it proves nothing about the delegate. - if !state.el_applied_last_forkchoice { + let local_safe = heads.local_safe_head().block_info; + let el_hash = self.el_canonical_hash_at(state, local_safe.number).await?; + let safe_head = match el_hash { + Some(el_hash) if el_hash != local_safe.hash => { + debug!(target: "engine", ?heads, %el_hash, "Delegated local-safe head behind the engine's diverges from the execution layer"); + return self.reorg_onto_delegated_safe_heads(state, heads).await; + } + Some(_) => self.safe_label_on_engine_chain(state, heads).await?, + None => Self::unverified_advancing_safe_head(state, heads), + }; + let Some(safe_head) = safe_head else { debug!( target: "engine", - safe_l2 = %safe_l2, - "Ignoring a delegated safe head behind the safe head while the execution layer syncs" + ?heads, + local_safe_number = state.sync_state.local_safe_head().block_info.number, + "Ignoring delegated safe heads behind the local-safe head" ); return Ok(()); - } - debug!( - target: "engine", - safe_l2 = %safe_l2, - "Looking up the execution layer's block at a delegated safe head behind the safe head" - ); - let el_block = - self.client.get_l2_block(safe_l2.block_info.number.into()).await.map_err(|err| { - warn!( - target: "engine", - ?err, - safe_l2 = %safe_l2, - "Failed to look up the execution layer's block at a delegated safe head" - ); - ConsolidateTaskError::FailedToFetchDelegatedSafeBlock(safe_l2.block_info.number) - })?; - debug!( - target: "engine", - safe_l2 = %safe_l2, - el_hash = ?el_block.as_ref().map(|block| block.header.hash), - "Looked up the execution layer's block at a delegated safe head" - ); - match el_block { - Some(el_block) if el_block.header.hash != safe_l2.block_info.hash => { - debug!( - target: "engine", - safe_l2 = %safe_l2, - el_hash = %el_block.header.hash, - "Delegated safe head behind the safe head diverges from the execution layer" - ); - self.reconcile_to_safe_head(state, safe_l2).await + }; + self.apply_delegated_safe_heads( + state, + heads, + EngineSyncStateUpdate { safe_head: Some(safe_head), ..Default::default() }, + "Updated safe head from a delegate behind the local-safe head", + ) + .await + } + + /// The safe label after consolidating onto `consolidated`, which the execution layer has: that + /// block for derived attributes, and the judged cross-safe head for a delegate. + async fn consolidated_safe_head( + &self, + state: &EngineState, + consolidated: L2BlockInfo, + ) -> Result, ConsolidateTaskError> { + let ConsolidateInput::Delegated(heads) = &self.input else { + return Ok(Some(consolidated)); + }; + self.safe_label_on_engine_chain(state, heads).await + } + + /// Handles a block that does not match the input: derived attributes build their own block, + /// and delegated safe heads reorg the engine onto the delegate's chain. + async fn reconcile_mismatch( + &self, + state: &mut EngineState, + ) -> Result<(), ConsolidateTaskError> { + match &self.input { + ConsolidateInput::Attributes(attributes) => { + self.build_and_import_attributes(state, attributes).await } - _ => { - debug!( - target: "engine", - safe_l2 = %safe_l2, - safe_number = state.sync_state.safe_head().block_info.number, - "Ignoring a delegated safe head behind the safe head" - ); - Ok(()) + ConsolidateInput::Delegated(heads) => { + self.reorg_onto_delegated_safe_heads(state, heads).await } } } - /// Handles the fallback case when the block doesn't match the input or does not exist. + /// Handles input at or above the unsafe head: derived attributes build their block, and + /// delegated safe heads move the engine onto them. async fn reconcile_unsafe_to_safe( &self, state: &mut EngineState, @@ -262,8 +460,8 @@ impl ConsolidateTask { ConsolidateInput::Attributes(attributes) => { self.build_and_import_attributes(state, attributes).await } - ConsolidateInput::BlockInfo(safe_l2) => { - self.reconcile_to_safe_head(state, safe_l2).await + ConsolidateInput::Delegated(heads) => { + self.extend_onto_delegated_safe_heads(state, heads).await } } } @@ -282,8 +480,8 @@ impl ConsolidateTask { let block = match self.client.l2_block_by_label(block_num.into()).await { Ok(Some(block)) => block, Ok(None) => { - if let ConsolidateInput::BlockInfo(safe_l2) = &self.input { - return self.follow_safe_head_during_el_sync(state, safe_l2).await; + if let ConsolidateInput::Delegated(heads) = &self.input { + return self.follow_safe_head_during_el_sync(state, heads).await; } warn!(target: "engine", "Received `None` block for {}", block_num); return Err(ConsolidateTaskError::MissingUnsafeL2Block(block_num)); @@ -294,98 +492,107 @@ impl ConsolidateTask { }; let block_hash = block.header.hash; - if self.input.is_consistent_with_block(&self.cfg, &block) { - trace!( + if !self.input.is_consistent_with_block(&self.cfg, &block) { + // A different block from an execution layer that has not applied the engine's + // forkchoice may be from before its sync, so it does not prove the delegate diverged. + if let ConsolidateInput::Delegated(heads) = &self.input && + !state.el_applied_last_forkchoice + { + return self.follow_safe_head_during_el_sync(state, heads).await; + } + debug!( target: "engine", input = ?self.input, block_hash = %block_hash, - "Consolidating engine state", + "ConsolidateInput mismatch! Initiating reorg", ); - let consensus_block = - block.into_consensus().map_transactions(|t| t.inner.inner.into_inner()); - match L2BlockInfo::from_block_and_genesis(&consensus_block, &self.cfg.genesis) { - // A span batch's blocks before its last one only move the safe label; the last - // block's forkchoice update delivers them all. This avoids sending a FCU call for - // every block in the span batch. - Ok(block_info) if self.input.is_intermediate_span_block() => { - // The next attributes built are this block's child, and ask for its config. - self.block_sink.block_imported(consensus_block, block_info); - - // Apply a transient update to the safe head. - state.sync_state = state.sync_state.apply_update(EngineSyncStateUpdate { - safe_head: Some(block_info), - local_safe_head: Some(block_info), - ..Default::default() - }); - - info!( - target: "engine", - hash = %block_info.block_info.hash, - number = block_info.block_info.number, - "Updated safe head via L1 consolidation" - ); - - return Ok(()); - } - Ok(block_info) => { - // The next attributes built are this block's child, and ask for its config. - self.block_sink.block_imported(consensus_block, block_info); - - let update = EngineSyncStateUpdate { - safe_head: Some(block_info), - local_safe_head: Some(block_info), - ..Default::default() - }; - let synchronized = match &self.input { - // Derived attributes move the safe head on every consolidation, so an - // unchanged forkchoice was already sent. - ConsolidateInput::Attributes(_) => { - synchronize(self.client.as_ref(), state, update).await - } - // A delegated safe head can repeat a forkchoice the execution layer - // answered `SYNCING` or lost, so it is always sent. - ConsolidateInput::BlockInfo(_) => { - synchronize_unconditionally(self.client.as_ref(), state, update).await - } - }; - synchronized.map_err(|e| { - warn!(target: "engine", ?e, "Consolidation failed"); - e - })?; - - info!( - target: "engine", - hash = %block_info.block_info.hash, - number = block_info.block_info.number, - "Updated safe head via L1 consolidation" - ); - - return Ok(()); - } - Err(e) => { - // Continue on to build the block since we failed to construct the block info. - warn!(target: "engine", ?e, "Failed to construct L2BlockInfo, proceeding to build task"); - } - } - } - - // A different block from an execution layer that has not applied the engine's forkchoice - // may be from before its sync, so it does not prove the delegate diverged. - if let ConsolidateInput::BlockInfo(safe_l2) = &self.input && - !state.el_applied_last_forkchoice - { - return self.follow_safe_head_during_el_sync(state, safe_l2).await; + return self.reconcile_mismatch(state).await; } - debug!( + trace!( target: "engine", input = ?self.input, block_hash = %block_hash, - "ConsolidateInput mismatch! Initiating reorg", + "Consolidating engine state", + ); + let consensus_block = + block.into_consensus().map_transactions(|t| t.inner.inner.into_inner()); + let block_info = match ( + L2BlockInfo::from_block_and_genesis(&consensus_block, &self.cfg.genesis), + &self.input, + ) { + (Ok(block_info), _) => block_info, + // The block matches the delegated local-safe head by hash, so the delegate's info + // describes it, and the block stays where it is on the engine's chain. + (Err(e), ConsolidateInput::Delegated(heads)) => { + warn!(target: "engine", ?e, "Failed to construct L2BlockInfo, using the delegated local-safe head's"); + heads.local_safe_head() + } + (Err(e), ConsolidateInput::Attributes(attributes)) => { + // Continue on to build the block since we failed to construct the block info. + warn!(target: "engine", ?e, "Failed to construct L2BlockInfo, proceeding to build task"); + return self.build_and_import_attributes(state, attributes).await; + } + }; + + // A span batch's blocks before its last one only move the safe label; the last block's + // forkchoice update delivers them all. This avoids sending a FCU call for every block in + // the span batch. + if self.input.is_intermediate_span_block() { + // The next attributes built are this block's child, and ask for its config. + self.block_sink.block_imported(consensus_block, block_info); + + // Apply a transient update to the safe head. + state.sync_state = state.sync_state.apply_update(EngineSyncStateUpdate { + local_safe_head: Some(block_info), + safe_head: Some(block_info), + ..Default::default() + }); + + info!( + target: "engine", + hash = %block_info.block_info.hash, + number = block_info.block_info.number, + "Updated safe head via L1 consolidation" + ); + + return Ok(()); + } + + let safe_head = self.consolidated_safe_head(state, block_info).await?; + // The next attributes built are this block's child, and ask for its config. + self.block_sink.block_imported(consensus_block, block_info); + + let update = EngineSyncStateUpdate { + local_safe_head: Some(block_info), + safe_head, + ..Default::default() + }; + let synchronized = match &self.input { + // Derived attributes move the safe head on every consolidation, so an unchanged + // forkchoice was already sent. + ConsolidateInput::Attributes(_) => { + synchronize(self.client.as_ref(), state, update).await + } + // A delegated safe head can repeat a forkchoice the execution layer answered `SYNCING` + // or lost, so it is always sent. + ConsolidateInput::Delegated(_) => { + synchronize_unconditionally(self.client.as_ref(), state, update).await + } + }; + synchronized.map_err(|e| { + warn!(target: "engine", ?e, "Consolidation failed"); + e + })?; + + info!( + target: "engine", + hash = %block_info.block_info.hash, + number = block_info.block_info.number, + "Updated safe head via L1 consolidation" ); - // Handle mismatch case - called when consistency check fails - // or when L2BlockInfo construction fails in Attributes branch - self.reconcile_unsafe_to_safe(state).await + + Ok(()) } } @@ -401,29 +608,36 @@ impl ExecutableEngineTask for ConsolidateTask { // DerivationActor and EngineActor coordinate both safe and unsafe heads. In this case, we // consolidate as long as the unsafe head has not fallen behind. // - // - `BlockInfo`: The safe head is injected externally by the DerivationActor while delegating - // derivation, and is not coordinated with the EngineActor's safe/unsafe heads. If the - // injected safe head is ahead of the EngineActor's unsafe head, we reconcile the unsafe chain - // up to the safe head instead of consolidating. + // - `Delegated`: The safe heads are injected externally by the DerivationActor while delegating + // derivation, and are not coordinated with the EngineActor's heads. A local-safe head below + // the engine's comes from a lagging or diverged delegate; one below the unsafe head is + // consolidated against the execution layer's block; one at or above the unsafe head moves the + // unsafe chain onto it. async fn execute(&self, state: &mut EngineState) -> Result<(), ConsolidateTaskError> { - if let ConsolidateInput::BlockInfo(safe_l2) = &self.input { - state.sync_state.ensure_respects_finality(safe_l2.block_info.id())?; - // Labels with the unsafe head below the safe head take the paths below, which move the - // labels back to a consistent state; ignoring a lagging delegate would keep them broken - // until a finalization above the unsafe head fails. - let are_labels_consistent = state.sync_state.unsafe_head().block_info.number >= - state.sync_state.safe_head().block_info.number; - if are_labels_consistent && - safe_l2.block_info.number < state.sync_state.safe_head().block_info.number + let ConsolidateInput::Delegated(heads) = &self.input else { + return if state.sync_state.safe_head().block_info.number < + state.sync_state.unsafe_head().block_info.number { - return self.follow_delegated_safe_head_behind_safe_head(state, safe_l2).await; - } - } - let safe_head_number = match &self.input { - ConsolidateInput::Attributes { .. } => state.sync_state.safe_head().block_info.number, - ConsolidateInput::BlockInfo(safe_block_info) => safe_block_info.block_info.number, + self.consolidate(state).await + } else { + self.reconcile_unsafe_to_safe(state).await + }; }; - if safe_head_number < state.sync_state.unsafe_head().block_info.number { + + let local_safe_number = heads.local_safe_head().block_info.number; + state.sync_state.ensure_respects_finality(heads.local_safe_head().block_info.id())?; + // Labels with the unsafe head below the safe labels take the paths below, which move them + // back into order; ignoring a lagging delegate would keep them broken until a finalization + // above the unsafe head fails. + let unsafe_number = state.sync_state.unsafe_head().block_info.number; + let are_labels_consistent = unsafe_number >= + state.sync_state.local_safe_head().block_info.number && + unsafe_number >= state.sync_state.safe_head().block_info.number; + if are_labels_consistent && + local_safe_number < state.sync_state.local_safe_head().block_info.number + { + self.follow_lagging_delegated_safe_heads(state, heads).await + } else if local_safe_number < state.sync_state.unsafe_head().block_info.number { self.consolidate(state).await } else { self.reconcile_unsafe_to_safe(state).await @@ -435,11 +649,11 @@ impl ExecutableEngineTask for ConsolidateTask { mod tests { use super::*; use crate::{ - EngineSyncState, EngineTaskError, FinalityViolation, ImportedBlockSink, NoopBlockSink, - SealTaskError, + EngineSyncState, EngineTaskError, EngineTaskErrorSeverity, FinalityViolation, + ImportedBlockSink, NoopBlockSink, SealTaskError, test_utils::{ - TestAttributesBuilder, TestEngineStateBuilder, l1_info_deposit, test_block_info, - test_engine_client, + RpcMock, TestAttributesBuilder, TestEngineStateBuilder, delegated_safe_heads, + l1_info_deposit, test_block_info, test_engine_client, }, }; use alloy_consensus::transaction::Recovered; @@ -481,7 +695,7 @@ mod tests { }; let sink = Arc::new(RecordingSink::default()); let task = - ConsolidateTask::new(client, cfg, ConsolidateInput::BlockInfo(safe_l2), sink.clone()); + ConsolidateTask::new(client, cfg, delegated_safe_heads(safe_l2, safe_l2), sink.clone()); task.consolidate(&mut EngineState::default()).await.unwrap(); @@ -622,6 +836,7 @@ mod tests { fn engine_state_with_safe_head(safe_head: L2BlockInfo) -> EngineState { TestEngineStateBuilder::new() .with_unsafe_head(test_block_info(20)) + .with_local_safe_head(safe_head) .with_safe_head(safe_head) .with_finalized_head(test_block_info(5)) .build() @@ -660,9 +875,10 @@ mod tests { el_has_block.then_some(block) } - /// Either `target` itself or an older block, for a label that may already sit at the target. - fn target_or_older_block(is_at_target: bool, target: L2BlockInfo) -> L2BlockInfo { - if is_at_target { target } else { test_block_info(5) } + /// The safe head before consolidation: either already the delegated target, as a `SYNCING` + /// answer or an earlier poll leaves it, or an older block. + fn initial_safe_head(starts_at_target: bool, target: L2BlockInfo) -> L2BlockInfo { + if starts_at_target { target } else { test_block_info(5) } } /// Below the unsafe head, a delegated safe head always reaches the execution layer: when the @@ -682,10 +898,8 @@ mod tests { ) { let (block, cfg) = genesis_pinned_block(10); let delegated_safe_head = block_info_of(&cfg, &block); - let mut state = engine_state_with_safe_head(target_or_older_block( - starts_at_target, - delegated_safe_head, - )); + let mut state = + engine_state_with_safe_head(initial_safe_head(starts_at_target, delegated_safe_head)); let expected_sync_state = with_safe_head(&state, delegated_safe_head); let (client, l1, l2) = test_engine_client(cfg.clone()); l2.expect_params( @@ -698,7 +912,7 @@ mod tests { let task = ConsolidateTask::new( Arc::new(client), cfg, - ConsolidateInput::BlockInfo(delegated_safe_head), + delegated_safe_heads(delegated_safe_head, delegated_safe_head), sink.clone(), ); @@ -807,7 +1021,7 @@ mod tests { l2.assert_finished(); } - /// Where a delegated safe head sits against the finalized head. + /// Where a delegated local-safe head sits against the finalized head. #[derive(Clone, Copy, Debug)] enum TargetAgainstFinalized { Below, @@ -830,302 +1044,835 @@ mod tests { } } - /// The outcome of checking a delegated safe head against finality. - #[derive(Clone, Copy, Debug, PartialEq, Eq)] - enum FinalityCheck { - Accepted, - BehindFinalizedHead, - ConflictsWithFinalizedHead, - } - - impl FinalityCheck { - fn of(result: &Result<(), ConsolidateTaskError>) -> Self { - match result { - Ok(()) => Self::Accepted, - Err(ConsolidateTaskError::BreaksFinality( - FinalityViolation::BehindFinalizedHead { .. }, - )) => Self::BehindFinalizedHead, - Err(ConsolidateTaskError::BreaksFinality( - FinalityViolation::ConflictsWithFinalizedHead { .. }, - )) => Self::ConflictsWithFinalizedHead, - Err(err) => panic!("unexpected error {err:?}"), - } - } + fn is_behind_finality(result: &Result<(), ConsolidateTaskError>) -> bool { + matches!( + result, + Err(ConsolidateTaskError::BreaksFinality( + FinalityViolation::BehindFinalizedHead { .. } + )) + ) + } + + fn is_conflicting_with_finality(result: &Result<(), ConsolidateTaskError>) -> bool { + matches!( + result, + Err(ConsolidateTaskError::BreaksFinality( + FinalityViolation::ConflictsWithFinalizedHead { .. } + )) + ) } - /// A delegated safe head that breaks finality, as a stale delegate snapshot reports, is skipped - /// before the execution layer is asked anything, and leaves every label where it was. + /// A delegated local-safe head that breaks finality, as a stale delegate snapshot reports, is + /// skipped before the execution layer is asked anything, and leaves every label where it was. #[rstest::rstest] - #[case::below(TargetAgainstFinalized::Below, FinalityCheck::BehindFinalizedHead)] - #[case::conflicting( - TargetAgainstFinalized::ConflictingAt, - FinalityCheck::ConflictsWithFinalizedHead - )] + #[case::below(TargetAgainstFinalized::Below, is_behind_finality)] + #[case::conflicting(TargetAgainstFinalized::ConflictingAt, is_conflicting_with_finality)] #[tokio::test] - async fn delegated_safe_head_breaking_finality_is_skipped( + async fn delegated_local_safe_head_breaking_finality_is_skipped( #[case] target: TargetAgainstFinalized, - #[case] expected: FinalityCheck, + #[case] is_expected: fn(&Result<(), ConsolidateTaskError>) -> bool, ) { - let cfg = Arc::new(RollupConfig::default()); let mut state = engine_state_with_safe_head(test_block_info(12)); let state_before = state; - let delegated_safe_head = target.delegated_safe_head(state.sync_state.finalized_head()); + let delegated = target.delegated_safe_head(state.sync_state.finalized_head()); + + let result = run_delegated(&mut state, delegated, delegated, |_| {}).await; + + assert!(is_expected(&result), "got {result:?}"); + assert_eq!(result.unwrap_err().severity(), crate::EngineTaskErrorSeverity::Skip); + assert_eq!(state, state_before); + } + + /// A block at `number` on the chain identified by `chain`, and its info. Blocks on different + /// chains at the same height have different hashes. + fn chain_block(number: u64, chain: u64) -> (RpcBlock, L2BlockInfo) { + let header = RpcHeader::new(alloy_consensus::Header { + number, + timestamp: chain, + ..Default::default() + }); + let info = L2BlockInfo { + block_info: BlockInfo { hash: header.hash, number, ..Default::default() }, + ..Default::default() + }; + (RpcBlock::new(header, BlockTransactions::Full(vec![])), info) + } + + /// The engine's chain. + const ENGINE_CHAIN: u64 = 0; + /// A chain that diverged from the engine's. + const OTHER_CHAIN: u64 = 1; + + fn engine_chain_block(number: u64) -> L2BlockInfo { + chain_block(number, ENGINE_CHAIN).1 + } + + fn other_chain_block(number: u64) -> L2BlockInfo { + chain_block(number, OTHER_CHAIN).1 + } + + /// What the execution layer holds at a height: the engine chain's block, another chain's, or + /// nothing. + #[derive(Clone, Copy, Debug)] + enum ElHas { + EngineChain, + Missing, + OtherChain, + } + + impl ElHas { + fn block(self, number: u64) -> Option> { + match self { + Self::EngineChain => Some(chain_block(number, ENGINE_CHAIN).0), + Self::Missing => None, + Self::OtherChain => Some(chain_block(number, OTHER_CHAIN).0), + } + } + } + + /// An engine state with the unsafe head at 20 and the finalized head at 5 on the engine's + /// chain, and its local-safe and safe heads at the given heights, which differ under interop. + fn engine_state_with_safe_labels(local_safe: u64, safe: u64) -> EngineState { + TestEngineStateBuilder::new() + .with_unsafe_head(engine_chain_block(20)) + .with_local_safe_head(engine_chain_block(local_safe)) + .with_safe_head(engine_chain_block(safe)) + .with_finalized_head(engine_chain_block(5)) + .build() + } + + /// Runs a consolidation of the delegated `local_safe` and `cross_safe` heads against an + /// execution layer that `expect_engine_calls` scripts, and checks every scripted call was made. + /// Genesis is pinned to the local-safe head, so its empty block decodes without an L1-info + /// deposit. + async fn run_delegated( + state: &mut EngineState, + local_safe: L2BlockInfo, + cross_safe: L2BlockInfo, + expect_engine_calls: impl FnOnce(&RpcMock), + ) -> Result<(), ConsolidateTaskError> { + let cfg = Arc::new(RollupConfig { + genesis: ChainGenesis { l2: local_safe.block_info.id(), ..Default::default() }, + ..Default::default() + }); let (client, l1, l2) = test_engine_client(cfg.clone()); + expect_engine_calls(&l2); let task = ConsolidateTask::new( Arc::new(client), cfg, - ConsolidateInput::BlockInfo(delegated_safe_head), + delegated_safe_heads(local_safe, cross_safe), Arc::new(NoopBlockSink), ); - let result = task.execute(&mut state).await; + let result = task.execute(state).await; - assert_eq!(FinalityCheck::of(&result), expected); - assert_eq!(result.unwrap_err().severity(), crate::EngineTaskErrorSeverity::Skip); - assert_eq!(state, state_before); l1.assert_finished(); l2.assert_finished(); + result + } + + /// The labels after the engine moves its local-safe head, and its safe head to `safe`. + fn labels_after( + state: &EngineState, + local_safe: L2BlockInfo, + safe: L2BlockInfo, + ) -> EngineSyncState { + state.sync_state.apply_update(EngineSyncStateUpdate { + local_safe_head: Some(local_safe), + safe_head: Some(safe), + ..Default::default() + }) + } + + /// The labels after a reorg onto the delegated heads. + fn labels_after_reorg( + state: &EngineState, + local_safe: L2BlockInfo, + cross_safe: L2BlockInfo, + ) -> EngineSyncState { + state.sync_state.apply_update(EngineSyncStateUpdate { + local_safe_head: Some(local_safe), + safe_head: Some(cross_safe), + unsafe_head: Some(local_safe), + ..Default::default() + }) + } + + /// Below the unsafe head, a delegated local-safe head on the engine's chain that the execution + /// layer has becomes the local-safe label, and a cross-safe head on that chain above the safe + /// label moves it once the execution layer's block at its height confirms it. + #[tokio::test] + async fn advancing_cross_safe_head_moves_the_safe_label() { + let (local_safe, cross_safe) = (engine_chain_block(10), engine_chain_block(8)); + let mut state = engine_state_with_safe_labels(9, 7); + let expected = labels_after(&state, local_safe, cross_safe); + + run_delegated(&mut state, local_safe, cross_safe, |l2| { + l2.expect_block_by_number(10, ElHas::EngineChain.block(10)); + l2.expect_block_header_by_number(8, ElHas::EngineChain.block(8)); + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected); + } + + /// The safe label's own block is known, so a cross-safe head there needs no lookup. + fn cross_safe_at_safe_label(l2: &RpcMock) { + l2.expect_block_by_number(10, ElHas::EngineChain.block(10)); + } + + /// A cross-safe head behind the safe label can neither advance nor replace it, so it is not + /// looked up. + fn cross_safe_behind_safe_label(l2: &RpcMock) { + l2.expect_block_by_number(10, ElHas::EngineChain.block(10)); + } + + /// The execution layer still syncs toward the local-safe head, so the cross-safe head is not + /// looked up. + fn el_syncing_cross_safe_behind_safe_label(l2: &RpcMock) { + l2.expect_block_by_number(10, ElHas::Missing.block(10)); + } + + /// The execution layer has another block at the cross-safe height than the delegate's. + fn cross_safe_off_the_local_safe_chain(l2: &RpcMock) { + l2.expect_block_by_number(10, ElHas::EngineChain.block(10)); + l2.expect_block_header_by_number(8, ElHas::OtherChain.block(8)); } - /// A delegated safe head from a lagging delegate, between the finalized head, inclusive, and - /// the safe head, does not move the safe labels back, whether or not the execution layer has - /// its block, and sends no forkchoice update. + /// A cross-safe head at or below the safe label leaves it, so a load-balanced delegate whose + /// cross-safe head lags does not drop the forkchoice update's safe block; so does one that is + /// not on the chain of its own local-safe head, which the execution layer has. #[rstest::rstest] - #[case::el_has_block(true, false)] - #[case::el_missing_block(false, false)] - #[case::at_finalized_head(true, true)] + #[case::at_safe_label(7, cross_safe_at_safe_label)] + #[case::behind_safe_label(6, cross_safe_behind_safe_label)] + #[case::el_syncing_behind_safe_label(6, el_syncing_cross_safe_behind_safe_label)] + #[case::off_the_local_safe_chain(8, cross_safe_off_the_local_safe_chain)] #[tokio::test] - async fn lagging_delegated_safe_head_leaves_labels_unchanged( - #[case] el_has_block: bool, - #[case] finalized_at_target: bool, + async fn cross_safe_head_not_advancing_keeps_the_safe_label( + #[case] cross_safe_number: u64, + #[case] expect_lookups: fn(&RpcMock), ) { - let (block, cfg) = genesis_pinned_block(10); - let delegated_safe_head = block_info_of(&cfg, &block); - let mut state = TestEngineStateBuilder::new() - .with_unsafe_head(test_block_info(20)) - .with_safe_head(test_block_info(12)) - .with_finalized_head(target_or_older_block(finalized_at_target, delegated_safe_head)) - .build(); - let state_before = state; - let (client, l1, l2) = test_engine_client(cfg.clone()); - l2.expect_block_header_by_number(10, el_block(el_has_block, block)); - let sink = Arc::new(RecordingSink::default()); - let task = ConsolidateTask::new( - Arc::new(client), - cfg, - ConsolidateInput::BlockInfo(delegated_safe_head), - sink.clone(), - ); + let local_safe = engine_chain_block(10); + let mut state = engine_state_with_safe_labels(9, 7); + let expected = labels_after(&state, local_safe, state.sync_state.safe_head()); - task.execute(&mut state).await.unwrap(); + run_delegated(&mut state, local_safe, engine_chain_block(cross_safe_number), |l2| { + expect_lookups(l2); + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await + .unwrap(); - assert_eq!(state, state_before); - assert!(sink.0.lock().unwrap().is_empty()); - l1.assert_finished(); - l2.assert_finished(); + assert_eq!(state.sync_state, expected); } - /// A delegated safe head behind the safe head whose block differs from the execution layer's - /// means the delegate's chain diverged above it, so the engine still reorgs onto it, without - /// moving finalized. + /// A cross-safe head behind, or conflicting with, finality leaves the safe label without a + /// lookup, while the local-safe head from the same status still moves. + #[rstest::rstest] + #[case::behind_finalized(engine_chain_block(4))] + #[case::conflicting_with_finalized(other_chain_block(5))] #[tokio::test] - async fn diverged_delegated_safe_head_behind_safe_head_reorgs_the_engine() { - let (block, cfg) = genesis_pinned_block(10); - let delegated_safe_head = block_info_of(&cfg, &block); - let mut state = engine_state_with_safe_head(test_block_info(12)); - let finalized_before = state.sync_state.finalized_head(); - let expected_sync_state = state.sync_state.apply_update(EngineSyncStateUpdate { - local_safe_head: Some(delegated_safe_head), - safe_head: Some(delegated_safe_head), - unsafe_head: Some(delegated_safe_head), + async fn cross_safe_head_breaking_finality_keeps_only_the_safe_label( + #[case] cross_safe: L2BlockInfo, + ) { + let local_safe = engine_chain_block(10); + let mut state = engine_state_with_safe_labels(9, 7); + let expected = labels_after(&state, local_safe, state.sync_state.safe_head()); + + run_delegated(&mut state, local_safe, cross_safe, |l2| { + l2.expect_block_by_number(10, ElHas::EngineChain.block(10)); + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected); + } + + /// The execution layer has the lagging local-safe head at 10, and confirms the cross-safe head + /// at 9. + fn el_has_local_safe_10_and_cross_safe_9(l2: &RpcMock) { + l2.expect_block_header_by_number(10, ElHas::EngineChain.block(10)); + l2.expect_block_header_by_number(9, ElHas::EngineChain.block(9)); + } + + /// The execution layer lacks the lagging local-safe head at 10, so the cross-safe head is not + /// looked up. + fn el_missing_local_safe_10(l2: &RpcMock) { + l2.expect_block_header_by_number(10, ElHas::Missing.block(10)); + } + + /// A delegated local-safe head behind the engine's whose block the execution layer has, or + /// lacks, from a lagging delegate, leaves the local-safe and unsafe heads in place. Its + /// cross-safe head still moves the safe label when above it. + #[rstest::rstest] + #[case::el_has_local_safe(el_has_local_safe_10_and_cross_safe_9)] + #[case::el_missing_local_safe(el_missing_local_safe_10)] + #[tokio::test] + async fn lagging_delegate_moves_only_an_advancing_safe_label( + #[case] expect_lookups: fn(&RpcMock), + ) { + let (local_safe, cross_safe) = (engine_chain_block(10), engine_chain_block(9)); + let mut state = engine_state_with_safe_labels(12, 8); + let expected = state.sync_state.apply_update(EngineSyncStateUpdate { + safe_head: Some(cross_safe), ..Default::default() }); - let (client, l1, l2) = test_engine_client(cfg.clone()); - l2.expect_block_header_by_number(10, Some(other_chain_block(10))); - l2.expect_forkchoice_updated( - expected_sync_state.create_forkchoice_state(), - PayloadStatusEnum::Valid, - ); - let task = ConsolidateTask::new( - Arc::new(client), - cfg, - ConsolidateInput::BlockInfo(delegated_safe_head), - Arc::new(NoopBlockSink), - ); - task.execute(&mut state).await.unwrap(); + run_delegated(&mut state, local_safe, cross_safe, |l2| { + expect_lookups(l2); + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await + .unwrap(); - assert_eq!(state.sync_state, expected_sync_state); - assert_eq!(state.sync_state.finalized_head(), finalized_before); - l1.assert_finished(); - l2.assert_finished(); + assert_eq!(state.sync_state, expected); } - /// A delegated safe head behind the safe head is not judged against an execution layer that has - /// not applied the engine's last forkchoice, whether it is in its first sync or syncing again - /// after answering `SYNCING`: its canonical block there may predate the engine's labels, so no - /// lookup is made and the labels stay. + /// A delegate before interop, whose heads are the same block, that lags the engine leaves every + /// label where it is and sends no forkchoice update, whether or not the execution layer has the + /// block. #[rstest::rstest] - #[case::initial_sync(false, false)] - #[case::syncing_again(true, false)] + #[case::el_has_block(10, ElHas::EngineChain)] + #[case::el_missing_block(10, ElHas::Missing)] + #[case::at_finalized_head(5, ElHas::EngineChain)] #[tokio::test] - async fn delegated_safe_head_behind_safe_head_is_ignored_while_the_el_syncs( - #[case] el_sync_finished: bool, - #[case] el_applied_last_forkchoice: bool, + async fn lagging_delegate_before_interop_changes_nothing( + #[case] delegated_number: u64, + #[case] el_at_local_safe: ElHas, ) { - let (block, cfg) = genesis_pinned_block(10); - let delegated_safe_head = block_info_of(&cfg, &block); - let mut state = TestEngineStateBuilder::new() - .with_unsafe_head(test_block_info(20)) - .with_safe_head(test_block_info(12)) - .with_finalized_head(test_block_info(5)) - .with_el_sync_finished(el_sync_finished) - .with_el_applied_last_forkchoice(el_applied_last_forkchoice) - .build(); + let delegated = engine_chain_block(delegated_number); + let mut state = engine_state_with_safe_labels(12, 12); let state_before = state; - let (client, l1, l2) = test_engine_client(cfg.clone()); - let task = ConsolidateTask::new( - Arc::new(client), - cfg, - ConsolidateInput::BlockInfo(delegated_safe_head), - Arc::new(NoopBlockSink), - ); - task.execute(&mut state).await.unwrap(); + run_delegated(&mut state, delegated, delegated, |l2| { + l2.expect_block_header_by_number( + delegated_number, + el_at_local_safe.block(delegated_number), + ); + }) + .await + .unwrap(); assert_eq!(state, state_before); - l1.assert_finished(); - l2.assert_finished(); } - /// An empty block at `number` whose hash differs from [`genesis_pinned_block`]'s at the same - /// height, standing for a block on another chain. - fn other_chain_block(number: u64) -> RpcBlock { - RpcBlock::new( - RpcHeader::new(alloy_consensus::Header { number, timestamp: 1, ..Default::default() }), - BlockTransactions::Full(vec![]), - ) + /// A lagging delegate whose cross-safe head is behind the safe label leaves every label where + /// it is and sends no forkchoice update, without looking the cross-safe head up. + #[tokio::test] + async fn lagging_delegate_with_cross_safe_head_behind_the_safe_label_changes_nothing() { + let mut state = engine_state_with_safe_labels(12, 12); + let state_before = state; + + run_delegated(&mut state, engine_chain_block(10), engine_chain_block(7), |l2| { + l2.expect_block_header_by_number(10, ElHas::EngineChain.block(10)); + }) + .await + .unwrap(); + + assert_eq!(state, state_before); } - /// A reorg onto a diverged delegated safe head that the execution layer refuses as an invalid - /// forkchoice state resets the engine, which rebuilds the labels from the execution layer. + /// A lagging delegate whose cross-safe head conflicts with the finalized head leaves every + /// label where it is: the cross-safe head cannot become the safe label. #[tokio::test] - async fn rejected_reorg_onto_a_delegated_safe_head_resets_the_engine() { - let (block, cfg) = genesis_pinned_block(10); - let delegated_safe_head = block_info_of(&cfg, &block); - let mut state = engine_state_with_safe_head(test_block_info(12)); + async fn lagging_delegate_with_cross_safe_head_conflicting_with_finality_changes_nothing() { + let mut state = engine_state_with_safe_labels(12, 12); let state_before = state; - let (client, l1, l2) = test_engine_client(cfg.clone()); - l2.expect_block_header_by_number(10, Some(other_chain_block(10))); - l2.expect_error_code( - "engine_forkchoiceUpdatedV3", - i64::from(alloy_rpc_types_engine::INVALID_FORK_CHOICE_STATE_ERROR), - ); - let task = ConsolidateTask::new( - Arc::new(client), - cfg, - ConsolidateInput::BlockInfo(delegated_safe_head), - Arc::new(NoopBlockSink), - ); - let err = task.execute(&mut state).await.unwrap_err(); + run_delegated(&mut state, engine_chain_block(10), other_chain_block(5), |l2| { + l2.expect_block_header_by_number(10, ElHas::EngineChain.block(10)); + }) + .await + .unwrap(); - assert_eq!(err.severity(), crate::EngineTaskErrorSeverity::Reset, "got {err:?}"); assert_eq!(state, state_before); - l1.assert_finished(); - l2.assert_finished(); } - /// Labels with the unsafe head rewound below the safe head are moved back into order by any - /// delegated safe head above finality instead of being kept: at or above the unsafe head the - /// engine reorgs onto it, and below it the matching block consolidates. Keeping them would let - /// a later finalization above the unsafe head fail fatally. + /// A lagging delegate whose local-safe block the execution layer lacks cannot be judged against + /// the execution layer's canonical blocks, which may predate its sync, so a cross-safe head off + /// them is not looked up and reorgs nothing; not above the safe label, it changes nothing. + #[tokio::test] + async fn lagging_delegate_missing_from_the_el_never_reorgs() { + let mut state = engine_state_with_safe_labels(12, 12); + let state_before = state; + + run_delegated(&mut state, other_chain_block(10), other_chain_block(8), |l2| { + l2.expect_block_header_by_number(10, ElHas::Missing.block(10)); + }) + .await + .unwrap(); + + assert_eq!(state, state_before); + } + + /// A lagging delegate is not judged against an execution layer that has not applied the + /// engine's last forkchoice, even after its first sync finished: its canonical block at the + /// local-safe height may predate the engine's labels, so no lookup is made and a delegate on + /// another chain reorgs nothing. + #[tokio::test] + async fn lagging_delegate_is_not_judged_against_a_syncing_el() { + let mut state = engine_state_with_safe_labels(12, 12); + state.el_applied_last_forkchoice = false; + let state_before = state; + + run_delegated(&mut state, other_chain_block(10), other_chain_block(8), |_| {}) + .await + .unwrap(); + + assert_eq!(state, state_before); + } + + /// Lagging delegate: the execution layer has another block at its local-safe height. + fn lagging_local_safe_off_the_el_chain(l2: &RpcMock) { + l2.expect_block_header_by_number(10, ElHas::EngineChain.block(10)); + } + + /// Below the unsafe head: the execution layer has another block at the local-safe height. + fn local_safe_off_the_el_chain(l2: &RpcMock) { + l2.expect_block_by_number(18, ElHas::EngineChain.block(18)); + } + + /// At or above the unsafe head: the local-safe head is another block at the unsafe height. + const fn local_safe_off_the_unsafe_head(_: &RpcMock) {} + + /// Above the unsafe head but not its child: the local-safe head's ancestry is unknown, and the + /// execution layer does not hold the engine's safe label either. + fn local_safe_above_the_unsafe_head(l2: &RpcMock) { + l2.expect_block_header_by_number(12, ElHas::OtherChain.block(12)); + } + + /// Delegated heads on a chain that diverged from the engine's, which an execution-layer block + /// or the unsafe head at the local-safe height proves, or above the unsafe head without being + /// its child while the execution layer does not hold the safe label, reorg the engine onto + /// them: the unsafe and local-safe heads move to the local-safe head, and the safe label to the + /// cross-safe head even when lower, since the old safe block is on the abandoned chain. + /// Finalized stays. #[rstest::rstest] - #[case::at_or_above_unsafe_head(12, false)] - #[case::below_unsafe_head(8, true)] + #[case::lagging_local_safe(10, lagging_local_safe_off_the_el_chain)] + #[case::local_safe_below_unsafe(18, local_safe_off_the_el_chain)] + #[case::local_safe_at_unsafe(20, local_safe_off_the_unsafe_head)] + #[case::local_safe_above_unsafe(25, local_safe_above_the_unsafe_head)] #[tokio::test] - async fn delegated_safe_head_repairs_an_unsafe_head_below_the_safe_head( - #[case] delegated_number: u64, - #[case] is_below_unsafe_head: bool, + async fn delegated_heads_off_the_engine_chain_reorg_the_engine( + #[case] local_safe_number: u64, + #[case] expect_lookups: fn(&RpcMock), ) { - let (block, cfg) = genesis_pinned_block(delegated_number); - let delegated_safe_head = block_info_of(&cfg, &block); + let (local_safe, cross_safe) = (other_chain_block(local_safe_number), other_chain_block(8)); + let mut state = engine_state_with_safe_labels(12, 12); + let finalized_before = state.sync_state.finalized_head(); + let expected = labels_after_reorg(&state, local_safe, cross_safe); + + run_delegated(&mut state, local_safe, cross_safe, |l2| { + expect_lookups(l2); + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected); + assert_eq!(state.sync_state.finalized_head(), finalized_before); + } + + /// Delegated heads at or above the unsafe head on the engine's chain move the unsafe and + /// local-safe heads to the local-safe head, and a cross-safe head on that chain above the safe + /// label moves it. + #[rstest::rstest] + #[case::at_unsafe_head(engine_chain_block(20))] + #[case::child_of_unsafe_head(child_of_engine_unsafe_head())] + #[tokio::test] + async fn delegated_heads_extending_the_engine_chain_move_an_advancing_safe_label( + #[case] local_safe: L2BlockInfo, + ) { + let cross_safe = engine_chain_block(17); + let mut state = engine_state_with_safe_labels(15, 15); + let expected = labels_after_reorg(&state, local_safe, cross_safe); + + run_delegated(&mut state, local_safe, cross_safe, |l2| { + l2.expect_block_header_by_number(17, ElHas::EngineChain.block(17)); + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected); + } + + /// A safe label left on blocks a reorg abandoned does not prove the engine's chain: the + /// execution layer confirms the cross-safe head at the safe label's height, and the safe label + /// moves to it, keeping it an ancestor of the unsafe head. + #[tokio::test] + async fn cross_safe_head_replaces_a_stale_safe_label_at_its_height() { + let (local_safe, cross_safe) = (engine_chain_block(20), engine_chain_block(12)); let mut state = TestEngineStateBuilder::new() - .with_unsafe_head(test_block_info(10)) - .with_safe_head(test_block_info(15)) - .with_finalized_head(test_block_info(5)) + .with_unsafe_head(engine_chain_block(20)) + .with_local_safe_head(other_chain_block(12)) + .with_safe_head(other_chain_block(12)) + .with_finalized_head(engine_chain_block(5)) + .build(); + let expected = labels_after_reorg(&state, local_safe, cross_safe); + + run_delegated(&mut state, local_safe, cross_safe, |l2| { + l2.expect_block_header_by_number(12, ElHas::EngineChain.block(12)); + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected); + } + + /// While the execution layer has not finished syncing, its canonical blocks may be from before + /// the sync, so the cross-safe head is not checked against them and only advances the safe + /// label. + #[tokio::test] + async fn cross_safe_head_is_not_judged_against_a_syncing_el() { + let (local_safe, cross_safe) = (engine_chain_block(20), engine_chain_block(17)); + let mut state = TestEngineStateBuilder::new() + .with_unsafe_head(engine_chain_block(20)) + .with_safe_head(engine_chain_block(15)) + .with_finalized_head(engine_chain_block(5)) + .with_el_sync_finished(false) .build(); - let expected_sync_state = state.sync_state.apply_update(EngineSyncStateUpdate { - local_safe_head: Some(delegated_safe_head), - safe_head: Some(delegated_safe_head), - unsafe_head: (!is_below_unsafe_head).then_some(delegated_safe_head), + let expected = labels_after_reorg(&state, local_safe, cross_safe); + + run_delegated(&mut state, local_safe, cross_safe, |l2| { + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected); + } + + /// The engine chain's block at 21, the child of the engine's unsafe head at 20. + fn child_of_engine_unsafe_head() -> L2BlockInfo { + L2BlockInfo { + block_info: BlockInfo { + parent_hash: engine_chain_block(20).block_info.hash, + ..engine_chain_block(21).block_info + }, + ..Default::default() + } + } + + /// A delegated local-safe head that is the unsafe head's child extends the engine's chain, so + /// a cross-safe head below the safe label keeps it. + #[tokio::test] + async fn child_of_the_unsafe_head_keeps_the_safe_label() { + let local_safe = child_of_engine_unsafe_head(); + let mut state = engine_state_with_safe_labels(15, 15); + let expected = state.sync_state.apply_update(EngineSyncStateUpdate { + local_safe_head: Some(local_safe), + unsafe_head: Some(local_safe), ..Default::default() }); - let (client, l1, l2) = test_engine_client(cfg.clone()); - if is_below_unsafe_head { - l2.expect_params( - "eth_getBlockByNumber", - serde_json::json!([format!("{delegated_number:#x}"), true]), - block, + + run_delegated(&mut state, local_safe, engine_chain_block(12), |l2| { + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, ); - } - l2.expect_forkchoice_updated( - expected_sync_state.create_forkchoice_state(), - PayloadStatusEnum::Valid, + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected); + } + + /// A reorg whose cross-safe head is below the finalized head moves the safe label to the + /// finalized head, which the delegate's chain shares with the engine's. + #[tokio::test] + async fn reorg_with_cross_safe_head_behind_finality_moves_safe_to_finalized() { + let local_safe = other_chain_block(20); + let mut state = engine_state_with_safe_labels(15, 15); + let expected = labels_after_reorg(&state, local_safe, state.sync_state.finalized_head()); + + run_delegated(&mut state, local_safe, other_chain_block(4), |l2| { + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected); + } + + /// Delegated heads at the unsafe head on the engine's chain, as when the engine has no unsafe + /// gossip, keep the safe label for a cross-safe head at or below it, so a lagging backend + /// behind a load balancer does not move safe back. + #[rstest::rstest] + #[case::at_safe_label(15, no_lookup)] + #[case::behind_safe_label(12, no_lookup)] + #[tokio::test] + async fn delegated_heads_at_the_unsafe_head_keep_the_safe_label( + #[case] cross_safe_number: u64, + #[case] expect_cross_safe_lookup: fn(&RpcMock), + ) { + let local_safe = engine_chain_block(20); + let mut state = engine_state_with_safe_labels(15, 15); + let expected = state.sync_state.apply_update(EngineSyncStateUpdate { + local_safe_head: Some(local_safe), + unsafe_head: Some(local_safe), + ..Default::default() + }); + + run_delegated(&mut state, local_safe, engine_chain_block(cross_safe_number), |l2| { + expect_cross_safe_lookup(l2); + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected); + } + + /// A cross-safe head at one of the engine's labels is not looked up. + const fn no_lookup(_: &RpcMock) {} + + /// The execution layer has the engine chain's block at 15, which the safe label alone cannot + /// prove, since it can stay on blocks a reorg abandoned. + fn cross_safe_15_on_the_engine_chain(l2: &RpcMock) { + l2.expect_block_header_by_number(15, ElHas::EngineChain.block(15)); + } + + fn expect_invalid_forkchoice_state(l2: &RpcMock) { + l2.expect_error_code( + "engine_forkchoiceUpdatedV3", + i64::from(alloy_rpc_types_engine::INVALID_FORK_CHOICE_STATE_ERROR), ); + } + + /// A forkchoice update for delegated safe heads, whether a reorg off the engine's chain or an + /// extension of it, that the execution layer refuses as an invalid forkchoice state resets the + /// engine, which rebuilds the labels from the execution layer. + #[rstest::rstest] + #[case::reorg(other_chain_block(20))] + #[case::extension(engine_chain_block(25))] + #[tokio::test] + async fn rejected_forkchoice_for_delegated_heads_resets_the_engine( + #[case] delegated: L2BlockInfo, + ) { + let mut state = engine_state_with_safe_labels(15, 15); + let state_before = state; + + let err = run_delegated(&mut state, delegated, delegated, expect_invalid_forkchoice_state) + .await + .unwrap_err(); + + assert_eq!(err.severity(), EngineTaskErrorSeverity::Reset, "got {err:?}"); + assert_eq!(state, state_before); + } + + /// A reorg whose cross-safe head breaks finality is skipped before the execution layer is + /// asked: the cross-safe head cannot become the safe label, and the old safe label would sit + /// above the new unsafe head. + #[tokio::test] + async fn reorg_with_cross_safe_head_breaking_finality_is_skipped() { + let mut state = engine_state_with_safe_labels(15, 15); + let state_before = state; + + let err = run_delegated(&mut state, other_chain_block(20), other_chain_block(5), |_| {}) + .await + .unwrap_err(); + + assert!(matches!(err, ConsolidateTaskError::BreaksFinality(_)), "got {err:?}"); + assert_eq!(err.severity(), crate::EngineTaskErrorSeverity::Skip); + assert_eq!(state, state_before); + } + + /// While the execution layer syncs, its canonical blocks are still those from before the sync, + /// so a cross-safe head is not compared with them: it only advances the safe label, and never + /// reorgs the engine or retargets the sync. + #[rstest::rstest] + #[case::other_chain_below_safe_label(other_chain_block(12), engine_chain_block(15))] + #[case::advancing(engine_chain_block(16), engine_chain_block(16))] + #[tokio::test] + async fn el_sync_moves_the_safe_label_only_forward_without_lookups( + #[case] cross_safe: L2BlockInfo, + #[case] expected_safe: L2BlockInfo, + ) { + let local_safe = engine_chain_block(18); + let mut state = engine_state_with_safe_labels(15, 15); + let expected = labels_after(&state, local_safe, expected_safe); + + run_delegated(&mut state, local_safe, cross_safe, |l2| { + l2.expect_block_by_number(18, ElHas::Missing.block(18)); + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected); + } + + /// A cross-safe head that is the engine's safe label leaves it in place without asking the + /// execution layer. One that differs from the safe label at its height is checked against the + /// execution layer, which has the engine chain's block there, so the safe label stays too. + #[rstest::rstest] + #[case::at_safe_label(engine_chain_block(15), no_lookup)] + #[case::conflicting_with_safe_label(other_chain_block(15), cross_safe_15_on_the_engine_chain)] + #[tokio::test] + async fn cross_safe_head_off_the_engine_chain_keeps_the_safe_label( + #[case] cross_safe: L2BlockInfo, + #[case] expect_cross_safe_lookup: fn(&RpcMock), + ) { + let local_safe = engine_chain_block(18); + let mut state = engine_state_with_safe_labels(15, 15); + let expected = labels_after(&state, local_safe, engine_chain_block(15)); + + run_delegated(&mut state, local_safe, cross_safe, |l2| { + l2.expect_block_by_number(18, ElHas::EngineChain.block(18)); + expect_cross_safe_lookup(l2); + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected); + } + + /// A delegated local-safe head whose block the execution layer has, but whose info cannot be + /// built from it, is consolidated with the delegate's info instead of reorging the unsafe head + /// back to it. + #[tokio::test] + async fn undecodable_delegated_block_is_consolidated_without_a_reorg() { + let (block, local_safe) = chain_block(18, ENGINE_CHAIN); + let cfg = Arc::new(RollupConfig::default()); + let mut state = engine_state_with_safe_labels(15, 15); + let expected = labels_after(&state, local_safe, local_safe); + let (client, l1, l2) = test_engine_client(cfg.clone()); + l2.expect_block_by_number(18, Some(block)); + l2.expect_forkchoice_updated(expected.create_forkchoice_state(), PayloadStatusEnum::Valid); let task = ConsolidateTask::new( Arc::new(client), cfg, - ConsolidateInput::BlockInfo(delegated_safe_head), + delegated_safe_heads(local_safe, local_safe), Arc::new(NoopBlockSink), ); task.execute(&mut state).await.unwrap(); - assert_eq!(state.sync_state, expected_sync_state); - assert!( - state.sync_state.unsafe_head().block_info.number >= - state.sync_state.safe_head().block_info.number - ); + assert_eq!(state.sync_state, expected); l1.assert_finished(); l2.assert_finished(); } - /// A delegated safe head whose block differs from the execution layer's is not treated as a - /// divergence while the execution layer has not applied the engine's last forkchoice: only the - /// safe labels move to it, and the unsafe head stays the sync target. + /// A delegated block that differs from the execution layer's is not a divergence while the + /// execution layer has not applied the engine's last forkchoice: only the safe labels move, the + /// cross-safe head only advancing, and the unsafe head stays the sync target. #[tokio::test] - async fn mismatched_delegated_safe_head_does_not_reorg_while_the_el_syncs() { - let (block, cfg) = genesis_pinned_block(15); - let delegated_safe_head = block_info_of(&cfg, &block); + async fn delegated_block_off_a_syncing_el_moves_only_the_safe_labels() { + let (local_safe, cross_safe) = (engine_chain_block(18), engine_chain_block(17)); + let mut state = engine_state_with_safe_labels(15, 15); + state.el_applied_last_forkchoice = false; + let expected = labels_after(&state, local_safe, cross_safe); + + run_delegated(&mut state, local_safe, cross_safe, |l2| { + l2.expect_block_by_number(18, ElHas::OtherChain.block(18)); + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Syncing, + ); + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected); + } + + /// A delegate catching up several blocks above the unsafe head keeps the engine's safe label + /// when the execution layer holds it, rather than dropping it to a lower cross-safe head; only + /// a safe label the execution layer does not hold falls to the cross-safe head. + #[rstest::rstest] + #[case::el_holds_safe_label(ElHas::EngineChain, 15)] + #[case::el_off_safe_label(ElHas::OtherChain, 12)] + #[tokio::test] + async fn catch_up_above_the_unsafe_head_keeps_a_safe_label_the_el_holds( + #[case] el_at_safe_label: ElHas, + #[case] expected_safe: u64, + ) { + let (local_safe, cross_safe) = (engine_chain_block(23), engine_chain_block(12)); + let mut state = engine_state_with_safe_labels(15, 15); + let expected = state.sync_state.apply_update(EngineSyncStateUpdate { + local_safe_head: Some(local_safe), + safe_head: Some(engine_chain_block(expected_safe)), + unsafe_head: Some(local_safe), + ..Default::default() + }); + + run_delegated(&mut state, local_safe, cross_safe, |l2| { + l2.expect_block_header_by_number(15, el_at_safe_label.block(15)); + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await + .unwrap(); + + assert_eq!(state.sync_state, expected); + } + + /// Labels with the unsafe head rewound below the safe labels are moved back into order by + /// delegated heads above finality instead of being kept as a lagging delegate's; the execution + /// layer is not asked about heights above the rewound unsafe head. + #[tokio::test] + async fn delegated_heads_repair_an_unsafe_head_below_the_safe_labels() { + let (local_safe, cross_safe) = (engine_chain_block(12), engine_chain_block(12)); let mut state = TestEngineStateBuilder::new() - .with_unsafe_head(test_block_info(20)) - .with_safe_head(test_block_info(12)) - .with_finalized_head(test_block_info(5)) - .with_el_applied_last_forkchoice(false) + .with_unsafe_head(engine_chain_block(10)) + .with_safe_head(engine_chain_block(15)) + .with_finalized_head(engine_chain_block(5)) .build(); - let expected_sync_state = with_safe_head(&state, delegated_safe_head); - let (client, l1, l2) = test_engine_client(cfg.clone()); - l2.expect_params( - "eth_getBlockByNumber", - serde_json::json!(["0xf", true]), - other_chain_block(15), - ); - l2.expect_forkchoice_updated( - expected_sync_state.create_forkchoice_state(), - PayloadStatusEnum::Syncing, - ); - let task = ConsolidateTask::new( - Arc::new(client), - cfg, - ConsolidateInput::BlockInfo(delegated_safe_head), - Arc::new(NoopBlockSink), - ); + let expected = labels_after_reorg(&state, local_safe, cross_safe); - task.execute(&mut state).await.unwrap(); + run_delegated(&mut state, local_safe, cross_safe, |l2| { + l2.expect_forkchoice_updated( + expected.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + }) + .await + .unwrap(); - assert_eq!(state.sync_state, expected_sync_state); - l1.assert_finished(); - l2.assert_finished(); + assert_eq!(state.sync_state, expected); } } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/mod.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/mod.rs index 9eddaa76d00..12ae6c8ba16 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/mod.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/mod.rs @@ -25,7 +25,10 @@ mod canonicalize; pub use canonicalize::{CanonicalizeTask, CanonicalizeTaskError}; mod consolidate; -pub use consolidate::{ConsolidateInput, ConsolidateTask, ConsolidateTaskError}; +pub use consolidate::{ + ConsolidateInput, ConsolidateTask, ConsolidateTaskError, DelegatedSafeHeads, + InconsistentSafeHeadsError, +}; mod finalize; pub use finalize::{FinalizeBlockId, FinalizeTask, FinalizeTaskError}; diff --git a/rust/kona/crates/node/engine/src/test_utils/engine_state.rs b/rust/kona/crates/node/engine/src/test_utils/engine_state.rs index 1ed3c5b8178..a1399be192e 100644 --- a/rust/kona/crates/node/engine/src/test_utils/engine_state.rs +++ b/rust/kona/crates/node/engine/src/test_utils/engine_state.rs @@ -3,7 +3,10 @@ use alloy_eips::BlockNumHash; use alloy_primitives::{B256, b256}; use kona_protocol::{BlockInfo, L2BlockInfo}; -/// Builder for creating test `EngineState` instances with sensible defaults +/// Builder for creating test `EngineState` instances with sensible defaults. +/// +/// An unset local-safe head equals the safe head, as on every chain before interop; tests of the +/// local-safe and safe labels apart set it with [`Self::with_local_safe_head`]. #[derive(Debug)] pub struct TestEngineStateBuilder { unsafe_head: L2BlockInfo, @@ -45,6 +48,12 @@ impl TestEngineStateBuilder { self } + /// Sets the local-safe head, which otherwise follows the safe head + pub const fn with_local_safe_head(mut self, block: L2BlockInfo) -> Self { + self.local_safe_head = Some(block); + self + } + /// Sets the safe head pub const fn with_safe_head(mut self, block: L2BlockInfo) -> Self { self.safe_head = Some(block); @@ -80,7 +89,9 @@ impl TestEngineStateBuilder { // Set unsafe head (required) state.sync_state = state.sync_state.apply_update(EngineSyncStateUpdate { unsafe_head: Some(self.unsafe_head), - local_safe_head: Some(self.local_safe_head.unwrap_or(self.unsafe_head)), + local_safe_head: Some( + self.local_safe_head.or(self.safe_head).unwrap_or(self.unsafe_head), + ), safe_head: Some(self.safe_head.unwrap_or(self.unsafe_head)), finalized_head: Some(self.finalized_head.unwrap_or(self.unsafe_head)), }); diff --git a/rust/kona/crates/node/engine/src/test_utils/misc.rs b/rust/kona/crates/node/engine/src/test_utils/misc.rs index fd77bf7ccf4..c7675444b0d 100644 --- a/rust/kona/crates/node/engine/src/test_utils/misc.rs +++ b/rust/kona/crates/node/engine/src/test_utils/misc.rs @@ -1,3 +1,4 @@ +use crate::{ConsolidateInput, DelegatedSafeHeads}; use alloy_eips::BlockNumHash; use alloy_primitives::B256; use kona_protocol::{BlockInfo, L2BlockInfo}; @@ -15,3 +16,14 @@ pub fn test_block_info(number: u64) -> L2BlockInfo { seq_num: 0, } } + +/// Delegated safe heads as a consolidation input. Pass the same block twice for the heads of a +/// delegate before interop. +pub fn delegated_safe_heads( + local_safe_head: L2BlockInfo, + safe_head: L2BlockInfo, +) -> ConsolidateInput { + DelegatedSafeHeads::try_new(local_safe_head, safe_head) + .expect("the cross-safe head must be at or below the local-safe head, on its chain") + .into() +} diff --git a/rust/kona/crates/node/engine/src/test_utils/mod.rs b/rust/kona/crates/node/engine/src/test_utils/mod.rs index dd8c55020bd..397a57b40c3 100644 --- a/rust/kona/crates/node/engine/src/test_utils/mod.rs +++ b/rust/kona/crates/node/engine/src/test_utils/mod.rs @@ -14,4 +14,4 @@ mod engine_state; pub use engine_state::TestEngineStateBuilder; mod misc; -pub use misc::test_block_info; +pub use misc::{delegated_safe_heads, test_block_info}; diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs index 37dc0e00185..b3d64c7202b 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs @@ -6,7 +6,9 @@ use alloy_primitives::BlockHash; use async_trait::async_trait; use jsonrpsee::core::ClientError; use kona_derive::{ChainProvider, Signal}; -use kona_engine::{EngineState, FinalityViolation, FinalizeBlockId}; +use kona_engine::{ + DelegatedSafeHeads, EngineState, FinalityViolation, FinalizeBlockId, InconsistentSafeHeadsError, +}; use kona_protocol::{BlockInfo, L2BlockInfo, SyncStatus}; use kona_rpc::RollupNodeApiClient; use thiserror::Error; @@ -48,6 +50,14 @@ impl SyncStatusSource for C { /// /// Once validated, the actor sends the derived safe and finalized L2 info /// to the [`NodeActor`] responsible for the execution sub-routine. +/// +/// The delegate's local-safe and cross-safe heads are imported +/// separately: the local-safe head is the block the execution layer must have, or sync to, and +/// sets the engine's local-safe label; the cross-safe head sets the safe label that forkchoice +/// updates carry, and finalization never goes above it. Before interop both are the same block. +/// Under interop the delegate must verify cross-chain dependencies, for example op-node with a +/// supervisor, since this node checks none itself: its safe label is only as safe as the +/// delegate's cross-safe head. #[derive(Debug)] pub struct DelegateDerivationActor where @@ -78,11 +88,11 @@ where /// Ticker driving periodic polls of the derivation delegate. delegated_derivation_ticker: time::Interval, - /// A safe head behind the engine's that was forwarded while the execution layer applied the - /// engine's forkchoice, with the engine's safe head at the time. The engine checks such a head - /// against its execution layer once; until either changes, sending it again repeats that - /// lookup to the same answer. - forwarded_lagging_safe_head: Option<(BlockNumHash, BlockNumHash)>, + /// Safe heads behind the engine's that were forwarded while the execution layer applied the + /// engine's forkchoice, with the engine's local-safe and safe heads at the time. The engine + /// checks such heads against its execution layer once; until either side changes, sending + /// them again repeats that lookup to the same answer. + forwarded_lagging_safe_heads: Option<(DelegatedSafeHeads, BlockNumHash, BlockNumHash)>, /// The last rejection of a delegate status, so a delegate stuck on one inconsistency is /// warned about once rather than on every poll. last_rejection: Option, @@ -116,7 +126,7 @@ where engine_l2_safe_head: L2BlockInfo::default(), has_engine_sync_completed: false, delegated_derivation_ticker, - forwarded_lagging_safe_head: None, + forwarded_lagging_safe_heads: None, last_rejection: None, } } @@ -186,31 +196,34 @@ where Ok(()) } + /// The delegate's local-safe head, or its safe head when it reports a zero local-safe head: + /// `SyncStatus` zeroes a label the delegate has not derived, and a zero head carries nothing. + fn reported_local_safe_head(status: &SyncStatus) -> L2BlockInfo { + if status.local_safe_l2 == L2BlockInfo::default() { + status.safe_l2 + } else { + status.local_safe_l2 + } + } + /// Rejects a status whose labels are out of order: its finalized block ahead of its safe head, - /// or its safe head ahead of its local safe head. No consistent delegate reports either, and a - /// finalized block ahead of the safe head would let a stale safe head sit below finality. A - /// zero local safe head is a label the delegate has not derived or does not report, so it does - /// not order anything. - fn validate_label_order(status: &SyncStatus) -> Result<(), DerivationDelegationError> { + /// or its safe head ahead of its local safe head or a different block at its height. No + /// consistent delegate reports any of these, and a finalized block ahead of the safe head would + /// let a stale safe head sit below finality. Returns its safe heads otherwise. + fn validate_label_order( + status: &SyncStatus, + ) -> Result { let finalized = status.finalized_l2.block_info.number; let safe = status.safe_l2.block_info.number; - let local_safe = if status.local_safe_l2 == L2BlockInfo::default() { - safe - } else { - status.local_safe_l2.block_info.number - }; if finalized > safe { return Err(DerivationDelegationError::FinalizedAheadOfSafe { finalized, safe }); } - if safe > local_safe { - return Err(DerivationDelegationError::SafeAheadOfLocalSafe { local_safe, safe }); - } - Ok(()) + Ok(DelegatedSafeHeads::try_new(Self::reported_local_safe_head(status), status.safe_l2)?) } /// Verifies that the labels reported by the derivation delegate are in order, that its /// finalized L2 block does not move back from the engine's, and that its L1 info is consistent - /// with the canonical L1 chain. + /// with the canonical L1 chain. Returns its safe heads when it passes. /// /// Only a fork at the engine's finalized height is caught here. A chain that forked below it /// and reports a higher finalized block passes, and the execution layer rejects the forkchoice @@ -218,14 +231,16 @@ where async fn validate_sync_status( &mut self, v: &SyncStatus, - ) -> Result<(), DerivationDelegationError> { - Self::validate_label_order(v)?; + ) -> Result { + let safe_heads = Self::validate_label_order(v)?; self.engine_state_rx .borrow() .sync_state .ensure_respects_finality(v.finalized_l2.block_info.id())?; + let local_safe = safe_heads.local_safe_head(); let checks = [ + ("L1 Origin of Local Safe L2", local_safe.l1_origin.number, local_safe.l1_origin.hash), ("L1 Origin of Safe L2", v.safe_l2.l1_origin.number, v.safe_l2.l1_origin.hash), ( "L1 Origin of Finalized L2", @@ -243,7 +258,7 @@ where self.validate_l1_block(context, number, hash).await?; validated_l1_blocks.push((number, hash)); } - Ok(()) + Ok(safe_heads) } /// Logs a rejected delegate status at `warn!` when the reason changed, and at `debug!` while @@ -258,28 +273,23 @@ where } } - /// Whether the delegated safe head is the lagging one already forwarded against the same - /// engine safe head, so the engine would repeat its lookup to the same answer. - fn is_lagging_safe_head_forwarded(&self, safe_l2: &L2BlockInfo) -> bool { - self.lagging_safe_head_to_remember(safe_l2) == self.forwarded_lagging_safe_head && - self.forwarded_lagging_safe_head.is_some() - } - - /// The pair to remember for a delegated safe head behind the engine's consistent safe head - /// while its execution layer applied the engine's forkchoice, or `None` for any other head, - /// which the engine must always see. - fn lagging_safe_head_to_remember( + /// What to remember for safe heads behind the engine's consistent local-safe head while its + /// execution layer applied the engine's forkchoice, or `None` for any other heads, which the + /// engine must always see. + fn lagging_safe_heads_to_remember( &self, - safe_l2: &L2BlockInfo, - ) -> Option<(BlockNumHash, BlockNumHash)> { + safe_heads: DelegatedSafeHeads, + ) -> Option<(DelegatedSafeHeads, BlockNumHash, BlockNumHash)> { let engine_state = self.engine_state_rx.borrow(); - let engine_safe_head = engine_state.sync_state.safe_head().block_info; - let is_consistent = - engine_state.sync_state.unsafe_head().block_info.number >= engine_safe_head.number; + let labels = &engine_state.sync_state; + let engine_local_safe = labels.local_safe_head().block_info; + let engine_safe = labels.safe_head().block_info; + let are_labels_consistent = labels.unsafe_head().block_info.number >= engine_safe.number && + labels.unsafe_head().block_info.number >= engine_local_safe.number; (engine_state.el_applied_last_forkchoice && - is_consistent && - safe_l2.block_info.number < engine_safe_head.number) - .then(|| (safe_l2.block_info.id(), engine_safe_head.id())) + are_labels_consistent && + safe_heads.local_safe_head().block_info.number < engine_local_safe.number) + .then(|| (safe_heads, engine_local_safe.id(), engine_safe.id())) } /// Fetches, validates, and applies sync status from the derivation delegate. @@ -296,27 +306,30 @@ where } }; - if let Err(err) = self.validate_sync_status(&sync_status).await { - // Validation failures here are expected to be transient, so we skip processing - // this sync status and continue delegating derivation instead of treating it as - // fatal. - self.log_rejected_status(&err); - return Ok(()); - } + let safe_heads = match self.validate_sync_status(&sync_status).await { + Ok(safe_heads) => safe_heads, + Err(err) => { + // Validation failures here are expected to be transient, so we skip processing + // this sync status and continue delegating derivation instead of treating it as + // fatal. + self.log_rejected_status(&err); + return Ok(()); + } + }; self.last_rejection = None; self.derivation_origin_tx.send_replace(Some(sync_status.current_l1)); - if self.is_lagging_safe_head_forwarded(&sync_status.safe_l2) { + let lagging_safe_heads = self.lagging_safe_heads_to_remember(safe_heads); + if lagging_safe_heads.is_some() && lagging_safe_heads == self.forwarded_lagging_safe_heads { debug!( target: "derivation", - safe_l2 = ?sync_status.safe_l2, - "Not forwarding a lagging safe head the engine already checked" + ?safe_heads, + "Not forwarding lagging safe heads the engine already checked" ); } else { - self.forwarded_lagging_safe_head = - self.lagging_safe_head_to_remember(&sync_status.safe_l2); + self.forwarded_lagging_safe_heads = lagging_safe_heads; self.engine_client - .send_safe_l2_signal(sync_status.safe_l2.into()) + .send_safe_l2_signal(safe_heads.into()) .await .map_err(|e| DerivationError::Sender(Box::new(e)))?; } @@ -333,6 +346,7 @@ where debug!( target: "derivation", + local_safe_l2 = ?sync_status.local_safe_l2, safe_l2 = ?sync_status.safe_l2, finalized_l2 = ?sync_status.finalized_l2, "Processed sync status from delegate" @@ -381,6 +395,10 @@ enum DerivationDelegationError { #[error("Finalized L2 moves the engine's finality back: {0}")] FinalizedL2BreaksFinality(#[from] FinalityViolation), + /// The delegate's safe heads are inconsistent with each other. + #[error(transparent)] + InconsistentSafeHeads(#[from] InconsistentSafeHeadsError), + /// The L1 provider returned an error (network, RPC, etc.) #[error("L1 provider error: {0}")] L1Provider(String), @@ -388,8 +406,4 @@ enum DerivationDelegationError { /// The hash provided by the derivation delegation does not match the canonical chain. #[error("L1 inconsistency in {context} at block {number}: expected {expected}, got {actual}")] L1ValidationFailed { context: String, number: u64, expected: BlockHash, actual: BlockHash }, - - /// The delegate's safe head is ahead of its own local safe head. - #[error("Safe L2 {safe} is ahead of local safe L2 {local_safe}")] - SafeAheadOfLocalSafe { local_safe: u64, safe: u64 }, } diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs index b586f878488..738f6d4397b 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs @@ -5,7 +5,8 @@ use alloy_primitives::B256; use async_trait::async_trait; use kona_derive::test_utils::TestChainProvider; use kona_engine::{ - ConsolidateInput, EngineState, EngineSyncState, EngineSyncStateUpdate, FinalizeBlockId, + ConsolidateInput, DelegatedSafeHeads, EngineState, EngineSyncState, EngineSyncStateUpdate, + FinalizeBlockId, }; use kona_protocol::{BlockInfo, L2BlockInfo, SyncStatus}; use rstest::rstest; @@ -108,25 +109,27 @@ impl Harness { enum Forwarded { Finalized(FinalizeBlockId), Other(String), - Safe(L2BlockInfo), + Safe(Box), } impl From for Forwarded { fn from(request: EngineActorRequest) -> Self { match request { EngineActorRequest::ProcessFinalizedL2Block(finalized) => Self::Finalized(*finalized), - EngineActorRequest::ProcessSafeL2Signal(ConsolidateInput::BlockInfo(safe)) => { - Self::Safe(safe) + EngineActorRequest::ProcessSafeL2Signal(ConsolidateInput::Delegated(heads)) => { + Self::Safe(heads) } other => Self::Other(format!("{other:?}")), } } } -/// What one poll of `status` forwards: the safe head, then the finalized block pinned by hash. +/// What one poll of `status` forwards: the safe heads, then the finalized block pinned by hash. fn forwarded_for(status: &SyncStatus) -> Vec { vec![ - Forwarded::Safe(status.safe_l2), + Forwarded::Safe(Box::new( + DelegatedSafeHeads::try_new(status.local_safe_l2, status.safe_l2).unwrap(), + )), Forwarded::Finalized(FinalizeBlockId::ByHash(status.finalized_l2.block_info.id())), ] } @@ -270,6 +273,7 @@ enum DelegateReply { CurrentL1NotCanonical, FetchFailed, FinalizedOriginNotCanonical, + LocalSafeOriginNotCanonical, SafeOriginNotCanonical, Valid, } @@ -287,6 +291,9 @@ impl DelegateReply { Self::FinalizedOriginNotCanonical => { status.finalized_l2.l1_origin = non_canonical(status.finalized_l2.l1_origin) } + Self::LocalSafeOriginNotCanonical => { + status.local_safe_l2.l1_origin = non_canonical(status.local_safe_l2.l1_origin) + } Self::SafeOriginNotCanonical => { status.safe_l2.l1_origin = non_canonical(status.safe_l2.l1_origin) } @@ -315,6 +322,7 @@ async fn polls_wait_for_engine_sync_completion() { #[rstest] #[case::valid(DelegateReply::Valid, true)] #[case::fetch_failed(DelegateReply::FetchFailed, false)] +#[case::local_safe_origin_not_canonical(DelegateReply::LocalSafeOriginNotCanonical, false)] #[case::safe_origin_not_canonical(DelegateReply::SafeOriginNotCanonical, false)] #[case::finalized_origin_not_canonical(DelegateReply::FinalizedOriginNotCanonical, false)] #[case::current_l1_not_canonical(DelegateReply::CurrentL1NotCanonical, false)] @@ -408,6 +416,33 @@ async fn poll_rejects_status_with_labels_out_of_order( assert_eq!(harness.current_l1.borrow().is_some(), accepted); } +/// The delegate's local-safe and cross-safe heads are forwarded as separate labels, so a cross-safe +/// head that lags the local-safe head under interop is not raised to it, and before interop both +/// carry the same block. +#[rstest] +#[case::pre_interop(20, 20)] +#[case::cross_safe_lags_local_safe(18, 20)] +#[tokio::test(start_paused = true)] +async fn poll_forwards_local_safe_and_cross_safe_heads_separately( + #[case] safe: u64, + #[case] local_safe: u64, +) { + let status = SyncStatus { + local_safe_l2: l2_block(local_safe, SAFE_ORIGIN), + safe_l2: l2_block(safe, SAFE_ORIGIN), + ..delegate_status() + }; + let mut harness = Harness::new([Ok(status.clone())]); + harness.complete_engine_sync().await; + + harness.actor.step().await.unwrap(); + + let expected_heads = + DelegatedSafeHeads::try_new(l2_block(local_safe, SAFE_ORIGIN), l2_block(safe, SAFE_ORIGIN)) + .unwrap(); + assert_eq!(harness.forwarded()[0], Forwarded::Safe(Box::new(expected_heads))); +} + /// A zero local safe head is a label the delegate has not derived or does not report, so it does /// not put the safe head out of order and the status is forwarded. #[tokio::test(start_paused = true)] @@ -418,7 +453,8 @@ async fn unreported_local_safe_head_does_not_order_the_labels() { harness.actor.step().await.unwrap(); - assert_eq!(harness.forwarded(), forwarded_for(&status)); + let reported_as_safe = SyncStatus { local_safe_l2: status.safe_l2, ..status }; + assert_eq!(harness.forwarded(), forwarded_for(&reported_as_safe)); } /// An engine sync state with consistent labels: unsafe at 30, safe at `safe`, finalized at 15. @@ -436,7 +472,7 @@ fn finalized_only(status: &SyncStatus) -> Vec { vec![Forwarded::Finalized(FinalizeBlockId::ByHash(status.finalized_l2.block_info.id()))] } -/// A safe head behind the engine's is forwarded once while the execution layer applied the +/// Safe heads behind the engine's are forwarded once while the execution layer applied the /// engine's forkchoice, since the engine's answer to it cannot change; finalized still goes out /// on every poll. A moved engine safe head, or an execution layer that did not apply the last /// forkchoice, sends it again. @@ -445,7 +481,7 @@ fn finalized_only(status: &SyncStatus) -> Vec { #[case::el_not_applied(false, 25, forwarded_for)] #[case::engine_safe_moved(true, 26, forwarded_for)] #[tokio::test(start_paused = true)] -async fn lagging_safe_head_is_forwarded_once_per_engine_view( +async fn lagging_safe_heads_are_forwarded_once_per_engine_view( #[case] el_applied_last_forkchoice: bool, #[case] engine_safe_on_second_poll: u64, #[case] second_poll_forwards: fn(&SyncStatus) -> Vec, diff --git a/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs b/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs index c7399efc6e6..c6eda0d9f38 100644 --- a/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs +++ b/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs @@ -5,11 +5,10 @@ use alloy_rpc_types_engine::{ ExecutionPayloadV1, ForkchoiceState, PayloadStatus, PayloadStatusEnum, }; use kona_engine::{ - ConsolidateInput, EngineState, EngineSyncStateUpdate, FinalizeBlockId, NoopBlockSink, - ResetRequest, + EngineState, EngineSyncStateUpdate, FinalizeBlockId, NoopBlockSink, ResetRequest, test_utils::{ - RpcMock, TestEngineStateBuilder, TestRpcBlock, expect_sync_start, test_block_info, - test_engine_client, test_l2_block, test_rpc_block, + RpcMock, TestEngineStateBuilder, TestRpcBlock, delegated_safe_heads, expect_sync_start, + test_block_info, test_engine_client, test_l2_block, test_rpc_block, }, }; use op_alloy_consensus::{OpBlock, OpTxEnvelope}; @@ -118,7 +117,8 @@ async fn delegated_safe_head_is_sent_on_every_poll() { // accepts the next poll. for _ in 0..3 { requests - .send(EngineActorRequest::ProcessSafeL2Signal(ConsolidateInput::BlockInfo( + .send(EngineActorRequest::ProcessSafeL2Signal(delegated_safe_heads( + delegated_safe_head, delegated_safe_head, ))) .await @@ -143,6 +143,7 @@ async fn delegated_poll_during_el_sync_moves_safe_labels_and_skips_finalization( let (client, l1, l2) = test_engine_client(config.clone()); let state = TestEngineStateBuilder::new() .with_unsafe_head(test_block_info(20)) + .with_local_safe_head(test_block_info(5)) .with_safe_head(test_block_info(5)) .with_finalized_head(test_block_info(3)) .build(); @@ -175,7 +176,8 @@ async fn delegated_poll_during_el_sync_moves_safe_labels_and_skips_finalization( // One poll: the safe head, then the finalized block pinned by hash. Each step drains the // previous request's task and then accepts the next request. requests - .send(EngineActorRequest::ProcessSafeL2Signal(ConsolidateInput::BlockInfo( + .send(EngineActorRequest::ProcessSafeL2Signal(delegated_safe_heads( + delegated_safe_head, delegated_safe_head, ))) .await @@ -320,7 +322,13 @@ async fn finalization_target_survives_resets_below_it(#[case] target_kind: Targe requests.send(target_kind.finalize_request(target)).await.unwrap(); actor.step().await.unwrap(); assert!(second_reset_result.try_recv().unwrap().is_ok()); - requests.send(EngineActorRequest::ProcessSafeL2Signal(restored_safe.into())).await.unwrap(); + requests + .send(EngineActorRequest::ProcessSafeL2Signal(delegated_safe_heads( + restored_safe, + restored_safe, + ))) + .await + .unwrap(); actor.step().await.unwrap(); assert_eq!(actor.engine.len(), 1, "only the safe head update may remain queued"); assert!(!actor.reset_pending); diff --git a/rust/kona/crates/node/service/src/actors/engine/tests.rs b/rust/kona/crates/node/service/src/actors/engine/tests.rs index 0d0169c241f..9c23f9aafc8 100644 --- a/rust/kona/crates/node/service/src/actors/engine/tests.rs +++ b/rust/kona/crates/node/service/src/actors/engine/tests.rs @@ -4,7 +4,7 @@ use alloy_primitives::B256; use alloy_rpc_types_engine::{ForkchoiceUpdated, PayloadStatus, PayloadStatusEnum}; use kona_engine::{ Engine, EngineActorRequest, EngineState, EngineSyncStateUpdate, FinalizeBlockId, NoopBlockSink, - test_utils::{TestAttributesBuilder, test_engine_client}, + test_utils::{TestAttributesBuilder, delegated_safe_heads, test_engine_client}, }; use kona_genesis::RollupConfig; use kona_protocol::{BlockInfo, L2BlockInfo}; @@ -76,7 +76,7 @@ async fn invalid_forkchoice_resets_and_resumes(#[case] rebuild: bool, #[case] ho .build() .into() } else { - l2_block(1).into() + delegated_safe_heads(l2_block(1), l2_block(1)) }; requests.send(EngineActorRequest::ProcessSafeL2Signal(signal)).await.unwrap(); actor.step().await.unwrap(); @@ -101,7 +101,10 @@ async fn invalid_forkchoice_resets_and_resumes(#[case] rebuild: bool, #[case] ho ForkchoiceUpdated::new(PayloadStatus::from_status(PayloadStatusEnum::Valid)), ); let recovered = l2_block(2); - requests.send(EngineActorRequest::ProcessSafeL2Signal(recovered.into())).await.unwrap(); + requests + .send(EngineActorRequest::ProcessSafeL2Signal(delegated_safe_heads(recovered, recovered))) + .await + .unwrap(); actor.step().await.unwrap(); assert_eq!(state_rx.borrow().sync_state.unsafe_head(), genesis, "publish the reset head"); assert_eq!(state_rx.borrow().sync_state.safe_head(), genesis);