Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
c1c8522
kona-engine: deliver and retry delegated forkchoice updates
einar-oplabs Nov 7, 2026
3ca8583
kona-engine: prevent delegated finalized-head regression
einar-oplabs Nov 7, 2026
4f6808d
kona-engine: keep delegated finalization pending during EL catch-up
einar-oplabs Nov 7, 2026
cbefcee
Merge branch 'develop' into einar/23676-delegated-forkchoice
einar-oplabs Nov 10, 2026
a4754b0
Merge branch 'einar/23676-delegated-forkchoice' into einar/23677-dele…
einar-oplabs Nov 10, 2026
a278097
Merge branch 'einar/23677-delegated-finality' into einar/23675-delega…
einar-oplabs Nov 10, 2026
b73f56a
kona-engine: replace ForkchoiceDelivery with two synchronize functions
einar-oplabs Nov 12, 2026
baf6556
Merge branch 'einar/23676-delegated-forkchoice' into einar/23677-dele…
einar-oplabs Nov 12, 2026
e9305c4
kona-engine: inline the finalize task's finality checks
einar-oplabs Nov 12, 2026
d92d1fc
kona-node-service: report sync completion with the safe head only
einar-oplabs Nov 12, 2026
e2bcf65
kona-node-service: reject a delegate status whose finalized is above …
einar-oplabs Nov 12, 2026
5c7d492
kona-node-service: drive a skipped finalization through the engine actor
einar-oplabs Nov 12, 2026
bcab9ed
Merge branch 'einar/23677-delegated-finality' into einar/23675-delega…
einar-oplabs Nov 12, 2026
55610da
kona-engine: skip only a null finalization lookup and name the safe h…
einar-oplabs Nov 12, 2026
d732306
kona-engine: make the finalize task's missing-block classification const
einar-oplabs Nov 12, 2026
548e139
kona-node-service: retry finalization only while derivation is idle
einar-oplabs Nov 12, 2026
1942da9
Merge develop (up to 5b3854d301) into einar/23675-delegated-catchup
einar-oplabs Nov 12, 2026
afbce56
Merge develop's squash of #23900 (b636ba6fee), whose content this bra…
einar-oplabs Nov 12, 2026
bc688f1
Merge branch 'develop' into einar/23675-delegated-catchup
einar-oplabs Nov 12, 2026
539115a
kona-node: pin finalize lookup failures and the reset resend in tests
einar-oplabs Nov 13, 2026
fb19fab
kona-node-service: drop the unused Duration import in the delegate tests
einar-oplabs Nov 13, 2026
68d3df1
Merge branch 'develop' into einar/23675-delegated-catchup
joshklop Nov 17, 2026
21c7556
kona-node-service: expect newPayloadV2 in the finalization reset test
joshklop Nov 17, 2026
973f8af
Merge branch 'develop' into einar/23675-delegated-catchup
joshklop Nov 17, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 3 additions & 9 deletions rust/kona/crates/node/engine/src/task_queue/core.rs
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,8 @@ mod recovery_tests {
ConsolidateInput, ConsolidateTask, FinalizeBlockId, FinalizeTask, InsertTask,
NoopBlockSink,
test_utils::{
TestAttributesBuilder, TestEngineStateBuilder, test_block_info, test_engine_client,
TestAttributesBuilder, TestEngineStateBuilder, expect_sync_start, test_block_info,
test_engine_client,
},
};
use alloy_rpc_types_engine::{
Expand Down Expand Up @@ -368,14 +369,7 @@ mod recovery_tests {
let state = TestEngineStateBuilder::new().with_unsafe_head(genesis).build();
let forkchoice = state.sync_state.create_forkchoice_state();
let (client, l1, l2) = test_engine_client(config.clone());
for tag in ["finalized", "safe", "latest"] {
l2.expect_params("eth_getBlockByNumber", serde_json::json!([tag, true]), &block);
}
l1.expect(
"eth_getBlockByHash",
alloy_rpc_types_eth::Block::<alloy_rpc_types_eth::Transaction>::default(),
);
l2.expect_forkchoice_updated(forkchoice, PayloadStatusEnum::Valid);
expect_sync_start(&l1, &l2, &block, forkchoice);
let (state_tx, _state_rx) = tokio::sync::watch::channel(state);
let mut engine = Engine::new(state, state_tx);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,16 @@ pub enum FinalizeTaskError {
/// The block to finalize was not found.
#[error("The block to finalize was not found: Number {0}")]
BlockNotFound(u64),
/// The block is not safe, and therefore cannot be finalized.
#[error("Attempted to finalize a block that is not yet safe")]
BlockNotSafe,
/// The block is not safe yet, and therefore cannot be finalized. A reset can lower the safe
/// head below a target that was safe when it was requested, and a delegate can run ahead of
/// the local safe head.
#[error("Finalization target {target} is above the safe head {safe}")]
BlockNotSafe {
/// The number of the safe head.
safe: u64,
/// The number of the finalization target.
target: u64,
},
/// The engine returned a block at a different height than the one requested.
#[error("Requested block {requested} to finalize, but the engine returned block {returned}")]
BlockNumberMismatch {
Expand Down Expand Up @@ -57,6 +64,12 @@ pub enum FinalizeTaskError {
/// The finalization target.
target: BlockNumHash,
},
/// The engine has no block at the pinned target's height yet, for example while it catches up.
#[error("The engine does not have finalization target {target:?} yet")]
NotYetAvailable {
/// The finalization target.
target: BlockNumHash,
},
/// A temporary RPC failure.
#[error(transparent)]
TransportError(#[from] RpcError<TransportErrorKind>),
Expand All @@ -66,12 +79,13 @@ impl EngineTaskError for FinalizeTaskError {
fn severity(&self) -> EngineTaskErrorSeverity {
match self {
Self::BehindFinalizedHead { .. } |
Self::BlockNotSafe { .. } |
Self::ConflictsWithFinalizedHead { .. } |
Self::NotCanonical { .. } => EngineTaskErrorSeverity::Skip,
Self::BlockNotFound(_) |
Self::BlockNotSafe |
Self::BlockNumberMismatch { .. } |
Self::FromBlock(_) => EngineTaskErrorSeverity::Critical,
Self::NotCanonical { .. } |
Self::NotYetAvailable { .. } => EngineTaskErrorSeverity::Skip,
Self::BlockNotFound(_) | Self::BlockNumberMismatch { .. } | Self::FromBlock(_) => {
EngineTaskErrorSeverity::Critical
}
Self::ForkchoiceUpdateFailed(inner) => inner.severity(),
Self::TransportError(_) => EngineTaskErrorSeverity::Temporary,
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@ use alloy_eips::BlockNumHash;
/// `L2Finalizer`). The number is sufficient because there is only one chain to consult.
/// - [`FinalizeBlockId::ByHash`] is used when an upstream supplies `(number, hash)` and the engine
/// must finalize the specific block identified by hash. If the engine's canonical block at that
/// height has a different hash, the task skips the target rather than finalize the engine's block
/// in its place.
/// height has a different hash, or the engine has no block at that height yet, the task skips the
/// target rather than finalize the engine's block in its place.
///
/// EL finalization is irreversible, so finalizing the wrong block is unrecoverable. Forcing every
/// caller to pick a variant prevents accidental hash-loss at the boundary.
Expand All @@ -21,7 +21,8 @@ pub enum FinalizeBlockId {
/// Finalize whatever block the engine has at the given L2 block number.
ByNumber(u64),
/// Finalize the block matching the given `(number, hash)` pair, only once it is the engine's
/// canonical block at that height.
/// canonical block at that height. A skipped target leaves the state unchanged and may be
/// requested again.
ByHash(BlockNumHash),
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,9 @@ use std::sync::Arc;
/// [`EngineState`], and dispatches a forkchoice update to finalize the block.
///
/// Finality only moves forward. A target behind the finalized head, a different block at the
/// finalized height, or a pinned hash that is not the engine's canonical block fails with an error
/// that skips the task and leaves the engine state unchanged.
/// finalized height, a pinned hash that is not the engine's canonical block, a target above the
/// safe head, or a pinned target the engine has no block for yet fails with an error that skips
/// the task and leaves the engine state unchanged.
#[derive(Debug, Clone, Constructor)]
pub struct FinalizeTask {
/// The engine client.
Expand All @@ -28,6 +29,17 @@ pub struct FinalizeTask {
}

impl FinalizeTask {
/// A pinned target the execution layer answers with null is not available yet, for example
/// while it catches up: the task is skipped rather than held at the head of the queue ahead
/// of the imports that let it catch up. A missing by-number target is below the safe head, so
/// the engine must already have it.
const fn block_missing(&self) -> FinalizeTaskError {
match self.block_id {
FinalizeBlockId::ByHash(target) => FinalizeTaskError::NotYetAvailable { target },
FinalizeBlockId::ByNumber(number) => FinalizeTaskError::BlockNotFound(number),
}
}

/// Whether `target` is the finalized head itself; a different block at its height is an error.
fn is_already_finalized(
finalized_head: &L2BlockInfo,
Expand Down Expand Up @@ -61,7 +73,7 @@ impl FinalizeTask {
.full()
.await
.map_err(FinalizeTaskError::TransportError)?
.ok_or(FinalizeTaskError::BlockNotFound(requested))?;
.ok_or_else(|| self.block_missing())?;
let canonical = BlockNumHash { hash: block.header.hash, number: block.header.number };
debug!(target: "engine", ?canonical, "Fetched the block to finalize");

Expand Down Expand Up @@ -106,9 +118,14 @@ impl ExecutableEngineTask for FinalizeTask {
return Ok(());
}

// Sanity check that the block that is being finalized is at least safe.
if state.sync_state.safe_head().block_info.number < target_number {
return Err(FinalizeTaskError::BlockNotSafe);
// Sanity check that the block that is being finalized is at least safe. A later request
// finalizes a target skipped here, or a newer block.
let safe_number = state.sync_state.safe_head().block_info.number;
if safe_number < target_number {
return Err(FinalizeTaskError::BlockNotSafe {
safe: safe_number,
target: target_number,
});
}

let block_info = self.fetch_canonical_block_info().await?;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
use crate::{
EngineState, EngineSyncState, EngineSyncStateUpdate, EngineTaskError, EngineTaskErrorSeverity,
EngineState, EngineSyncState, EngineSyncStateUpdate, EngineTaskError,
EngineTaskErrorSeverity::{self, Critical, Skip, Temporary},
ExecutableEngineTask, FinalizeBlockId, FinalizeTask, FinalizeTaskError,
test_utils::{
RpcMock, TestEngineStateBuilder, TestRpcBlock as RpcBlock, test_block_info,
Expand Down Expand Up @@ -117,16 +118,59 @@ const fn is_number_mismatch(err: &FinalizeTaskError) -> bool {
matches!(err, FinalizeTaskError::BlockNumberMismatch { requested: ADVANCED, .. })
}

const fn is_not_safe(err: &FinalizeTaskError) -> bool {
matches!(err, FinalizeTaskError::BlockNotSafe { .. })
}

fn is_not_yet_available(err: &FinalizeTaskError) -> bool {
let pinned_id = l2_block_info(&canonical(ADVANCED)).block_info.id();
matches!(err, FinalizeTaskError::NotYetAvailable { target } if *target == pinned_id)
}

const fn is_block_not_found(err: &FinalizeTaskError) -> bool {
matches!(err, FinalizeTaskError::BlockNotFound(ADVANCED))
}

/// Targets behind the finalized head, or a pinned hash that conflicts with it, are skipped before
/// asking the execution layer for the block: the test engine expects no request at all.
const fn is_transport_error(err: &FinalizeTaskError) -> bool {
matches!(err, FinalizeTaskError::TransportError(_))
}

/// A target above the finalized head, pinned to the engine's canonical block at its height.
fn pinned_target() -> FinalizeBlockId {
by_hash(&canonical(ADVANCED))
}

/// A target above the finalized head, identified by number only.
fn numbered_target() -> FinalizeBlockId {
by_number(&canonical(ADVANCED))
}

/// What the engine answers for a block it cannot serve.
#[derive(Clone, Copy, Debug)]
enum Unavailable {
ErrorResponse,
Null,
TransportFailure,
}

fn expect_unavailable_block(l2: &RpcMock, number: u64, answer: Unavailable) {
match answer {
Unavailable::ErrorResponse => l2.expect_error_code("eth_getBlockByNumber", -32000),
Unavailable::Null => l2.expect_block_by_number(number, None),
Unavailable::TransportFailure => l2.expect_transport_error("eth_getBlockByNumber"),
}
}

/// Targets behind the finalized head, a pinned hash that conflicts with it, and targets above the
/// safe head are skipped before asking the execution layer for the block: the test engine expects
/// no request at all. A reset can lower the safe head below a target that was safe when it was
/// requested, and a later request finalizes it.
#[rstest]
#[case::stale_by_hash(by_hash(&canonical(STALE)), is_behind)]
#[case::stale_by_number(by_number(&canonical(STALE)), is_behind)]
#[case::conflicting_by_hash(by_hash(&conflicting(FINALIZED)), is_conflict)]
#[case::above_safe_by_hash(by_hash(&canonical(UNSAFE)), is_not_safe)]
#[case::above_safe_by_number(by_number(&canonical(UNSAFE)), is_not_safe)]
#[tokio::test]
async fn finalize_task_skips_target_without_lookup(
#[case] target: FinalizeBlockId,
Expand All @@ -138,7 +182,7 @@ async fn finalize_task_skips_target_without_lookup(
let err = finalize(target, &mut state, |_| {}).await.unwrap_err();

assert!(is_expected_error(&err), "{err:?}");
assert_eq!(err.severity(), EngineTaskErrorSeverity::Skip);
assert_eq!(err.severity(), Skip);
assert_eq!(state, original_state);
}

Expand All @@ -151,8 +195,11 @@ async fn finalize_task_refuses_target_above_safe_head_without_lookup() {

let err = finalize(by_hash(&canonical(SAFE + 5)), &mut state, |_| {}).await.unwrap_err();

assert!(matches!(err, FinalizeTaskError::BlockNotSafe), "{err:?}");
assert_eq!(err.severity(), EngineTaskErrorSeverity::Critical);
assert!(
matches!(err, FinalizeTaskError::BlockNotSafe { safe: SAFE, target } if target == SAFE + 5),
"{err:?}"
);
assert_eq!(err.severity(), EngineTaskErrorSeverity::Skip);
assert_eq!(state, original_state);
}

Expand All @@ -178,7 +225,7 @@ async fn finalize_task_skips_target_off_the_canonical_chain(
.unwrap_err();

assert!(is_expected_error(&err), "{err:?}");
assert_eq!(err.severity(), EngineTaskErrorSeverity::Skip);
assert_eq!(err.severity(), Skip);
assert_eq!(state, original_state);
}

Expand Down Expand Up @@ -226,38 +273,100 @@ async fn finalize_task_skips_lookup_for_finalized_head() {
assert_eq!(state, original_state);
}

/// The engine serving no block, or a block at a height other than the requested one, stops the
/// node instead of finalizing anything. A misplaced block could otherwise bypass the guards against
/// moving finality backward.
/// The engine serving a block at a height other than the requested one stops the node instead of
/// finalizing anything. A misplaced block could otherwise bypass the guards against moving
/// finality backward.
#[rstest]
#[case::stale_block_by_number(by_number(&canonical(ADVANCED)), canonical(STALE))]
#[case::higher_block_by_number(by_number(&canonical(ADVANCED)), canonical(SAFE))]
#[case::stale_block_by_hash(by_hash(&canonical(ADVANCED)), canonical(STALE))]
#[tokio::test]
async fn finalize_task_stops_on_block_at_unrequested_height(
#[case] target: FinalizeBlockId,
#[case] served_block: RpcBlock,
) {
let mut state = engine_state(established());
let original_state = state;

let err = finalize(target, &mut state, |l2| {
l2.expect_block_by_number(target.number(), Some(served_block))
})
.await
.unwrap_err();

assert!(is_number_mismatch(&err), "{err:?}");
assert_eq!(err.severity(), Critical);
assert_eq!(state, original_state);
}

/// While the execution layer catches up, it answers null for a pinned target's height, and the
/// target is skipped until its upstream resubmits it. An error response or a transport failure is a
/// failed lookup and is retried for either kind of target. A by-number target is below the safe
/// head, so the engine must have it: its absence stops the node.
#[rstest]
#[case::missing_by_number(FinalizeBlockId::ByNumber(ADVANCED), None, is_block_not_found)]
#[case::missing_by_hash(by_hash(&canonical(ADVANCED)), None, is_block_not_found)]
#[case::stale_block_by_number(
/* target */ FinalizeBlockId::ByNumber(ADVANCED),
/* served_block */ Some(canonical(STALE)),
/* is_expected_error */ is_number_mismatch
#[case::pinned_null(pinned_target(), Unavailable::Null, is_not_yet_available, Skip)]
#[case::pinned_error_response(
Comment thread
einar-oplabs marked this conversation as resolved.
Resolved
pinned_target(),
Unavailable::ErrorResponse,
is_transport_error,
Temporary
)]
#[case::pinned_transport_failure(
pinned_target(),
Unavailable::TransportFailure,
is_transport_error,
Temporary
)]
#[case::higher_block_by_number(
/* target */ FinalizeBlockId::ByNumber(ADVANCED),
/* served_block */ Some(canonical(SAFE)),
/* is_expected_error */ is_number_mismatch
#[case::by_number_null(numbered_target(), Unavailable::Null, is_block_not_found, Critical)]
#[case::by_number_error_response(
numbered_target(),
Unavailable::ErrorResponse,
is_transport_error,
Temporary
)]
#[case::stale_block_by_hash(by_hash(&canonical(ADVANCED)), Some(canonical(STALE)), is_number_mismatch)]
#[tokio::test]
async fn finalize_task_stops_on_unexpected_lookup_result(
async fn finalize_task_handles_block_the_engine_cannot_serve(
#[case] target: FinalizeBlockId,
#[case] served_block: Option<RpcBlock>,
#[case] answer: Unavailable,
#[case] is_expected_error: fn(&FinalizeTaskError) -> bool,
#[case] severity: EngineTaskErrorSeverity,
) {
let mut state = engine_state(established());
let original_state = state;

let err =
finalize(target, &mut state, |l2| l2.expect_block_by_number(target.number(), served_block))
finalize(target, &mut state, |l2| expect_unavailable_block(l2, target.number(), answer))
.await
.unwrap_err();

assert!(is_expected_error(&err), "{err:?}");
assert_eq!(err.severity(), EngineTaskErrorSeverity::Critical);
assert_eq!(err.severity(), severity, "{err:?}");
Comment thread
einar-oplabs marked this conversation as resolved.
Resolved
assert_eq!(state, original_state);
}

/// A pinned target skipped while the engine catches up finalizes once its upstream resubmits it
/// and the engine has the block.
#[tokio::test]
async fn resubmitted_pinned_target_finalizes_once_engine_has_it() {
let mut state = engine_state(established());
let served_block = canonical(ADVANCED);
let target = by_hash(&served_block);
let expected_sync_state = with_finalized_head(&state, l2_block_info(&served_block));

let err = finalize(target, &mut state, |l2| l2.expect_block_by_number(ADVANCED, None))
.await
.unwrap_err();
assert!(matches!(err, FinalizeTaskError::NotYetAvailable { .. }), "{err:?}");

finalize(target, &mut state, |l2| {
l2.expect_block_by_number(ADVANCED, Some(served_block));
l2.expect_forkchoice_updated(
expected_sync_state.create_forkchoice_state(),
PayloadStatusEnum::Valid,
);
})
.await
.unwrap();

assert_eq!(state.sync_state, expected_sync_state);
}
2 changes: 1 addition & 1 deletion rust/kona/crates/node/engine/src/test_utils/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ mod deposits;
pub use deposits::l1_info_deposit;

mod rpc;
pub use rpc::{RpcMock, test_engine_client};
pub use rpc::{RpcMock, expect_sync_start, test_engine_client};

mod engine_state;
pub use engine_state::TestEngineStateBuilder;
Expand Down
Loading
Loading