From f9e7eec66600a3ed508f758a920db99fc561c911 Mon Sep 17 00:00:00 2001 From: Josh Klopfenstein Date: Tue, 3 Nov 2026 09:12:32 -0800 Subject: [PATCH] kona-engine,kona-node!: drop derived work from before an engine reset Derivation keeps handling the requests queued ahead of an engine reset with its old pipeline, so attributes and finalizations derived before the reset reach the engine after it. Consolidated, those attributes moved the safe head past blocks that were never re-derived, recorded the wrong block in the safe head database after an L1 reorg, and could leave derivation waiting forever for a confirmation; a stale finalization above the rewound safe head stopped the engine. Now: - both actors count engine resets: the engine counts the reset signals it sends, derivation the ones it handles, and derivation stamps its attributes and finalizations with its count. The engine drops work stamped before its latest reset, and drops consolidations and finalizations queued when a reset succeeds; - derived attributes whose parent is not the local-safe head mean the actors disagree, so the engine resets instead of consolidating them; - the engine reports the local-safe head before starting a reset or backing off, so a confirmation earlier in the same drain is not lost; - derivation ignores a safe head that does not confirm the attributes it is waiting for, instead of adopting it and leaving a gap in the safe head database. Derivation counts a reset when the engine's reset signal arrives, not on each retry of its pipeline reset. BREAKING CHANGE: EngineActorRequest::ProcessSafeL2Signal is replaced by ProcessDerivedAttributes and ProcessDerivedFinalization, which carry the derivation actor's reset epoch, and ProcessDelegatedSafeHead. DerivationEngineClient::send_safe_l2_signal is replaced by send_derived_attributes, send_derived_finalization and send_delegated_safe_head, and Engine::drop_derived_tasks is new. ConsolidateTaskError has a new DoesNotExtendLocalSafe variant, which requests a reset. Co-Authored-By: Claude Opus 5.5 --- rust/kona/crates/node/engine/src/request.rs | 26 ++++- .../crates/node/engine/src/task_queue/core.rs | 57 ++++++++++ .../src/task_queue/tasks/consolidate/error.rs | 15 ++- .../src/task_queue/tasks/consolidate/task.rs | 60 ++++++++++ .../src/actors/derivation/actor/core.rs | 61 +++++----- .../src/actors/derivation/actor/tests.rs | 74 ++++++++++-- .../actors/derivation/actor/tests/database.rs | 9 +- .../derivation/actor/tests/op_node_parity.rs | 18 ++- .../actor/tests/service_shutdown.rs | 4 +- .../derivation/actor/tests/signal_failures.rs | 4 +- .../src/actors/derivation/delegated/actor.rs | 2 +- .../src/actors/derivation/delegated/tests.rs | 8 +- .../src/actors/derivation/engine_client.rs | 69 ++++++++++-- .../node/service/src/actors/engine/actor.rs | 82 ++++++++++++-- .../service/src/actors/engine/queue_tests.rs | 106 ++++++++++++++++-- .../node/service/src/actors/engine/tests.rs | 26 +++-- 16 files changed, 512 insertions(+), 109 deletions(-) diff --git a/rust/kona/crates/node/engine/src/request.rs b/rust/kona/crates/node/engine/src/request.rs index 7def359094c..6abcd746dec 100644 --- a/rust/kona/crates/node/engine/src/request.rs +++ b/rust/kona/crates/node/engine/src/request.rs @@ -1,10 +1,8 @@ //! Requests sent to the node engine actor and their replies. -use crate::{ - BuildTaskError, CanonicalizeTaskError, ConsolidateInput, FinalizeBlockId, SealTaskError, -}; +use crate::{BuildTaskError, CanonicalizeTaskError, FinalizeBlockId, SealTaskError}; use alloy_rpc_types_engine::PayloadId; -use kona_protocol::OpAttributesWithParent; +use kona_protocol::{L2BlockInfo, OpAttributesWithParent}; use op_alloy_rpc_types_engine::OpExecutionPayloadEnvelope; use thiserror::Error; use tokio::sync::mpsc; @@ -14,8 +12,24 @@ use tokio::sync::mpsc; pub enum EngineActorRequest { /// Request to start building a block. Build(Box), - /// Request to process derived attributes or delegated safe block information. - ProcessSafeL2Signal(ConsolidateInput), + /// Request to consolidate attributes from the derivation pipeline. + ProcessDerivedAttributes { + /// The derived attributes. + attributes: Box, + /// How many engine resets derivation had handled when it derived the attributes. The + /// engine drops attributes derived before its latest reset. + reset_epoch: u64, + }, + /// Request to finalize an L2 block that derivation recorded as safe. + ProcessDerivedFinalization { + /// The L2 block number to finalize. + number: u64, + /// How many engine resets derivation had handled when it recorded the block. The engine + /// drops finalizations recorded before its latest reset. + reset_epoch: u64, + }, + /// Request to adopt a safe block reported by a derivation delegate. + ProcessDelegatedSafeHead(Box), /// Request to process the finalized L2 block. ProcessFinalizedL2Block(Box), /// Request to process a received unsafe L2 block. diff --git a/rust/kona/crates/node/engine/src/task_queue/core.rs b/rust/kona/crates/node/engine/src/task_queue/core.rs index 2808f0a2323..1a4ca34c0c1 100644 --- a/rust/kona/crates/node/engine/src/task_queue/core.rs +++ b/rust/kona/crates/node/engine/src/task_queue/core.rs @@ -172,6 +172,17 @@ impl Engine { self.active = None; } + /// Drops every queued [`EngineTask::Consolidate`] and [`EngineTask::Finalize`]. Call it when a + /// reset succeeds: derivation resets too and re-derives from the reset head, so attributes it + /// derived and blocks it asked to finalize before the reset are stale, even when they still + /// extend that head. In delegated mode this also drops the delegate's safe and finalized + /// heads, which it sends again on its next poll. + pub fn drop_derived_tasks(&mut self) { + self.tasks.retain(|queued| { + !matches!(queued.task, EngineTask::Consolidate(_) | EngineTask::Finalize(_)) + }); + } + /// Attempts to drain the queue by executing all [`EngineTask`]s in-order. If any task returns /// an error along the way, the error is returned and the task is not popped from the queue (in /// case it must be retried), unless it is a flush or skip error. @@ -442,4 +453,50 @@ mod recovery_tests { l1.assert_finished(); l2.assert_finished(); } + + // Attributes derived and finalizations requested before a reset must not run after it. Other + // queued work, such as unsafe payloads received meanwhile, survives. + #[test] + fn drop_derived_tasks_keeps_other_tasks() { + let config = Arc::new(RollupConfig::default()); + let (client, _l1, _l2) = test_engine_client(config.clone()); + let client = Arc::new(client); + let consolidate = || { + EngineTask::Consolidate(Box::new(ConsolidateTask::new( + client.clone(), + config.clone(), + ConsolidateInput::Attributes(Box::new( + TestAttributesBuilder::new().with_parent(test_block_info(10)).build(), + )), + Arc::new(NoopBlockSink), + ))) + }; + let insert = EngineTask::Insert(Box::new(InsertTask::new( + client.clone(), + config.clone(), + OpExecutionPayloadEnvelope::V1(ExecutionPayloadV1::from_block_slow( + &alloy_consensus::Block::::default(), + )), + false, + Arc::new(NoopBlockSink), + ))); + let (state_tx, _state_rx) = tokio::sync::watch::channel(EngineState::default()); + let mut engine = Engine::new(EngineState::default(), state_tx); + engine.enqueue(consolidate()); + engine.enqueue(insert); + engine.enqueue(EngineTask::Finalize(Box::new(crate::FinalizeTask::new( + client.clone(), + config.clone(), + crate::FinalizeBlockId::ByNumber(10), + )))); + engine.enqueue(consolidate()); + assert_eq!(engine.len(), 4); + + engine.drop_derived_tasks(); + assert_eq!(engine.len(), 1); + assert!(matches!( + engine.tasks.peek().map(|queued| &queued.task), + Some(EngineTask::Insert(_)) + )); + } } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/error.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/error.rs index 2ce1b187a76..bd8d45d5a8d 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/error.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/error.rs @@ -31,6 +31,17 @@ pub enum ConsolidateTaskError { /// The consolidation forkchoice update call to the engine api failed. #[error(transparent)] ForkchoiceUpdateFailed(#[from] SynchronizeTaskError), + /// Derived attributes do not extend the local-safe head, so derivation and the engine + /// disagree about the safe chain. + #[error( + "Derived attributes with parent {parent:?} do not extend the local-safe head {local_safe:?}" + )] + DoesNotExtendLocalSafe { + /// The attributes' parent. + parent: alloy_eips::BlockNumHash, + /// The engine's local-safe head. + local_safe: alloy_eips::BlockNumHash, + }, } impl From for ConsolidateTaskError { @@ -46,7 +57,9 @@ impl EngineTaskError for ConsolidateTaskError { fn severity(&self) -> EngineTaskErrorSeverity { match self { Self::BreaksFinality(_) => EngineTaskErrorSeverity::Skip, - Self::MissingUnsafeL2Block(_) => EngineTaskErrorSeverity::Reset, + Self::MissingUnsafeL2Block(_) | Self::DoesNotExtendLocalSafe { .. } => { + EngineTaskErrorSeverity::Reset + } Self::FailedToFetchDelegatedSafeBlock(_) | Self::FailedToFetchUnsafeL2Block => { EngineTaskErrorSeverity::Temporary } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs index 00f31457729..6af6297a642 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs @@ -406,6 +406,19 @@ impl ExecutableEngineTask for ConsolidateTask { // injected safe head is ahead of the EngineActor's unsafe head, we reconcile the unsafe chain // up to the safe head instead of consolidating. async fn execute(&self, state: &mut EngineState) -> Result<(), ConsolidateTaskError> { + if let ConsolidateInput::Attributes(attributes) = &self.input { + let parent = attributes.parent.block_info; + let local_safe = state.sync_state.local_safe_head().block_info; + if parent.id() != local_safe.id() { + // The engine drops attributes derived before its latest reset, so these are + // current: derivation and the engine disagree about the safe chain. Consolidating + // them would mark blocks safe that were never derived on it, so reset instead. + return Err(ConsolidateTaskError::DoesNotExtendLocalSafe { + parent: parent.id(), + local_safe: local_safe.id(), + }); + } + } if let ConsolidateInput::BlockInfo(safe_l2) = &self.input { state.sync_state.ensure_respects_finality(safe_l2.block_info.id())?; // Labels with the unsafe head below the safe head take the paths below, which move the @@ -717,6 +730,8 @@ mod tests { let cfg = Arc::new(RollupConfig::default()); let attributes = derived_attributes(true); let mut state = engine_state_with_safe_head(attributes.parent); + // Derived attributes extend the local-safe head. + state.sync_state = with_safe_head(&state, attributes.parent); let state_before = state; let (client, l1, l2) = test_engine_client(cfg.clone()); l2.expect_params( @@ -784,6 +799,8 @@ mod tests { let block = unsafe_block_matching(&attributes); let derived_safe_head = block_info_of(&cfg, &block); let mut state = engine_state_with_safe_head(attributes.parent); + // Derived attributes extend the local-safe head. + state.sync_state = with_safe_head(&state, attributes.parent); let expected_sync_state = with_safe_head(&state, derived_safe_head); let (client, l1, l2) = test_engine_client(cfg.clone()); l2.expect_params("eth_getBlockByNumber", serde_json::json!(["0xa", true]), block); @@ -1128,4 +1145,47 @@ mod tests { l1.assert_finished(); l2.assert_finished(); } + + // Derived attributes whose parent is not the local-safe head mean derivation and the engine + // disagree about the safe chain. The task asks for a reset, with no engine call, instead of + // marking blocks safe that were never derived or moving the safe head backwards. + #[rstest::rstest] + #[case::parent_above_local_safe(-1, false)] + #[case::parent_below_local_safe(1, false)] + #[case::parent_on_another_chain(0, true)] + #[tokio::test] + async fn attributes_off_the_local_safe_chain_reset( + #[case] local_safe_offset: i64, + #[case] other_hash: bool, + ) { + use crate::ExecutableEngineTask; + let attributes = derived(vec![alloy_primitives::Bytes::from(vec![DEPOSIT, 0x01])]); + // The local-safe head differs from the attributes' parent in only one way: its number, or + // its hash at the same number. + let mut head = attributes.parent; + head.block_info.number = + head.block_info.number.checked_add_signed(local_safe_offset).unwrap(); + if other_hash { + head.block_info.hash = B256::repeat_byte(0xab); + } + let mut state = + crate::test_utils::TestEngineStateBuilder::new().with_unsafe_head(head).build(); + let before = state.sync_state; + let cfg = holocene_cfg(); + let (client, l1, l2) = test_engine_client(cfg.clone()); + let task = ConsolidateTask::new( + Arc::new(client), + cfg, + ConsolidateInput::Attributes(Box::new(attributes)), + Arc::new(crate::NoopBlockSink), + ); + + let err = task.execute(&mut state).await.unwrap_err(); + + assert!(matches!(err, ConsolidateTaskError::DoesNotExtendLocalSafe { .. }), "{err:?}"); + assert_eq!(err.severity(), crate::EngineTaskErrorSeverity::Reset); + assert_eq!(state.sync_state, before); + l1.assert_finished(); + l2.assert_finished(); + } } diff --git a/rust/kona/crates/node/service/src/actors/derivation/actor/core.rs b/rust/kona/crates/node/service/src/actors/derivation/actor/core.rs index 2b602e11ff9..2b5b97b309b 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/actor/core.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/actor/core.rs @@ -11,7 +11,6 @@ use kona_derive::{ ActivationSignal, Pipeline, PipelineError, PipelineErrorKind, ResetError, ResetSignal, Signal, SignalReceiver, StepResult, }; -use kona_engine::FinalizeBlockId; use kona_protocol::{BlockInfo, L2BlockInfo, OpAttributesWithParent}; use kona_safedb::{SafeDb, SafeDbError}; use std::{sync::Arc, time::Duration}; @@ -53,6 +52,9 @@ where pub(super) attributes_awaiting_confirmation: Option, /// A pipeline reset that has not yet succeeded. Derivation must not resume before it does. pending_reset: Option, + /// How many engine resets this actor has handled. Attributes and finalizations carry it so + /// that the engine can drop those derived before its latest reset. + reset_epoch: u64, } /// A pipeline reset that has not yet succeeded. @@ -122,6 +124,7 @@ where l1_block_zero, attributes_awaiting_confirmation: None, pending_reset: None, + reset_epoch: 0, } } @@ -333,7 +336,11 @@ where ) -> Result<(), DerivationError> { match request_type { DerivationActorRequest::ProcessEngineSignalRequest(signal) => match *signal { - Signal::Reset(reset) => self.reset_pipeline_and_resume(reset).await?, + Signal::Reset(reset) => { + // Count each engine reset once, not each retry of its pipeline reset. + self.reset_epoch += 1; + self.reset_pipeline_and_resume(reset).await? + } Signal::Activation(_) => { self.forward_non_reset_signal(*signal).await; self.derivation_state_machine @@ -369,9 +376,24 @@ where } DerivationActorRequest::ProcessEngineSafeHeadUpdateRequest(safe_head) => { info!(target: "derivation", safe_head = ?*safe_head, "Received safe head from engine."); + // Only consolidating the attributes derived here moves the engine's safe head, and + // each consolidation advances it. Adopting any other head would skip blocks that + // were never derived and leave a gap in the safe head database. + let Some(attributes) = self + .attributes_awaiting_confirmation + .take_if(|attributes| attributes_produce(attributes, &safe_head)) + else { + warn!( + target: "derivation", + safe_head = safe_head.block_info.number, + awaiting = ?self.attributes_awaiting_confirmation.as_ref().map(|attributes| attributes.block_number()), + "Ignoring a safe head that does not confirm the derived attributes" + ); + return Ok(()); + }; self.derivation_state_machine .update(&DerivationStateUpdate::NewAttributesConfirmed(safe_head.clone()))?; - self.settle_derived_attributes(*safe_head)?; + self.record_confirmed_attributes(&attributes, *safe_head)?; self.attempt_derivation().await?; } @@ -388,32 +410,15 @@ where Ok(()) } - /// Records and queues for finalization the attributes that `safe_head` confirms. + /// Records and queues for finalization the derived `attributes` that `safe_head` confirms. /// - /// A head that confirms nothing pending was adopted by the engine rather than derived here, - /// so there is nothing to record. Finalization is queued only once the record is written, so - /// the engine's finalized head can never run ahead of the recorded history. - pub(super) fn settle_derived_attributes( + /// Finalization is queued only once the record is written, so the engine's finalized head can + /// never run ahead of the recorded history. + pub(super) fn record_confirmed_attributes( &mut self, + attributes: &OpAttributesWithParent, safe_head: L2BlockInfo, ) -> Result<(), DerivationError> { - let Some(attributes) = self.attributes_awaiting_confirmation.take() else { - debug!( - target: "derivation", - safe_head = safe_head.block_info.number, - "Safe head confirms no derived attributes" - ); - return Ok(()); - }; - if !attributes_produce(&attributes, &safe_head) { - warn!( - target: "derivation", - expected = attributes.block_number(), - safe_head = safe_head.block_info.number, - "Safe head does not match the derived attributes awaiting confirmation" - ); - return Ok(()); - } if let (Some(database), Some(derived_from)) = (&self.safe_db, attributes.derived_from) { debug!(target: "derivation", l1_block = derived_from.number, safe_head = safe_head.block_info.number, "Recording safe head"); database.safe_head_updated(safe_head, derived_from.id()).inspect_err(|error| { @@ -421,7 +426,7 @@ where })?; debug!(target: "derivation", l1_block = derived_from.number, safe_head = safe_head.block_info.number, "Recorded safe head"); } - self.finalizer.enqueue_for_finalization(&attributes); + self.finalizer.enqueue_for_finalization(attributes); Ok(()) } @@ -478,7 +483,7 @@ where // Local L1-finality: the engine's own canonical chain is the authoritative source at this // height, so finalize by number. self.engine_client - .send_finalized_l2_block(FinalizeBlockId::ByNumber(l2_block_number)) + .send_derived_finalization(l2_block_number, self.reset_epoch) .await .map_err(|e| DerivationError::Sender(Box::new(e))) } @@ -521,7 +526,7 @@ where // Send payload attributes out for processing. self.engine_client - .send_safe_l2_signal(payload_attributes.into()) + .send_derived_attributes(payload_attributes, self.reset_epoch) .await .map_err(|e| DerivationError::Sender(Box::new(e)))?; diff --git a/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs b/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs index 818294a343b..54bbc07ea1f 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/actor/tests.rs @@ -10,7 +10,6 @@ use kona_derive::{ ActivationSignal, Pipeline, PipelineError, PipelineErrorKind, PipelineResult, ResetError, Signal, SignalReceiver, StepResult, }; -use kona_engine::FinalizeBlockId; use kona_genesis::{HardForkConfig, RollupConfig, SystemConfig}; use kona_protocol::{BlockInfo, L2BlockInfo, OpAttributesWithParent}; use kona_safedb::{SafeDatabase, SafeDb, SafeDbError, SafeHeadRecord}; @@ -494,7 +493,7 @@ fn derived_attributes() -> OpAttributesWithParent { fn engine_client_accepting_attributes() -> MockDerivationEngineClient { let mut engine_client = MockDerivationEngineClient::new(); - engine_client.expect_send_safe_l2_signal().returning(|_| Ok(())); + engine_client.expect_send_derived_attributes().returning(|_, _| Ok(())); engine_client } @@ -581,6 +580,57 @@ async fn a_confirmed_block_is_recorded_against_its_derivation_source() { assert_eq!(node.db.last_entry().unwrap(), record(DERIVED_FROM_L1, DERIVED_HEAD)); } +/// Derivation stamps its attributes with the number of engine resets it has handled, which the +/// engine compares with the number of resets it has sent to drop stale attributes. +#[tokio::test] +async fn attributes_carry_the_number_of_resets_handled() { + let mut engine_client = MockDerivationEngineClient::new(); + for epoch in [0, 1] { + engine_client + .expect_send_derived_attributes() + .withf(move |_, reset_epoch| *reset_epoch == epoch) + .times(1) + .returning(|_, _| Ok(())); + } + let mut node = RecordingActor::new(engine_client); + node.derive_and_confirm_one_block().await; + node.reset_to_resume_head().await; +} + +/// A safe head that does not confirm the attributes derivation is waiting for is ignored, whether +/// it is at another height or at the right height on another parent. The engine drops attributes +/// derived before a reset, so this is defense in depth: adopting such a head would skip blocks +/// that were never re-derived and leave a gap in the safe head database. +#[tokio::test] +async fn a_safe_head_that_confirms_no_awaited_attributes_is_ignored() { + let mut node = RecordingActor::new(engine_client_accepting_attributes()); + node.db.safe_head_updated(l2_block(RESUME_HEAD - 10), l1_block(EARLIEST_L1).id()).unwrap(); + node.db.safe_head_updated(l2_block(RESUME_HEAD), l1_block(EARLIER_L1).id()).unwrap(); + node.derive_and_confirm_one_block().await; + node.reset_to_resume_head().await; + let truncated = record(EARLIER_L1, RESUME_HEAD); + assert_eq!( + node.actor.derivation_state_machine.current_state(), + DerivationState::AwaitingSafeHeadConfirmation + ); + + let mut other_parent = l2_block(DERIVED_HEAD); + other_parent.block_info.parent_hash = B256::repeat_byte(0xab); + for head in [l2_block(DERIVED_HEAD + 40), other_parent] { + node.handle(DerivationActorRequest::ProcessEngineSafeHeadUpdateRequest(Box::new(head))) + .await; + assert_eq!(node.db.last_entry().unwrap(), truncated); + assert_eq!( + node.actor.derivation_state_machine.last_confirmed_safe_head(), + l2_block(RESUME_HEAD) + ); + assert!(node.actor.attributes_awaiting_confirmation.is_some()); + } + + node.confirm_safe_head(DERIVED_HEAD).await; + assert_eq!(node.db.last_entry().unwrap(), record(DERIVED_FROM_L1, DERIVED_HEAD)); +} + /// With no history below the reset point, the L1 block that made the reset head safe is /// unknown, so nothing is re-recorded and the reset head itself must not be written. #[tokio::test] @@ -653,10 +703,10 @@ async fn resuming_above_the_recorded_tip_clears_the_history() { async fn a_block_is_queued_for_finalization_only_once_it_is_recorded() { let mut engine_client = engine_client_accepting_attributes(); engine_client - .expect_send_finalized_l2_block() - .withf(|block| matches!(block, FinalizeBlockId::ByNumber(DERIVED_HEAD))) + .expect_send_derived_finalization() + .withf(|number, _| *number == DERIVED_HEAD) .times(1) - .returning(|_| Ok(())); + .returning(|_, _| Ok(())); let mut node = RecordingActor::new(engine_client); node.complete_el_sync_at(RESUME_HEAD).await; @@ -672,10 +722,10 @@ async fn a_block_is_queued_for_finalization_only_once_it_is_recorded() { async fn a_block_safe_after_its_l1_finalized_is_finalized_on_the_next_l1_head() { let mut engine_client = engine_client_accepting_attributes(); engine_client - .expect_send_finalized_l2_block() - .withf(|block| matches!(block, FinalizeBlockId::ByNumber(DERIVED_HEAD))) + .expect_send_derived_finalization() + .withf(|number, _| *number == DERIVED_HEAD) .times(1) - .returning(|_| Ok(())); + .returning(|_, _| Ok(())); let mut node = RecordingActor::new(engine_client); node.complete_el_sync_at(RESUME_HEAD).await; node.finalize_l1(DERIVED_FROM_L1).await; @@ -696,10 +746,10 @@ async fn a_block_safe_after_its_l1_finalized_is_finalized_on_the_next_l1_head() async fn a_block_derived_again_after_a_reset_is_finalized_on_the_next_l1_head() { let mut engine_client = engine_client_accepting_attributes(); engine_client - .expect_send_finalized_l2_block() - .withf(|block| matches!(block, FinalizeBlockId::ByNumber(DERIVED_HEAD))) + .expect_send_derived_finalization() + .withf(|number, _| *number == DERIVED_HEAD) .times(2) - .returning(|_| Ok(())); + .returning(|_, _| Ok(())); let mut node = RecordingActor::new(engine_client); node.complete_el_sync_at(RESUME_HEAD).await; node.confirm_safe_head(DERIVED_HEAD).await; @@ -718,7 +768,7 @@ async fn a_block_derived_again_after_a_reset_is_finalized_on_the_next_l1_head() #[tokio::test] async fn an_l1_head_update_does_not_finalize_while_attributes_await_confirmation() { let mut engine_client = engine_client_accepting_attributes(); - engine_client.expect_send_finalized_l2_block().times(0); + engine_client.expect_send_derived_finalization().times(0); let mut node = RecordingActor::new(engine_client); node.actor.pipeline.attributes.push_back(OpAttributesWithParent { parent: l2_block(DERIVED_HEAD), diff --git a/rust/kona/crates/node/service/src/actors/derivation/actor/tests/database.rs b/rust/kona/crates/node/service/src/actors/derivation/actor/tests/database.rs index 699bc1e6933..047a5e026bc 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/actor/tests/database.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/actor/tests/database.rs @@ -66,9 +66,8 @@ fn a_confirmed_block_is_written_directly_to_its_derivation_source() { .once() .returning(|_, _| Ok(())); let mut actor = actor_with_database(Some(Arc::new(database))); - actor.attributes_awaiting_confirmation = Some(derived_attributes()); - actor.settle_derived_attributes(l2_block(DERIVED_HEAD)).unwrap(); + actor.record_confirmed_attributes(&derived_attributes(), l2_block(DERIVED_HEAD)).unwrap(); assert_eq!(actor.finalizer.try_finalize_next(l1_block(DERIVED_FROM_L1)), Some(DERIVED_HEAD)); } @@ -82,9 +81,8 @@ fn a_failed_write_is_returned_without_retry_or_finalization() { .once() .returning(|_, _| Err(SafeDbError::InvalidEntry)); let mut actor = actor_with_database(Some(Arc::new(database))); - actor.attributes_awaiting_confirmation = Some(derived_attributes()); - let result = actor.settle_derived_attributes(l2_block(DERIVED_HEAD)); + let result = actor.record_confirmed_attributes(&derived_attributes(), l2_block(DERIVED_HEAD)); assert!(matches!(result, Err(DerivationError::SafeDbError(SafeDbError::InvalidEntry)))); assert!(actor.finalizer.try_finalize_next(l1_block(DERIVED_FROM_L1)).is_none()); @@ -160,9 +158,8 @@ async fn a_failed_truncation_stops_request_processing(#[case] request: Derivatio #[test] fn an_absent_database_skips_writes_and_truncation() { let mut actor = actor_with_database(None); - actor.attributes_awaiting_confirmation = Some(derived_attributes()); - actor.settle_derived_attributes(l2_block(DERIVED_HEAD)).unwrap(); + actor.record_confirmed_attributes(&derived_attributes(), l2_block(DERIVED_HEAD)).unwrap(); actor.truncate_safe_db(l2_block(RESUME_HEAD), "engine reset").unwrap(); assert_eq!(actor.finalizer.try_finalize_next(l1_block(DERIVED_FROM_L1)), Some(DERIVED_HEAD)); diff --git a/rust/kona/crates/node/service/src/actors/derivation/actor/tests/op_node_parity.rs b/rust/kona/crates/node/service/src/actors/derivation/actor/tests/op_node_parity.rs index 104544f2522..e36d0950121 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/actor/tests/op_node_parity.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/actor/tests/op_node_parity.rs @@ -31,8 +31,13 @@ async fn temporary_reset_failure_holds_derivation_until_the_retry_like_op_node( #[case] request_during_backoff: DerivationActorRequest, ) { let mut engine_client = MockDerivationEngineClient::new(); - engine_client.expect_send_safe_l2_signal().times(1).returning(|_| Ok(())); - engine_client.expect_send_finalized_l2_block().times(0); + // The retry belongs to the same engine reset, so it is not counted as another one. + engine_client + .expect_send_derived_attributes() + .withf(|_, reset_epoch| *reset_epoch == 1) + .times(1) + .returning(|_, _| Ok(())); + engine_client.expect_send_derived_finalization().times(0); let mut node = RecordingActor::new(engine_client); node.actor.pipeline.attributes.clear(); if let Some(head) = el_sync_completed_before_reset_at { @@ -76,8 +81,13 @@ async fn temporary_reset_failure_holds_derivation_until_the_retry_like_op_node( async fn reorged_reset_head_requests_a_fresh_engine_reset_like_op_node() { let mut engine_client = MockDerivationEngineClient::new(); engine_client.expect_reset_engine_forkchoice().times(1).returning(|| Ok(())); - engine_client.expect_send_safe_l2_signal().times(1).returning(|_| Ok(())); - engine_client.expect_send_finalized_l2_block().times(0); + // The fresh engine reset is counted once, on top of the first. + engine_client + .expect_send_derived_attributes() + .withf(|_, reset_epoch| *reset_epoch == 2) + .times(1) + .returning(|_, _| Ok(())); + engine_client.expect_send_derived_finalization().times(0); let mut node = RecordingActor::new(engine_client); node.actor.pipeline.attributes.clear(); node.complete_el_sync_at(DERIVED_HEAD).await; diff --git a/rust/kona/crates/node/service/src/actors/derivation/actor/tests/service_shutdown.rs b/rust/kona/crates/node/service/src/actors/derivation/actor/tests/service_shutdown.rs index 986884b48d6..5f862e0aab6 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/actor/tests/service_shutdown.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/actor/tests/service_shutdown.rs @@ -11,8 +11,8 @@ use tokio_util::sync::CancellationToken; async fn critical_pipeline_reset_failure_stops_service_and_rpc() { let error = l2_provider_error().crit(); let mut engine_client = MockDerivationEngineClient::new(); - engine_client.expect_send_safe_l2_signal().times(0); - engine_client.expect_send_finalized_l2_block().times(0); + engine_client.expect_send_derived_attributes().times(0); + engine_client.expect_send_derived_finalization().times(0); let mut node = RecordingActor::new(engine_client); node.actor.pipeline.attributes.clear(); node.complete_el_sync_at(DERIVED_HEAD).await; diff --git a/rust/kona/crates/node/service/src/actors/derivation/actor/tests/signal_failures.rs b/rust/kona/crates/node/service/src/actors/derivation/actor/tests/signal_failures.rs index 2bbdd75b0b2..86b9373c39d 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/actor/tests/signal_failures.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/actor/tests/signal_failures.rs @@ -4,8 +4,8 @@ use super::*; async fn critical_pipeline_reset_failure_cannot_resume_queued_attributes() { let error = l2_provider_error().crit(); let mut engine_client = MockDerivationEngineClient::new(); - engine_client.expect_send_safe_l2_signal().times(0); - engine_client.expect_send_finalized_l2_block().times(0); + engine_client.expect_send_derived_attributes().times(0); + engine_client.expect_send_derived_finalization().times(0); let mut node = RecordingActor::new(engine_client); node.actor.pipeline.attributes.clear(); node.complete_el_sync_at(DERIVED_HEAD).await; diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs index 37dc0e00185..aa22a850341 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/actor.rs @@ -316,7 +316,7 @@ where self.forwarded_lagging_safe_head = self.lagging_safe_head_to_remember(&sync_status.safe_l2); self.engine_client - .send_safe_l2_signal(sync_status.safe_l2.into()) + .send_delegated_safe_head(sync_status.safe_l2) .await .map_err(|e| DerivationError::Sender(Box::new(e)))?; } diff --git a/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs index b586f878488..63052c8687c 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/delegated/tests.rs @@ -4,9 +4,7 @@ use alloy_eips::BlockNumHash; use alloy_primitives::B256; use async_trait::async_trait; use kona_derive::test_utils::TestChainProvider; -use kona_engine::{ - ConsolidateInput, EngineState, EngineSyncState, EngineSyncStateUpdate, FinalizeBlockId, -}; +use kona_engine::{EngineState, EngineSyncState, EngineSyncStateUpdate, FinalizeBlockId}; use kona_protocol::{BlockInfo, L2BlockInfo, SyncStatus}; use rstest::rstest; use std::{collections::VecDeque, sync::Mutex}; @@ -115,9 +113,7 @@ impl From for Forwarded { fn from(request: EngineActorRequest) -> Self { match request { EngineActorRequest::ProcessFinalizedL2Block(finalized) => Self::Finalized(*finalized), - EngineActorRequest::ProcessSafeL2Signal(ConsolidateInput::BlockInfo(safe)) => { - Self::Safe(safe) - } + EngineActorRequest::ProcessDelegatedSafeHead(safe) => Self::Safe(*safe), other => Self::Other(format!("{other:?}")), } } diff --git a/rust/kona/crates/node/service/src/actors/derivation/engine_client.rs b/rust/kona/crates/node/service/src/actors/derivation/engine_client.rs index ad024f605ad..012fb5edc02 100644 --- a/rust/kona/crates/node/service/src/actors/derivation/engine_client.rs +++ b/rust/kona/crates/node/service/src/actors/derivation/engine_client.rs @@ -1,7 +1,8 @@ use crate::{EngineActorRequest, EngineClientError, EngineClientResult, ResetRequest}; use async_trait::async_trait; use derive_more::Constructor; -use kona_engine::{ConsolidateInput, FinalizeBlockId}; +use kona_engine::FinalizeBlockId; +use kona_protocol::{L2BlockInfo, OpAttributesWithParent}; use std::fmt::Debug; use tokio::sync::mpsc; @@ -16,14 +17,34 @@ pub trait DerivationEngineClient: Debug + Send + Sync { /// Note: This does not wait for the engine to process it. async fn send_finalized_l2_block(&self, block_id: FinalizeBlockId) -> EngineClientResult<()>; - /// Sends a consolidation signal to the engine. + /// Sends attributes from the derivation pipeline to the engine for consolidation. /// - /// This is the unified entry point for all consolidation-related inputs, - /// including derived attributes and safe L2 block information, as represented - /// by [`ConsolidateInput`]. + /// `reset_epoch` is the number of engine resets derivation has handled, so the engine can + /// drop attributes derived before its latest reset. /// /// Note: This does not wait for the engine to process it. - async fn send_safe_l2_signal(&self, signal: ConsolidateInput) -> EngineClientResult<()>; + async fn send_derived_attributes( + &self, + attributes: OpAttributesWithParent, + reset_epoch: u64, + ) -> EngineClientResult<()>; + + /// Asks the engine to finalize the L2 block at `number`, which derivation recorded as safe. + /// + /// `reset_epoch` is the number of engine resets derivation has handled, so the engine can + /// drop finalizations recorded before its latest reset. + /// + /// Note: This does not wait for the engine to process it. + async fn send_derived_finalization( + &self, + number: u64, + reset_epoch: u64, + ) -> EngineClientResult<()>; + + /// Sends the engine a safe block reported by a derivation delegate. + /// + /// Note: This does not wait for the engine to process it. + async fn send_delegated_safe_head(&self, safe_head: L2BlockInfo) -> EngineClientResult<()>; } /// Client to use to send messages to the Engine Actor's inbound channel. @@ -33,6 +54,15 @@ pub struct QueuedDerivationEngineClient { pub engine_actor_request_tx: mpsc::Sender, } +impl QueuedDerivationEngineClient { + async fn send(&self, request: EngineActorRequest) -> EngineClientResult<()> { + self.engine_actor_request_tx + .send(request) + .await + .map_err(|_| EngineClientError::RequestError("request channel closed.".to_string())) + } +} + #[async_trait] impl DerivationEngineClient for QueuedDerivationEngineClient { async fn reset_engine_forkchoice(&self) -> EngineClientResult<()> { @@ -58,12 +88,27 @@ impl DerivationEngineClient for QueuedDerivationEngineClient { Ok(()) } - async fn send_safe_l2_signal(&self, signal: ConsolidateInput) -> EngineClientResult<()> { - self.engine_actor_request_tx - .send(EngineActorRequest::ProcessSafeL2Signal(signal)) - .await - .map_err(|_| EngineClientError::RequestError("request channel closed.".to_string()))?; + async fn send_derived_attributes( + &self, + attributes: OpAttributesWithParent, + reset_epoch: u64, + ) -> EngineClientResult<()> { + self.send(EngineActorRequest::ProcessDerivedAttributes { + attributes: Box::new(attributes), + reset_epoch, + }) + .await + } - Ok(()) + async fn send_derived_finalization( + &self, + number: u64, + reset_epoch: u64, + ) -> EngineClientResult<()> { + self.send(EngineActorRequest::ProcessDerivedFinalization { number, reset_epoch }).await + } + + async fn send_delegated_safe_head(&self, safe_head: L2BlockInfo) -> EngineClientResult<()> { + self.send(EngineActorRequest::ProcessDelegatedSafeHead(Box::new(safe_head))).await } } diff --git a/rust/kona/crates/node/service/src/actors/engine/actor.rs b/rust/kona/crates/node/service/src/actors/engine/actor.rs index 080fcab3598..72bea37a497 100644 --- a/rust/kona/crates/node/service/src/actors/engine/actor.rs +++ b/rust/kona/crates/node/service/src/actors/engine/actor.rs @@ -4,9 +4,9 @@ use crate::{ use async_trait::async_trait; use kona_derive::{ResetSignal, Signal}; use kona_engine::{ - BuildTask, CanonicalizeTask, ConsolidateTask, Engine, EngineActorRequest, EngineClient, - EngineTask, EngineTaskError, EngineTaskErrorSeverity, EngineTaskErrors, FinalizeTask, - ImportedBlockSink, InsertTask, SealTask, + BuildTask, CanonicalizeTask, ConsolidateInput, ConsolidateTask, Engine, EngineActorRequest, + EngineClient, EngineTask, EngineTaskError, EngineTaskErrorSeverity, EngineTaskErrors, + FinalizeBlockId, FinalizeTask, ImportedBlockSink, InsertTask, SealTask, }; use kona_genesis::RollupConfig; use kona_protocol::L2BlockInfo; @@ -37,6 +37,12 @@ where /// The error of the last skipped engine task, so an identical repeat logs no new warning /// until a reset changes the state the task was judged against. last_skipped_task_error: Option, + /// How many reset signals have been sent to the [`crate::DerivationActor`]. Derivation counts + /// the reset signals it handles and stamps its attributes and finalizations with that count. + /// The channel is FIFO, a failed send stops this actor, and the node stops every actor when + /// one exits, so both counts start together and a lower count means the work was derived + /// before the latest reset. + reset_epoch: u64, /// A channel to use to relay the current unsafe head. /// ## Note /// This is `Some` when the node is in sequencer mode, and `None` when the node is in validator @@ -85,6 +91,7 @@ where engine, last_engine_safe_head_sent: L2BlockInfo::default(), last_skipped_task_error: None, + reset_epoch: 0, rollup: config, unsafe_head_tx, inbound_request_rx, @@ -151,6 +158,10 @@ where } else { self.engine.reset(self.client.clone(), self.rollup.clone()).await? }; + // Requests received while the reset was pending were queued behind it. Derivation resets + // too, so work it derived before the reset is stale. Work still on its way carries an + // older reset epoch and is dropped when it arrives. + self.engine.drop_derived_tasks(); // The sequencer must observe the same reset head as RPC readers before its reset // acknowledgement can release a queued start command. self.publish_unsafe_head(); @@ -165,7 +176,10 @@ where // Signal the derivation actor to reset. let signal = Signal::Reset(ResetSignal { l2_safe_head }); match self.derivation_client.send_signal(signal).await { - Ok(_) => info!(target: "engine", "Sent reset signal to derivation actor"), + Ok(_) => { + self.reset_epoch += 1; + info!(target: "engine", reset_epoch = self.reset_epoch, "Sent reset signal to derivation actor"); + } Err(err) => { error!(target: "engine", ?err, "Failed to send reset signal to the derivation actor"); return Err(EngineError::ChannelClosed); @@ -209,6 +223,9 @@ where } EngineTaskErrorSeverity::Reset => { warn!(target: "engine", ?err, "Received reset request"); + // Tasks that succeeded earlier in this drain may have moved the local-safe + // head. Report it before the reset, or derivation never records it. + self.send_derivation_actor_safe_head_if_updated().await?; self.begin_reset(); return Ok(()); } @@ -231,6 +248,9 @@ where } EngineTaskErrorSeverity::Temporary => { trace!(target: "engine", ?err, "Temporary error draining engine tasks"); + // Report a local-safe head that earlier tasks in this drain moved, + // rather than delaying it until the retry succeeds. + self.send_derivation_actor_safe_head_if_updated().await?; self.defer_retry(); return Ok(()); } @@ -267,10 +287,40 @@ where })? } + /// Whether derivation sent this work before handling the latest reset signal. + fn is_stale(&self, reset_epoch: u64, work: &str) -> bool { + debug_assert!( + reset_epoch <= self.reset_epoch, + "derivation counted {reset_epoch} engine resets, but only {} were sent", + self.reset_epoch + ); + let stale = reset_epoch < self.reset_epoch; + if stale { + warn!( + target: "engine", + work, + reset_epoch, + current = self.reset_epoch, + "Dropping derived work from before the latest reset" + ); + } + stale + } + + fn enqueue_consolidation(&mut self, input: ConsolidateInput) { + self.engine.enqueue(EngineTask::Consolidate(Box::new(ConsolidateTask::new( + self.client.clone(), + self.rollup.clone(), + input, + Arc::clone(&self.block_sink), + )))); + } + /// Sends the [`crate::DerivationActor`] the local-safe head if it changed since the last send. /// /// Derivation continues from, and records, the head that L1 data made safe. That is the - /// local-safe label; the safe label can lag it by cross-chain verification. + /// local-safe label; the safe label can lag it by cross-chain verification. Consolidating + /// derived attributes always advances it, because they must extend it. async fn send_derivation_actor_safe_head_if_updated(&mut self) -> Result<(), EngineError> { let local_safe_head = self.engine.state().sync_state.local_safe_head(); if local_safe_head == self.last_engine_safe_head_sent { @@ -332,14 +382,24 @@ where ))); self.engine.enqueue(task); } - EngineActorRequest::ProcessSafeL2Signal(safe_signal) => { - let task = EngineTask::Consolidate(Box::new(ConsolidateTask::new( + EngineActorRequest::ProcessDerivedAttributes { attributes, reset_epoch } => { + if self.is_stale(reset_epoch, "attributes") { + return Ok(()); + } + self.enqueue_consolidation(ConsolidateInput::Attributes(attributes)); + } + EngineActorRequest::ProcessDerivedFinalization { number, reset_epoch } => { + if self.is_stale(reset_epoch, "finalization") { + return Ok(()); + } + self.engine.enqueue(EngineTask::Finalize(Box::new(FinalizeTask::new( self.client.clone(), self.rollup.clone(), - safe_signal, - Arc::clone(&self.block_sink), - ))); - self.engine.enqueue(task); + FinalizeBlockId::ByNumber(number), + )))); + } + EngineActorRequest::ProcessDelegatedSafeHead(safe_head) => { + self.enqueue_consolidation(ConsolidateInput::BlockInfo(*safe_head)); } EngineActorRequest::ProcessFinalizedL2Block(finalized_l2_block_id) => { // Finalize the L2 block identified by the provided [`FinalizeBlockId`]. diff --git a/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs b/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs index c7399efc6e6..2dd4e913a5b 100644 --- a/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs +++ b/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs @@ -1,5 +1,5 @@ use super::*; -use crate::QueuedEngineDerivationClient; +use crate::{DerivationActorRequest, QueuedEngineDerivationClient}; use alloy_primitives::B256; use alloy_rpc_types_engine::{ ExecutionPayloadV1, ForkchoiceState, PayloadStatus, PayloadStatusEnum, @@ -118,9 +118,7 @@ async fn delegated_safe_head_is_sent_on_every_poll() { // accepts the next poll. for _ in 0..3 { requests - .send(EngineActorRequest::ProcessSafeL2Signal(ConsolidateInput::BlockInfo( - delegated_safe_head, - ))) + .send(EngineActorRequest::ProcessDelegatedSafeHead(Box::new(delegated_safe_head))) .await .unwrap(); time::timeout(Duration::from_secs(1), actor.step()).await.unwrap().unwrap(); @@ -175,9 +173,7 @@ async fn delegated_poll_during_el_sync_moves_safe_labels_and_skips_finalization( // One poll: the safe head, then the finalized block pinned by hash. Each step drains the // previous request's task and then accepts the next request. requests - .send(EngineActorRequest::ProcessSafeL2Signal(ConsolidateInput::BlockInfo( - delegated_safe_head, - ))) + .send(EngineActorRequest::ProcessDelegatedSafeHead(Box::new(delegated_safe_head))) .await .unwrap(); requests @@ -320,7 +316,10 @@ async fn finalization_target_survives_resets_below_it(#[case] target_kind: Targe requests.send(target_kind.finalize_request(target)).await.unwrap(); actor.step().await.unwrap(); assert!(second_reset_result.try_recv().unwrap().is_ok()); - requests.send(EngineActorRequest::ProcessSafeL2Signal(restored_safe.into())).await.unwrap(); + requests + .send(EngineActorRequest::ProcessDelegatedSafeHead(Box::new(restored_safe))) + .await + .unwrap(); actor.step().await.unwrap(); assert_eq!(actor.engine.len(), 1, "only the safe head update may remain queued"); assert!(!actor.reset_pending); @@ -390,3 +389,94 @@ async fn skipped_finalization_keeps_the_actor_running_until_a_reset_forgets_it() l1.assert_finished(); l2.assert_finished(); } + +/// Derivation can send attributes and finalizations from its old pipeline after the engine +/// reset, while the reset signal is still queued for it. They carry the reset epoch derivation +/// had then, and are dropped on arrival instead of running against the reset head. +#[tokio::test] +async fn work_derived_before_the_latest_reset_is_dropped() { + let config = Arc::new(RollupConfig::default()); + let (client, _l1, _l2) = test_engine_client(config.clone()); + let head = test_block_info(5); + let state = TestEngineStateBuilder::new().with_unsafe_head(head).build(); + let (state_tx, _state_rx) = watch::channel(state); + let (requests, request_rx) = mpsc::channel(2); + let (derivation_tx, _derivation_rx) = mpsc::channel(2); + let mut actor = EngineActor::new( + Arc::new(client), + config, + QueuedEngineDerivationClient { derivation_actor_request_tx: derivation_tx }, + Engine::new(state, state_tx), + None, + request_rx, + Arc::new(NoopBlockSink), + ); + actor.reset_epoch = 2; + let attributes = |reset_epoch| EngineActorRequest::ProcessDerivedAttributes { + attributes: Box::new( + kona_engine::test_utils::TestAttributesBuilder::new().with_parent(head).build(), + ), + reset_epoch, + }; + let finalization = + |reset_epoch| EngineActorRequest::ProcessDerivedFinalization { number: 5, reset_epoch }; + + for request in [attributes(1), finalization(1)] { + requests.send(request).await.unwrap(); + actor.step().await.unwrap(); + assert!(actor.engine.is_empty()); + } + // Hold the queue so the accepted work is counted, not run. + actor.retry_at = Instant::now() + Duration::from_secs(3600); + for (request, queued) in [(attributes(2), 1), (finalization(2), 2)] { + requests.send(request).await.unwrap(); + actor.step().await.unwrap(); + assert_eq!(actor.engine.len(), queued); + } +} + +/// A drain can move the local-safe head and then hit a task that asks for a reset. The new head +/// is reported before the reset starts, or derivation would never record it. +#[tokio::test] +async fn local_safe_head_is_reported_before_a_reset() { + let config = Arc::new(RollupConfig::default()); + let (client, _l1, _l2) = test_engine_client(config.clone()); + let client = Arc::new(client); + let head = test_block_info(5); + let state = TestEngineStateBuilder::new().with_unsafe_head(head).build(); + let (state_tx, _state_rx) = watch::channel(state); + let (_requests, request_rx) = mpsc::channel(1); + let (derivation_tx, mut derivation_rx) = mpsc::channel(2); + let mut actor = EngineActor::new( + client.clone(), + config.clone(), + QueuedEngineDerivationClient { derivation_actor_request_tx: derivation_tx }, + Engine::new(state, state_tx), + None, + request_rx, + Arc::new(NoopBlockSink), + ); + // As if earlier tasks in this drain moved the local-safe head past the last head sent. + actor.last_engine_safe_head_sent = test_block_info(4); + // Attributes that don't extend the local-safe head ask for a reset without an engine call. + actor.engine.enqueue(EngineTask::Consolidate(Box::new(ConsolidateTask::new( + client, + config, + ConsolidateInput::Attributes(Box::new( + kona_engine::test_utils::TestAttributesBuilder::new() + .with_parent(test_block_info(9)) + .build(), + )), + Arc::new(NoopBlockSink), + )))); + + actor.drain().await.unwrap(); + + assert!(actor.reset_pending); + let Ok(DerivationActorRequest::ProcessEngineSafeHeadUpdateRequest(sent)) = + derivation_rx.try_recv() + else { + panic!("expected the local-safe head before the reset"); + }; + assert_eq!(*sent, head); +} diff --git a/rust/kona/crates/node/service/src/actors/engine/tests.rs b/rust/kona/crates/node/service/src/actors/engine/tests.rs index 68a33db2828..5a8f5bd020b 100644 --- a/rust/kona/crates/node/service/src/actors/engine/tests.rs +++ b/rust/kona/crates/node/service/src/actors/engine/tests.rs @@ -47,7 +47,9 @@ async fn invalid_forkchoice_resets_and_resumes(#[case] rebuild: bool, #[case] ho let state = EngineState { sync_state: EngineState::default().sync_state.apply_update(EngineSyncStateUpdate { unsafe_head: Some(l2_block(10)), - local_safe_head: Some(genesis), + // Derivation builds its attributes on the local-safe head; the engine drops attributes + // that don't extend it. + local_safe_head: Some(if rebuild { l2_block(9) } else { genesis }), safe_head: Some(genesis), finalized_head: Some(genesis), }), @@ -67,18 +69,22 @@ async fn invalid_forkchoice_resets_and_resumes(#[case] rebuild: bool, #[case] ho request_rx, Arc::new(NoopBlockSink), ); - let signal = if rebuild { + let request = if rebuild { // The queued attributes still build on the rejected branch. Reset must reload the // canonical EL head instead of retrying this parent with different transactions. - TestAttributesBuilder::new() - .with_parent(l2_block(9)) - .with_transactions(vec![alloy_primitives::Bytes::from_static(&[0x02])]) - .build() - .into() + EngineActorRequest::ProcessDerivedAttributes { + attributes: Box::new( + TestAttributesBuilder::new() + .with_parent(l2_block(9)) + .with_transactions(vec![alloy_primitives::Bytes::from_static(&[0x02])]) + .build(), + ), + reset_epoch: 0, + } } else { - l2_block(1).into() + EngineActorRequest::ProcessDelegatedSafeHead(Box::new(l2_block(1))) }; - requests.send(EngineActorRequest::ProcessSafeL2Signal(signal)).await.unwrap(); + requests.send(request).await.unwrap(); actor.step().await.unwrap(); actor.step().await.unwrap(); assert_eq!(state_rx.borrow().sync_state.safe_head(), genesis); @@ -101,7 +107,7 @@ async fn invalid_forkchoice_resets_and_resumes(#[case] rebuild: bool, #[case] ho ForkchoiceUpdated::new(PayloadStatus::from_status(PayloadStatusEnum::Valid)), ); let recovered = l2_block(2); - requests.send(EngineActorRequest::ProcessSafeL2Signal(recovered.into())).await.unwrap(); + requests.send(EngineActorRequest::ProcessDelegatedSafeHead(Box::new(recovered))).await.unwrap(); actor.step().await.unwrap(); assert_eq!(state_rx.borrow().sync_state.unsafe_head(), genesis, "publish the reset head"); assert_eq!(state_rx.borrow().sync_state.safe_head(), genesis);