diff --git a/docs/public-docs/node-operators/kona-node/configuration.mdx b/docs/public-docs/node-operators/kona-node/configuration.mdx index 30db30bd372..bad550822a4 100644 --- a/docs/public-docs/node-operators/kona-node/configuration.mdx +++ b/docs/public-docs/node-operators/kona-node/configuration.mdx @@ -27,6 +27,7 @@ For more details on each flag, see the inline help (`kona-node node --help`) or | `--l1-beacon ` | `KONA_NODE_L1_BEACON` | URL of the L1 beacon API | Yes | - | | `--l2-engine-rpc ` | `KONA_NODE_L2_ENGINE_RPC` | URL of the engine API endpoint of an L2 execution client | Yes | - | | `--l2-engine-jwt-secret ` | `KONA_NODE_L2_ENGINE_AUTH` | Path to file containing the hex-encoded JWT secret for the execution client | No | - | +| `--l2-el-sync-stall-timeout ` | `KONA_NODE_L2_EL_SYNC_STALL_TIMEOUT` | How long to wait for a missing derived block while the execution client answers `SYNCING` and neither its reported sync progress nor its latest block moves, before resetting. Execution clients report no progress while they download headers, which can take hours | No | `86400` | | `--l2-config-file ` | `KONA_NODE_ROLLUP_CONFIG` | Path to a custom L2 rollup configuration file | No | - | | `--l1-runtime-config-reload-interval ` | `KONA_NODE_L1_RUNTIME_CONFIG_RELOAD_INTERVAL` | Poll interval for reloading runtime config | No | `600` | diff --git a/rust/kona/bin/node/src/commands/node/command.rs b/rust/kona/bin/node/src/commands/node/command.rs index b96712488c6..0faa36cd4f4 100644 --- a/rust/kona/bin/node/src/commands/node/command.rs +++ b/rust/kona/bin/node/src/commands/node/command.rs @@ -336,8 +336,12 @@ impl NodeCommand { let p2p_config = self.p2p_flags.clone().config(&cfg, args, Some(&l1_provider)).await?; let rpc_config = self.rpc_flags.clone().into(); - let engine_config = - EngineConfig { config: Arc::new(cfg.clone()), mode: self.node_mode, provider: engine }; + let engine_config = EngineConfig { + config: Arc::new(cfg.clone()), + el_sync_stall_timeout: self.l2_client_args.el_sync_stall_timeout(), + mode: self.node_mode, + provider: engine, + }; let dependency_set = self.load_dependency_set(&cfg)?; let safe_db = self.open_safe_db()?; diff --git a/rust/kona/bin/node/src/flags/engine/providers.rs b/rust/kona/bin/node/src/flags/engine/providers.rs index 20b434fb96a..f8217cb3011 100644 --- a/rust/kona/bin/node/src/flags/engine/providers.rs +++ b/rust/kona/bin/node/src/flags/engine/providers.rs @@ -1,9 +1,11 @@ use alloy_rpc_types_engine::JwtSecret; -use std::path::PathBuf; +use kona_engine::DEFAULT_EL_SYNC_STALL_TIMEOUT; +use std::{path::PathBuf, time::Duration}; use url::Url; -const DEFAULT_L2_ENGINE_TIMEOUT: u64 = 30_000; const DEFAULT_L1_TRUST_RPC: bool = false; +const DEFAULT_L2_EL_SYNC_STALL_TIMEOUT: u64 = DEFAULT_EL_SYNC_STALL_TIMEOUT.as_secs(); +const DEFAULT_L2_ENGINE_TIMEOUT: u64 = 30_000; /// L1 client arguments. #[derive(Clone, Debug, clap::Args)] @@ -73,6 +75,22 @@ pub struct L2ClientArgs { default_value_t = DEFAULT_L2_ENGINE_TIMEOUT )] pub l2_engine_timeout: u64, + /// Seconds to wait for a missing derived block while the execution layer answers `SYNCING` + /// and neither its reported sync progress nor its latest block moves, before resetting. + #[arg( + long, + visible_alias = "l2.el-sync-stall-timeout", + env = "KONA_NODE_L2_EL_SYNC_STALL_TIMEOUT", + default_value_t = DEFAULT_L2_EL_SYNC_STALL_TIMEOUT + )] + pub l2_el_sync_stall_timeout: u64, +} + +impl L2ClientArgs { + /// How long a consolidation waits for an execution layer whose sync reports no progress. + pub const fn el_sync_stall_timeout(&self) -> Duration { + Duration::from_secs(self.l2_el_sync_stall_timeout) + } } impl Default for L2ClientArgs { @@ -82,6 +100,7 @@ impl Default for L2ClientArgs { l2_engine_jwt_secret: None, l2_engine_jwt_encoded: None, l2_engine_timeout: DEFAULT_L2_ENGINE_TIMEOUT, + l2_el_sync_stall_timeout: DEFAULT_L2_EL_SYNC_STALL_TIMEOUT, } } } @@ -136,6 +155,20 @@ mod tests { assert_eq!(parsed.l1.l1_trust_rpc, l1); } + #[rstest] + #[case::default(&[], DEFAULT_EL_SYNC_STALL_TIMEOUT)] + #[case::dotted_alias(&["--l2.el-sync-stall-timeout=600"], Duration::from_secs(600))] + #[case::long_flag(&["--l2-el-sync-stall-timeout", "7200"], Duration::from_secs(7200))] + fn test_parse_el_sync_stall_timeout(#[case] flags: &[&str], #[case] expected: Duration) { + // Keep these CLI/default tests independent of the operator's environment. + let command = + ClientArgs::command().mut_arg("l2_el_sync_stall_timeout", |arg| arg.env(None::<&str>)); + let args = [CLIENT_ARGS, flags].concat(); + let matches = command.try_get_matches_from(args).unwrap(); + let parsed = ClientArgs::from_arg_matches(&matches).unwrap(); + assert_eq!(parsed.l2.el_sync_stall_timeout(), expected); + } + #[test] fn test_default_l1_trust_rpc() { assert!(!L1ClientArgs::default().l1_trust_rpc); diff --git a/rust/kona/crates/node/engine/src/lib.rs b/rust/kona/crates/node/engine/src/lib.rs index eb26238be30..990a21e2624 100644 --- a/rust/kona/crates/node/engine/src/lib.rs +++ b/rust/kona/crates/node/engine/src/lib.rs @@ -40,10 +40,10 @@ extern crate tracing; mod task_queue; pub use task_queue::{ BuildTask, BuildTaskError, CanonicalizeTask, CanonicalizeTaskError, ConsolidateInput, - ConsolidateTask, ConsolidateTaskError, Engine, EngineBuildError, EngineResetError, EngineTask, - EngineTaskError, EngineTaskErrorSeverity, EngineTaskErrors, ExecutableEngineTask, - FinalizeBlockId, FinalizeTask, FinalizeTaskError, InsertTask, InsertTaskError, SealTask, - SealTaskError, SynchronizeTaskError, + ConsolidateTask, ConsolidateTaskError, DEFAULT_EL_SYNC_STALL_TIMEOUT, DrainScope, Engine, + EngineBuildError, EngineResetError, EngineTask, EngineTaskError, EngineTaskErrorSeverity, + EngineTaskErrors, ExecutableEngineTask, FinalizeBlockId, FinalizeTask, FinalizeTaskError, + InsertTask, InsertTaskError, SealTask, SealTaskError, SynchronizeTaskError, }; mod attributes; diff --git a/rust/kona/crates/node/engine/src/task_queue/core.rs b/rust/kona/crates/node/engine/src/task_queue/core.rs index 65ef7bdc252..14b8364113e 100644 --- a/rust/kona/crates/node/engine/src/task_queue/core.rs +++ b/rust/kona/crates/node/engine/src/task_queue/core.rs @@ -49,8 +49,11 @@ impl Ord for QueuedTask { /// /// Tasks within the queue are also considered fallible. If they fail with a temporary error, /// they are not popped from the queue, the error is returned, and they are retried on the -/// next call to [`Engine::drain`]. A task that fails with a flush error has already replaced its -/// block, so it is popped before the error is returned. +/// next call to [`Engine::drain`]. A task that fails with a wait error is parked instead: until its +/// retry, a drain of [`DrainScope::AheadOfWaitingTask`] runs only tasks of other kinds, while tasks +/// of its own kind keep their place behind it. A task +/// that fails with a flush error has already replaced its block, so it is popped before the error +/// is returned. #[derive(Debug)] pub struct Engine { /// The state of the engine. @@ -62,7 +65,9 @@ pub struct Engine { next_sequence: u64, /// Preserve the task being retried ahead of newly enqueued work: it may have already /// performed part of an Engine API operation. - active: Option, + active: Option, + /// The task that last failed with a wait error, parked until the next [`Engine::drain`]. + waiting: Option, } impl Engine { @@ -74,6 +79,7 @@ impl Engine { tasks: BinaryHeap::default(), active: None, next_sequence: 0, + waiting: None, } } @@ -95,9 +101,55 @@ impl Engine { self.tasks.push(QueuedTask { sequence, task }); } - /// Number of queued and active tasks, including work awaiting a retry. + /// Number of queued, active and waiting tasks, including work awaiting a retry. pub fn len(&self) -> usize { - self.tasks.len() + usize::from(self.active.is_some()) + self.tasks.len() + usize::from(self.active.is_some()) + usize::from(self.waiting.is_some()) + } + + /// Whether a task is parked until its retry. + pub const fn has_waiting_task(&self) -> bool { + self.waiting.is_some() + } + + /// Whether a drain of [`DrainScope::AheadOfWaitingTask`] has a task to run: a task being + /// retried, or a queued task of another kind than the waiting task, or any queued task without + /// one. + pub fn has_work_ahead_of_waiting_task(&self) -> bool { + self.active.is_some() || + self.waiting.as_ref().map_or(!self.tasks.is_empty(), |waiting| { + self.tasks.iter().any(|queued| !is_same_kind(queued, waiting)) + }) + } + + /// Pops the next queued task that may run. Before the waiting task's retry, tasks of its kind + /// stay queued behind it, in their order, while tasks of other kinds run: a finalization + /// queued behind a consolidation that waits for the execution layer would otherwise wait as + /// long as its sync does. + fn pop_runnable_task(&mut self) -> Option { + let Some(waiting) = &self.waiting else { return self.tasks.pop() }; + let mut held_behind_waiting = Vec::new(); + let mut runnable = None; + while let Some(next) = self.tasks.pop() { + if is_same_kind(&next, waiting) { + held_behind_waiting.push(next); + } else { + runnable = Some(next); + break; + } + } + self.tasks.extend(held_behind_waiting); + runnable + } + + /// Parks `task` until its retry. A task parked before it goes back into the queue with its + /// sequence. + fn park(&mut self, task: QueuedTask) { + self.tasks.extend(self.waiting.replace(task)); + } + + /// Puts the waiting task back into the queue with its sequence, to be retried in its turn. + fn unpark(&mut self) { + self.tasks.extend(self.waiting.take()); } /// Whether there is no pending work. @@ -140,30 +192,42 @@ impl Engine { pub fn clear(&mut self) { self.tasks.clear(); self.active = None; + self.waiting = None; } - /// Attempts to drain the queue by executing all [`EngineTask`]s in-order. If any task returns - /// an error along the way, the error is returned and the task is not popped from the queue (in - /// case it must be retried), unless it is a flush or skip error. - pub async fn drain(&mut self) -> Result<(), EngineTaskErrors> { + /// Attempts to drain the queue by executing the [`EngineTask`]s that `scope` allows, in order. + /// If any task returns an error along the way, the error is returned and the task is not popped + /// from the queue (in case it must be retried), unless it is a flush or skip error. A waiting + /// task is retried only once every queued task of another kind has run, so a failure among + /// those leaves it parked; if it still waits, it is parked again. + pub async fn drain(&mut self, scope: DrainScope) -> Result<(), EngineTaskErrors> { // Drain tasks in order of priority, halting on errors for a retry to be attempted. loop { if self.active.is_none() { - self.active = self.tasks.pop().map(|queued| queued.task); + self.active = self.next_task(scope); } - let Some(task) = &self.active else { break }; - let result = task.execute(&mut self.state).await; - - // A flush reports that the task replaced its block with a deposits-only block, so the - // task is done; retrying it would replace that block again. A skip reports a task that - // no longer applies, so retrying it cannot succeed either. - if let Err(err) = &result && - !matches!( - err.severity(), - EngineTaskErrorSeverity::Flush | EngineTaskErrorSeverity::Skip - ) - { - return result; + let Some(active) = &self.active else { break }; + let result = active.task.execute(&mut self.state).await; + + if let Err(err) = &result { + match err.severity() { + // A waiting task leaves the engine state unchanged. Parked with its sequence, + // it keeps its place among same-kind tasks but no longer blocks + // higher-priority work enqueued before its retry. + EngineTaskErrorSeverity::Wait => { + if let Some(waiting) = self.active.take() { + self.park(waiting); + } + return result; + } + // A flush reports that the task replaced its block with a deposits-only block, + // so the task is done; retrying it would replace that block again. A skip + // reports a task that no longer applies, so retrying it cannot succeed either. + EngineTaskErrorSeverity::Flush | EngineTaskErrorSeverity::Skip => {} + EngineTaskErrorSeverity::Critical | + EngineTaskErrorSeverity::Reset | + EngineTaskErrorSeverity::Temporary => return result, + } } self.state_sender.send_replace(self.state); @@ -173,6 +237,34 @@ impl Engine { Ok(()) } + + /// The next task to run: a queued task of another kind than the waiting task, and once none is + /// left, for [`DrainScope::AllTasks`], the waiting task in its turn. + fn next_task(&mut self, scope: DrainScope) -> Option { + if let Some(task) = self.pop_runnable_task() { + return Some(task); + } + if scope == DrainScope::AllTasks && self.waiting.is_some() { + self.unpark(); + return self.tasks.pop(); + } + None + } +} + +/// Whether two queued tasks are of the same [`EngineTask`] kind, which the queue orders alike. +fn is_same_kind(a: &QueuedTask, b: &QueuedTask) -> bool { + a.task.cmp(&b.task) == Ordering::Equal +} + +/// Which tasks an [`Engine::drain`] runs. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DrainScope { + /// Only tasks of other kinds than the waiting task, which stays parked. Without a waiting + /// task, every task. + AheadOfWaitingTask, + /// Every task, retrying the waiting task in its turn. + AllTasks, } /// An error occurred while attempting to reset the [`Engine`]. @@ -242,10 +334,13 @@ mod recovery_tests { Arc::new(NoopBlockSink), )))); // The old inline retry loop never returned from this call. - let error = tokio::time::timeout(std::time::Duration::from_secs(1), engine.drain()) - .await - .expect("temporary errors must yield to the actor") - .unwrap_err(); + let error = tokio::time::timeout( + std::time::Duration::from_secs(1), + engine.drain(DrainScope::AllTasks), + ) + .await + .expect("temporary errors must yield to the actor") + .unwrap_err(); assert_eq!(error.severity(), EngineTaskErrorSeverity::Temporary); assert_eq!(engine.len(), 1); // Accumulate additional unsafe imports during the outage. Equal-priority heap @@ -274,13 +369,55 @@ mod recovery_tests { ForkchoiceUpdated::new(PayloadStatus::from_status(PayloadStatusEnum::Valid)), ); } - engine.drain().await.unwrap(); + engine.drain(DrainScope::AllTasks).await.unwrap(); assert!(engine.is_empty()); assert_eq!(engine.state().sync_state.unsafe_head().block_info.number, 3); l1.assert_finished(); l2.assert_finished(); } + // A drain that may retry the waiting task runs the queued tasks of other kinds first. When one + // of them fails, the waiting task is not retried yet and stays parked for its own retry. + #[tokio::test] + async fn failure_ahead_of_the_waiting_task_leaves_it_parked() { + let payload = OpExecutionPayloadEnvelope::V1(ExecutionPayloadV1::from_block_slow( + &alloy_consensus::Block::::default(), + )); + let block: op_alloy_consensus::OpBlock = payload.clone().try_into_block().unwrap(); + let mut config = RollupConfig::default(); + config.genesis.l2.hash = block.header.hash_slow(); + let config = Arc::new(config); + let (client, l1, l2) = test_engine_client(config.clone()); + let client = Arc::new(client); + l2.expect_error("engine_newPayloadV2"); + let (state_tx, _state_rx) = tokio::sync::watch::channel(EngineState::default()); + let mut engine = Engine::new(EngineState::default(), state_tx); + engine.enqueue(EngineTask::Consolidate(Box::new(ConsolidateTask::new( + client.clone(), + config.clone(), + ConsolidateInput::Attributes(Box::new(TestAttributesBuilder::new().build())), + Arc::new(NoopBlockSink), + crate::DEFAULT_EL_SYNC_STALL_TIMEOUT, + )))); + let waiting = engine.pop_runnable_task().expect("the queued consolidation"); + engine.park(waiting); + engine.enqueue(EngineTask::Insert(Box::new(InsertTask::new( + client, + config, + payload, + false, + Arc::new(NoopBlockSink), + )))); + + let error = engine.drain(DrainScope::AllTasks).await.unwrap_err(); + + assert_eq!(error.severity(), EngineTaskErrorSeverity::Temporary, "got {error:?}"); + assert!(engine.has_waiting_task()); + assert_eq!(engine.len(), 2); + l1.assert_finished(); + l2.assert_finished(); + } + // A flush means the task already replaced its block with a deposits-only block. Keeping it // would rebuild that replacement on every later drain. #[tokio::test] @@ -346,9 +483,10 @@ mod recovery_tests { config, ConsolidateInput::Attributes(Box::new(attributes)), Arc::new(NoopBlockSink), + crate::DEFAULT_EL_SYNC_STALL_TIMEOUT, )))); - let error = engine.drain().await.unwrap_err(); + let error = engine.drain(DrainScope::AllTasks).await.unwrap_err(); assert_eq!(error.severity(), EngineTaskErrorSeverity::Flush, "got {error:?}"); l1.assert_finished(); l2.assert_finished(); @@ -404,7 +542,7 @@ mod recovery_tests { FinalizeBlockId::ByNumber(5), )))); - let error = engine.drain().await.unwrap_err(); + let error = engine.drain(DrainScope::AllTasks).await.unwrap_err(); assert_eq!(error.severity(), EngineTaskErrorSeverity::Skip, "got {error:?}"); assert!(engine.is_empty()); @@ -413,3 +551,166 @@ mod recovery_tests { l2.assert_finished(); } } + +#[cfg(test)] +mod scheduling_tests { + use super::*; + use crate::{ + ConsolidateInput, ConsolidateTask, FinalizeBlockId, FinalizeTask, InsertTask, + NoopBlockSink, + test_utils::{TestAttributesBuilder, test_engine_client}, + }; + use TaskKind::*; + use alloy_rpc_types_engine::ExecutionPayloadV1; + use op_alloy_rpc_types_engine::OpExecutionPayloadEnvelope; + + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + enum TaskKind { + Consolidate, + Finalize, + Insert, + } + + /// A queued task, as its kind and sequence. + type Scheduled = (TaskKind, u64); + + /// An engine whose queue holds tasks that are scheduled but never executed. + struct Scheduler { + client: Arc, + config: Arc, + engine: Engine, + } + + impl Scheduler { + fn new() -> Self { + let config = Arc::new(RollupConfig::default()); + let (client, _, _) = test_engine_client(config.clone()); + let (state_tx, _) = tokio::sync::watch::channel(EngineState::default()); + Self { + client: Arc::new(client), + config, + engine: Engine::new(EngineState::default(), state_tx), + } + } + + fn enqueue(&mut self, kind: TaskKind) { + let client = self.client.clone(); + let config = self.config.clone(); + self.engine.enqueue(match kind { + Consolidate => EngineTask::Consolidate(Box::new(ConsolidateTask::new( + client, + config, + ConsolidateInput::Attributes(Box::new(TestAttributesBuilder::new().build())), + Arc::new(NoopBlockSink), + crate::DEFAULT_EL_SYNC_STALL_TIMEOUT, + ))), + Finalize => EngineTask::Finalize(Box::new(FinalizeTask::new( + client, + config, + FinalizeBlockId::ByNumber(1), + ))), + Insert => EngineTask::Insert(Box::new(InsertTask::new( + client, + config, + OpExecutionPayloadEnvelope::V1(ExecutionPayloadV1::from_block_slow( + &alloy_consensus::Block::::default(), + )), + false, + Arc::new(NoopBlockSink), + ))), + }); + } + + /// Pops the next queued task, as a drain would, and parks it as a task that waits. + fn pop_and_park(&mut self) -> Scheduled { + let task = self.engine.pop_runnable_task().expect("a queued task"); + let scheduled = scheduled(&task); + self.engine.park(task); + scheduled + } + + /// The order in which a drain would run the queued tasks that may run now. + fn runnable_order(&mut self) -> Vec { + std::iter::from_fn(|| self.engine.pop_runnable_task()).map(|t| scheduled(&t)).collect() + } + + fn waiting(&self) -> Option { + self.engine.waiting.as_ref().map(scheduled) + } + } + + fn scheduled(queued: &QueuedTask) -> Scheduled { + let kind = match queued.task { + EngineTask::Consolidate(_) => Consolidate, + EngineTask::Finalize(_) => Finalize, + EngineTask::Insert(_) => Insert, + _ => unreachable!("only consolidate, finalize and insert tasks are scheduled"), + }; + (kind, queued.sequence) + } + + /// A consolidation parked while it waits (sequence 0) lets tasks of other kinds run before its + /// retry, imports and finalizations alike, then runs in its turn ahead of later + /// consolidations. + #[rstest::rstest] + #[case::insert_and_finalize_run_ahead(&[Finalize, Insert], &[(Insert, 2), (Finalize, 1)], &[(Consolidate, 0)])] + #[case::finalize_runs_ahead_of_later_consolidation(&[Consolidate, Finalize], &[(Finalize, 2)], &[(Consolidate, 0), (Consolidate, 1)])] + #[case::later_consolidation_stays_behind(&[Consolidate], &[], &[(Consolidate, 0), (Consolidate, 1)])] + #[case::inserts_run_ahead_in_order(&[Insert, Insert], &[(Insert, 1), (Insert, 2)], &[(Consolidate, 0)])] + fn waiting_task_keeps_its_place( + #[case] queued_after_parking: &[TaskKind], + #[case] expected_ahead_of_retry: &[Scheduled], + #[case] expected_at_retry: &[Scheduled], + ) { + let mut scheduler = Scheduler::new(); + scheduler.enqueue(Consolidate); + scheduler.pop_and_park(); + for kind in queued_after_parking { + scheduler.enqueue(*kind); + } + + let ahead_of_retry = scheduler.runnable_order(); + scheduler.engine.unpark(); + let at_retry = scheduler.runnable_order(); + + assert_eq!(ahead_of_retry, expected_ahead_of_retry); + assert_eq!(at_retry, expected_at_retry); + } + + /// A waiting task that waits again at its retry is parked again with its sequence, still ahead + /// of a later consolidation that does not run before it. + #[test] + fn task_waiting_again_is_parked_again_with_its_sequence() { + let mut scheduler = Scheduler::new(); + scheduler.enqueue(Consolidate); + scheduler.pop_and_park(); + scheduler.enqueue(Consolidate); + + scheduler.engine.unpark(); + let retried = scheduler.pop_and_park(); + + assert_eq!(retried, (Consolidate, 0)); + assert_eq!(scheduler.waiting(), Some((Consolidate, 0))); + assert_eq!(scheduler.runnable_order(), []); + scheduler.engine.unpark(); + assert_eq!(scheduler.runnable_order(), [(Consolidate, 0), (Consolidate, 1)]); + } + + /// A task that outranks the waiting one and waits too is parked in its place, and the earlier + /// one goes back into the queue with its sequence instead of being dropped. + #[test] + fn parking_a_task_requeues_the_one_waiting_before_it() { + let mut scheduler = Scheduler::new(); + scheduler.enqueue(Consolidate); + scheduler.pop_and_park(); + scheduler.enqueue(Insert); + + let parked_second = scheduler.pop_and_park(); + + assert_eq!(parked_second, (Insert, 1)); + assert_eq!(scheduler.waiting(), Some((Insert, 1))); + assert_eq!(scheduler.engine.len(), 2); + scheduler.engine.unpark(); + assert_eq!(scheduler.runnable_order(), [(Insert, 1), (Consolidate, 0)]); + } +} diff --git a/rust/kona/crates/node/engine/src/task_queue/mod.rs b/rust/kona/crates/node/engine/src/task_queue/mod.rs index 57fd6ee9907..3dc2799860a 100644 --- a/rust/kona/crates/node/engine/src/task_queue/mod.rs +++ b/rust/kona/crates/node/engine/src/task_queue/mod.rs @@ -1,7 +1,7 @@ //! The [`Engine`] task queue and the [`EngineTask`]s it can execute. mod core; -pub use core::{Engine, EngineResetError}; +pub use core::{DrainScope, Engine, EngineResetError}; mod tasks; pub use tasks::*; diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/el_sync_progress.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/el_sync_progress.rs new file mode 100644 index 00000000000..2a592f9bcd1 --- /dev/null +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/el_sync_progress.rs @@ -0,0 +1,306 @@ +//! Whether the execution layer makes progress while a consolidation waits for it to sync. + +use serde_json::{Map, Value}; +use std::{mem, time::Duration}; +use tokio::time::Instant; + +/// The default bound on how long a consolidation waiting for the execution layer to sync tolerates +/// no reported progress before the engine resets. +/// +/// Neither execution layer reports progress while it downloads headers: op-geth answers +/// `eth_syncing` with `false` until its beacon skeleton links to the local chain, and op-reth's +/// headers stage moves its checkpoint only once the whole range is downloaded. That download can +/// take hours on a large chain, and a reset during it brings back the reset loop the wait exists to +/// avoid, while a longer bound only delays the reset of an execution layer that can never fetch the +/// missing block. +pub const DEFAULT_EL_SYNC_STALL_TIMEOUT: Duration = Duration::from_secs(24 * 60 * 60); + +/// How often a waiting consolidation checks on the execution layer: it reads the sync progress and +/// resends the unchanged forkchoice to learn whether the execution layer now has the unsafe head. +/// The retries in between only look the derived block up, which is cheap for an execution layer +/// that is busy syncing. +pub(super) const CHECK_INTERVAL: Duration = Duration::from_secs(30); + +/// How often a waiting consolidation reports the execution layer's progress, or its lack of it. +pub(super) const PROGRESS_REPORT_INTERVAL: Duration = Duration::from_secs(5 * 60); + +/// What the execution layer reports about its sync on one check of a waiting consolidation. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) enum ElSyncReading { + /// `eth_syncing` reports no sync in progress, and this is the latest block number. + NotSyncing { + /// The number of the execution layer's latest block. + latest_number: u64, + }, + /// `eth_syncing` reports a sync in progress, with these progress fields. + Syncing(Map), +} + +impl ElSyncReading { + /// Whether the execution layer reports no sync while its latest block is `parent_number`: it + /// has the parent of the next block but fetches nothing toward the unsafe head. + pub(super) const fn is_idle_at(&self, parent_number: u64) -> bool { + matches!(self, Self::NotSyncing { latest_number } if *latest_number == parent_number) + } + + /// The progress in an `eth_syncing` status object: every field but the block the sync started + /// from, which never changes, and the block it syncs toward, which can move while the sync + /// itself makes no progress. + pub(super) fn from_sync_status(mut status: Map) -> Self { + status.remove("startingBlock"); + status.remove("highestBlock"); + Self::Syncing(status) + } +} + +/// Whether a consolidation keeps waiting for the execution layer to sync. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum ElSyncWaitOutcome { + /// The execution layer reported progress. + Progressed, + /// The execution layer has reported no progress for the stall timeout. + Stalled, + /// The wait starts. + Started, + /// The execution layer reported no progress, or could not be read, within the stall timeout. + Unchanged, +} + +/// What one check of a waiting consolidation found. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) struct ElSyncWaitStep { + /// Whether the wait goes on, starts, or has stalled. + pub(super) outcome: ElSyncWaitOutcome, + /// Whether to report the execution layer's progress, or its lack of it. + pub(super) reports_progress: bool, + /// How long the execution layer may still report no progress before the engine resets. + pub(super) time_until_reset: Duration, +} + +/// Tracks the execution layer's sync across the retries of a waiting consolidation. +#[derive(Debug)] +pub(super) struct ElSyncProgress { + /// When a check's forkchoice probe was last answered. + checked_at: Option, + /// When the wait started or the execution layer last reported progress. + last_progress_at: Option, + /// The first reading that could be read, or the last one that differed from it. + last_reading: Option, + /// When the execution layer's progress was last reported. + progress_reported_at: Option, + /// How long the execution layer may report no progress. + stall_timeout: Duration, +} + +impl ElSyncProgress { + /// Tracks a wait bounded by `stall_timeout`. + pub(super) const fn new(stall_timeout: Duration) -> Self { + Self { + checked_at: None, + last_progress_at: None, + last_reading: None, + progress_reported_at: None, + stall_timeout, + } + } + + /// Whether a retry at `now` checks on the execution layer: when the wait starts, every + /// [`CHECK_INTERVAL`] after the last answered check, and on the retry after a check whose + /// forkchoice probe failed. + pub(super) fn is_check_due(&self, now: Instant) -> bool { + self.checked_at.is_none_or(|checked_at| now.duration_since(checked_at) >= CHECK_INTERVAL) + } + + /// Records the execution layer's sync `reading` at `now`, `None` when it could not be read, + /// and returns what this check found. + /// + /// Any change of a reading counts as progress, including a lower latest block: an execution + /// layer whose datadir is wiped while it is awaited starts over from genesis. An unreadable + /// reading is no evidence either way, so it neither counts as progress nor replaces the last + /// reading, and the first reading that can be read is the baseline later ones are compared + /// with. Progress, or its lack, is reported when the wait starts, when a sync starts or + /// finishes, and otherwise at most every [`PROGRESS_REPORT_INTERVAL`]. + pub(super) fn record( + &mut self, + reading: Option, + now: Instant, + ) -> ElSyncWaitStep { + let Some(last_progress_at) = self.last_progress_at else { + self.last_progress_at = Some(now); + self.last_reading = reading; + self.progress_reported_at = Some(now); + return ElSyncWaitStep { + outcome: ElSyncWaitOutcome::Started, + reports_progress: true, + time_until_reset: self.stall_timeout, + }; + }; + + let outcome = match (&self.last_reading, reading) { + (None, Some(first_readable)) => { + self.last_reading = Some(first_readable); + self.unchanged_or_stalled(last_progress_at, now) + } + (Some(last), Some(reading)) if *last != reading => { + let sync_kind_changed = mem::discriminant(last) != mem::discriminant(&reading); + self.last_reading = Some(reading); + self.last_progress_at = Some(now); + if sync_kind_changed { + self.progress_reported_at = None; + } + ElSyncWaitOutcome::Progressed + } + _ => self.unchanged_or_stalled(last_progress_at, now), + }; + + let reports_progress = outcome != ElSyncWaitOutcome::Stalled && + self.progress_reported_at.is_none_or(|reported_at| { + now.duration_since(reported_at) >= PROGRESS_REPORT_INTERVAL + }); + if reports_progress { + self.progress_reported_at = Some(now); + } + let since_progress = now.duration_since(self.last_progress_at.unwrap_or(now)); + ElSyncWaitStep { + outcome, + reports_progress, + time_until_reset: self.stall_timeout.saturating_sub(since_progress), + } + } + + /// Whether a wait has started, which an answered check's reading does. + #[cfg(test)] + pub(super) const fn has_started(&self) -> bool { + self.last_progress_at.is_some() + } + + /// Records that the execution layer answered a check's forkchoice probe at `now`. A probe that + /// failed is not recorded, so the next retry checks again. + pub(super) const fn check_answered(&mut self, now: Instant) { + self.checked_at = Some(now); + } + + /// Whether the execution layer has reported no progress for the stall timeout at `now`. + fn unchanged_or_stalled(&self, last_progress_at: Instant, now: Instant) -> ElSyncWaitOutcome { + if now.duration_since(last_progress_at) >= self.stall_timeout { + ElSyncWaitOutcome::Stalled + } else { + ElSyncWaitOutcome::Unchanged + } + } +} + +#[cfg(test)] +mod tests { + use super::{ElSyncWaitOutcome::*, *}; + + const STALL_SECS: u64 = 3600; + const CHECK_SECS: u64 = CHECK_INTERVAL.as_secs(); + const REPORT_SECS: u64 = PROGRESS_REPORT_INTERVAL.as_secs(); + + /// A reading of an execution layer that is not syncing a range, at `latest_number`. + const fn at(latest_number: u64) -> Option { + Some(ElSyncReading::NotSyncing { latest_number }) + } + + /// A reading of an execution layer syncing toward `highest_block`, whose `stage` checkpoint is + /// at `stage_block`. + fn staged(stage_block: u64, highest_block: u64) -> Option { + let status = serde_json::json!({ + "currentBlock": "0x0", + "highestBlock": format!("{highest_block:#x}"), + "stages": [{ "name": "Headers", "block": format!("{stage_block:#x}") }], + "startingBlock": "0x0", + }); + let serde_json::Value::Object(status) = status else { unreachable!() }; + Some(ElSyncReading::from_sync_status(status)) + } + + /// Records each check, `(seconds since the previous one, reading)`, and returns the last one. + fn last_step(checks: &[(u64, Option)]) -> ElSyncWaitStep { + let mut progress = ElSyncProgress::new(Duration::from_secs(STALL_SECS)); + let mut now = Instant::now(); + let mut last = None; + for (elapsed_secs, reading) in checks { + now += Duration::from_secs(*elapsed_secs); + last = Some(progress.record(reading.clone(), now)); + } + last.expect("at least one check") + } + + #[rstest::rstest] + #[case::first_reading_starts_the_wait(&[(0, at(100))], Started)] + #[case::first_unreadable_reading_starts_the_wait(&[(0, None)], Started)] + #[case::unchanged_just_below_bound_waits(&[(0, at(100)), (STALL_SECS - 1, at(100))], Unchanged)] + #[case::unchanged_for_bound_stalls(&[(0, at(100)), (STALL_SECS, at(100))], Stalled)] + #[case::advancing_latest_block_progresses(&[(0, at(100)), (STALL_SECS - 1, at(101))], Progressed)] + #[case::stalls_after_progress(&[(0, at(100)), (STALL_SECS - 1, at(101)), (STALL_SECS, at(101))], Stalled)] + #[case::wiped_el_restarting_counts_as_progress(&[(0, at(100)), (STALL_SECS - 1, at(0))], Progressed)] + #[case::sync_starting_counts_as_progress(&[(0, at(100)), (STALL_SECS - 1, staged(10, 500))], Progressed)] + #[case::unchanged_sync_for_bound_stalls(&[(0, staged(10, 500)), (STALL_SECS, staged(10, 500))], Stalled)] + #[case::advancing_stage_progresses(&[(0, staged(10, 500)), (STALL_SECS - 1, staged(20, 500))], Progressed)] + #[case::moving_sync_target_alone_stalls(&[(0, staged(10, 500)), (STALL_SECS, staged(10, 600))], Stalled)] + #[case::unreadable_reading_is_not_progress(&[(0, at(100)), (STALL_SECS - 1, None), (1, at(100))], Stalled)] + #[case::unreadable_readings_stall(&[(0, None), (STALL_SECS, None)], Stalled)] + #[case::first_readable_reading_is_not_progress(&[(0, None), (1, at(100))], Unchanged)] + #[case::first_readable_reading_keeps_the_bound(&[(0, None), (1, at(100)), (STALL_SECS - 1, at(100))], Stalled)] + fn sync_progress_bounds_the_wait( + #[case] checks: &[(u64, Option)], + #[case] expected: ElSyncWaitOutcome, + ) { + assert_eq!(last_step(checks).outcome, expected); + } + + /// Whether a retry `elapsed_secs` after the wait starts checks on the execution layer, when + /// the first check's forkchoice probe was answered or failed. + #[rstest::rstest] + #[case::wait_start_checks(None, 0, true)] + #[case::retry_before_the_interval_does_not_check(Some(true), CHECK_SECS - 1, false)] + #[case::retry_after_the_interval_checks(Some(true), CHECK_SECS, true)] + #[case::failed_probe_checks_on_the_next_retry(Some(false), 2, true)] + fn checks_are_due_on_their_cadence( + #[case] first_check_answered: Option, + #[case] elapsed_secs: u64, + #[case] expected: bool, + ) { + let mut progress = ElSyncProgress::new(Duration::from_secs(STALL_SECS)); + let started_at = Instant::now(); + if let Some(answered) = first_check_answered { + progress.record(at(100), started_at); + if answered { + progress.check_answered(started_at); + } + } + assert_eq!(progress.is_check_due(started_at + Duration::from_secs(elapsed_secs)), expected); + } + + #[rstest::rstest] + #[case::wait_start_reports(&[(0, at(100))], true)] + #[case::progress_before_the_interval_is_quiet(&[(0, at(100)), (REPORT_SECS - 1, at(101))], false)] + #[case::progress_after_the_interval_reports(&[(0, at(100)), (REPORT_SECS, at(101))], true)] + #[case::no_progress_before_the_interval_is_quiet(&[(0, at(100)), (REPORT_SECS - 1, at(100))], false)] + #[case::no_progress_after_the_interval_reports(&[(0, at(100)), (REPORT_SECS, at(100))], true)] + #[case::unreadable_after_the_interval_reports(&[(0, at(100)), (REPORT_SECS, None)], true)] + #[case::sync_starting_reports(&[(0, at(100)), (1, staged(10, 500))], true)] + #[case::sync_finishing_reports(&[(0, staged(10, 500)), (1, at(500))], true)] + #[case::first_readable_reading_is_quiet(&[(0, None), (1, at(100))], false)] + #[case::stall_is_not_reported_as_progress(&[(0, at(100)), (STALL_SECS, at(100))], false)] + fn progress_is_reported_on_its_cadence( + #[case] checks: &[(u64, Option)], + #[case] expected: bool, + ) { + assert_eq!(last_step(checks).reports_progress, expected); + } + + #[rstest::rstest] + #[case::wait_start_has_the_whole_bound(&[(0, at(100))], STALL_SECS)] + #[case::no_progress_uses_up_the_bound(&[(0, at(100)), (100, at(100))], STALL_SECS - 100)] + #[case::progress_restores_the_bound(&[(0, at(100)), (100, at(101))], STALL_SECS)] + #[case::first_readable_reading_does_not_restore_the_bound(&[(0, None), (100, at(100))], STALL_SECS - 100)] + fn time_until_reset_counts_from_the_last_progress( + #[case] checks: &[(u64, Option)], + #[case] expected_secs: u64, + ) { + assert_eq!(last_step(checks).time_until_reset, Duration::from_secs(expected_secs)); + } +} diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/error.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/error.rs index 2ce1b187a76..8a7ef63879f 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/error.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/error.rs @@ -19,9 +19,17 @@ pub enum ConsolidateTaskError { /// The unsafe L2 block is missing. #[error("Unsafe L2 block is missing {0}")] MissingUnsafeL2Block(u64), + /// The unsafe L2 block is missing because the execution layer is still syncing toward the + /// unsafe head. + #[error("Unsafe L2 block {0} is missing while the execution layer syncs")] + MissingUnsafeL2BlockDuringElSync(u64), /// Failed to fetch the unsafe L2 block. #[error("Failed to fetch the unsafe L2 block")] FailedToFetchUnsafeL2Block, + /// The unchanged forkchoice update asking whether the execution layer still syncs toward the + /// unsafe head, for a missing unsafe L2 block, failed. + #[error("Forkchoice update for the missing unsafe L2 block failed: {0}")] + MissingUnsafeL2BlockProbeFailed(#[source] SynchronizeTaskError), /// Starting the build of a block from the payload attributes failed. #[error(transparent)] BuildTaskFailed(#[from] BuildTaskError), @@ -47,12 +55,19 @@ impl EngineTaskError for ConsolidateTaskError { match self { Self::BreaksFinality(_) => EngineTaskErrorSeverity::Skip, Self::MissingUnsafeL2Block(_) => EngineTaskErrorSeverity::Reset, + // Only an execution layer that answers but still syncs makes the task wait on a fixed + // interval. A failed RPC is retried on the growing failure backoff, so an outage is + // not polled at the wait's cadence; the tasks that outrank this one would hit the same + // failing execution layer anyway. Self::FailedToFetchDelegatedSafeBlock(_) | Self::FailedToFetchUnsafeL2Block => { EngineTaskErrorSeverity::Temporary } + Self::MissingUnsafeL2BlockDuringElSync(_) => EngineTaskErrorSeverity::Wait, Self::BuildTaskFailed(inner) => inner.severity(), Self::SealTaskFailed(inner) => inner.severity(), - Self::ForkchoiceUpdateFailed(inner) => inner.severity(), + Self::ForkchoiceUpdateFailed(inner) | Self::MissingUnsafeL2BlockProbeFailed(inner) => { + inner.severity() + } } } } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/mod.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/mod.rs index 21b125bd135..17f14044070 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/mod.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/mod.rs @@ -1,5 +1,8 @@ //! Consolidation Task +mod el_sync_progress; +pub use el_sync_progress::DEFAULT_EL_SYNC_STALL_TIMEOUT; + mod error; pub use error::ConsolidateTaskError; diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs index 00f31457729..012959918bd 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs @@ -1,16 +1,25 @@ //! A task to consolidate the engine state. +use super::el_sync_progress::{ElSyncProgress, ElSyncReading, ElSyncWaitOutcome, ElSyncWaitStep}; use crate::{ ConsolidateTaskError, EngineClient, EngineState, ExecutableEngineTask, ImportedBlockSink, state::EngineSyncStateUpdate, - task_queue::{build_and_import, synchronize, synchronize_unconditionally}, + task_queue::{ + ForkchoiceReply, build_and_import, record_forkchoice_reply, send_forkchoice, synchronize, + synchronize_unconditionally, + }, }; +use alloy_provider::Provider; use alloy_rpc_types_eth::Block; use async_trait::async_trait; use kona_genesis::RollupConfig; use kona_protocol::{L2BlockInfo, OpAttributesWithParent}; use op_alloy_rpc_types::Transaction; -use std::sync::Arc; +use std::{ + sync::{Arc, Mutex, MutexGuard, PoisonError}, + time::Duration, +}; +use tokio::time::Instant; /// Input for consolidation - either derived attributes or safe L2 block #[derive(Debug, Clone)] @@ -58,6 +67,15 @@ impl ConsolidateInput { } } +/// What a consolidation waiting for the execution layer does with a derived block it lacked. +#[derive(Debug)] +enum DerivedBlock { + /// Build the block from the derived attributes on its parent, which the execution layer has. + BuildFromAttributes, + /// Consolidate the block the execution layer committed since the lookup. + Committed(Box>), +} + /// The [`ConsolidateTask`] attempts to consolidate the engine state /// using the specified payload attributes or block info. #[derive(Debug, Clone)] @@ -70,17 +88,28 @@ pub struct ConsolidateTask { pub input: ConsolidateInput, /// Where to hand the decoded block once the engine has canonicalized it. pub block_sink: Arc, + /// The execution layer's progress across the retries of this task while it waits for the + /// execution layer to sync. + el_sync_progress: Arc>, } impl ConsolidateTask { - /// Creates a new [`ConsolidateTask`] with the specified input - pub const fn new( + /// Creates a new [`ConsolidateTask`] with the specified input. A wait for the execution layer + /// to sync resets the engine once its sync reports no progress for `el_sync_stall_timeout`. + pub fn new( client: Arc, cfg: Arc, input: ConsolidateInput, block_sink: Arc, + el_sync_stall_timeout: Duration, ) -> Self { - Self { client, cfg, input, block_sink } + let el_sync_progress = Arc::new(Mutex::new(ElSyncProgress::new(el_sync_stall_timeout))); + Self { block_sink, cfg, client, el_sync_progress, input } + } + + /// The execution layer's progress across the retries of this task. + fn el_sync_progress(&self) -> MutexGuard<'_, ElSyncProgress> { + self.el_sync_progress.lock().unwrap_or_else(PoisonError::into_inner) } /// This is used when the [`ConsolidateTask`] fails to consolidate the engine state @@ -268,6 +297,196 @@ impl ConsolidateTask { } } + /// Waits for the execution layer to sync a derived block that it did not have, at or below the + /// unsafe head, and returns the block if the execution layer has it after all. + /// + /// Retries between checks only rely on the block lookup that precedes this call. A check reads + /// the execution layer's sync progress and resends the engine's unchanged forkchoice: + /// + /// - `SYNCING`: the execution layer is still syncing toward the unsafe head and will fill the + /// block in, so the task waits. Resetting instead would restart from the execution layer's + /// own labels, which are genesis after a datadir wipe, and re-derive the chain in a loop. + /// - `VALID`: the execution layer has the unsafe head, so it has the block unless it lost it. + /// It may have committed the block since the lookup, so the block is looked up once more. + /// + /// An execution layer that answers `SYNCING` while it reports no sync and sits at the derived + /// block's parent is not fetching anything, as without peers, so the block is built from the + /// derived attributes instead. A `SYNCING` answer after the progress has stalled for the bound + /// resets the engine. Only a `VALID` answer changes the engine state, so a task that then waits + /// has changed nothing. + async fn derived_block_once_el_syncs( + &self, + state: &mut EngineState, + block_number: u64, + ) -> Result { + let unsafe_number = state.sync_state.unsafe_head().block_info.number; + if !self.el_sync_progress().is_check_due(Instant::now()) { + debug!( + target: "engine", + block_number, + unsafe_number, + "Still waiting for the execution layer to sync the derived block" + ); + return Err(ConsolidateTaskError::MissingUnsafeL2BlockDuringElSync(block_number)); + } + + let reading = self.el_sync_reading().await; + // Every check probes, a stalled one included: the execution layer may have reached the + // unsafe head since the last check, and then the block is consolidated instead of reset. + if self.el_has_unsafe_head(state, block_number).await? { + record_forkchoice_reply(state, ForkchoiceReply::Valid); + return self + .derived_block_committed_since_lookup(block_number) + .await + .map(|block| DerivedBlock::Committed(Box::new(block))); + } + // The reading counts, and its report goes out, only once the probe is answered. + let now = Instant::now(); + self.el_sync_progress().check_answered(now); + let step = self.el_sync_progress().record(reading.clone(), now); + if reading + .as_ref() + .is_some_and(|reading| reading.is_idle_at(block_number.saturating_sub(1))) + { + info!( + target: "engine", + block_number, + ?reading, + unsafe_number, + "Execution layer is idle at the derived block's parent; building the derived block" + ); + return Ok(DerivedBlock::BuildFromAttributes); + } + if step.outcome == ElSyncWaitOutcome::Stalled { + warn!( + target: "engine", + block_number, + ?reading, + unsafe_number, + "Execution layer stopped making progress toward the unsafe head; resetting" + ); + return Err(ConsolidateTaskError::MissingUnsafeL2Block(block_number)); + } + Self::log_wait_for_el_sync(step, reading.as_ref(), block_number, unsafe_number); + Err(ConsolidateTaskError::MissingUnsafeL2BlockDuringElSync(block_number)) + } + + /// Resends the engine's unchanged forkchoice and returns whether the execution layer applied + /// it, rather than syncing toward its head. + async fn el_has_unsafe_head( + &self, + state: &EngineState, + block_number: u64, + ) -> Result { + debug!(target: "engine", block_number, "Asking the execution layer whether it has the unsafe head"); + let reply = send_forkchoice(self.client.as_ref(), &state.sync_state).await.map_err(|err| { + warn!(target: "engine", ?err, block_number, "Forkchoice update for a missing derived block failed"); + ConsolidateTaskError::MissingUnsafeL2BlockProbeFailed(err) + })?; + debug!(target: "engine", block_number, ?reply, "Execution layer answered the forkchoice update"); + Ok(reply == ForkchoiceReply::Valid) + } + + /// Logs a retry that keeps waiting: at `info!` when the wait starts and when progress is + /// reported, at `warn!` when no progress is reported on that cadence, and at `debug!` + /// otherwise. + fn log_wait_for_el_sync( + step: ElSyncWaitStep, + reading: Option<&ElSyncReading>, + block_number: u64, + unsafe_number: u64, + ) { + let time_until_reset = step.time_until_reset; + match step.outcome { + ElSyncWaitOutcome::Started => info!( + target: "engine", + block_number, + ?reading, + ?time_until_reset, + unsafe_number, + "Execution layer is syncing toward the unsafe head; waiting for the derived block" + ), + ElSyncWaitOutcome::Progressed if step.reports_progress => info!( + target: "engine", + block_number, + ?reading, + ?time_until_reset, + unsafe_number, + "Execution layer sync is progressing; still waiting for the derived block" + ), + _ if step.reports_progress => warn!( + target: "engine", + block_number, + ?reading, + ?time_until_reset, + unsafe_number, + "Execution layer reports no sync progress; the engine resets if none comes in time" + ), + _ => debug!( + target: "engine", + block_number, + ?reading, + ?time_until_reset, + "Still waiting for the execution layer to sync the derived block" + ), + } + } + + /// What the execution layer reports about its sync: the progress of the range it syncs, or + /// its latest block number when it syncs none. `None` if that cannot be read. + async fn el_sync_reading(&self) -> Option { + trace!(target: "engine", "Fetching the execution layer's sync status"); + match self.client.client().request_noparams::("eth_syncing").await { + Ok(serde_json::Value::Object(status)) => Some(ElSyncReading::from_sync_status(status)), + Ok(serde_json::Value::Bool(false)) => self + .el_latest_block_number() + .await + .map(|latest_number| ElSyncReading::NotSyncing { latest_number }), + Ok(status) => { + warn!(target: "engine", %status, "Execution layer returned an unexpected sync status"); + None + } + Err(err) => { + warn!(target: "engine", ?err, "Failed to fetch the execution layer's sync status"); + None + } + } + } + + /// The number of the execution layer's latest block, or `None` if it cannot be read. + async fn el_latest_block_number(&self) -> Option { + trace!(target: "engine", "Fetching the execution layer's latest block number"); + self.client + .get_block_number() + .await + .map_err(|err| { + warn!(target: "engine", ?err, "Failed to fetch the execution layer's latest block number"); + }) + .ok() + } + + /// Looks up the derived block again after the execution layer answered `VALID` for the unsafe + /// head; the engine resets if the block is still missing. + async fn derived_block_committed_since_lookup( + &self, + block_number: u64, + ) -> Result, ConsolidateTaskError> { + match self.client.l2_block_by_label(block_number.into()).await { + Ok(Some(block)) => { + info!(target: "engine", block_number, "Execution layer committed the derived block since the lookup"); + Ok(block) + } + Ok(None) => { + warn!(target: "engine", block_number, "Execution layer has the unsafe head but not the derived block"); + Err(ConsolidateTaskError::MissingUnsafeL2Block(block_number)) + } + Err(err) => { + warn!(target: "engine", ?err, block_number, "Failed to look up the derived block again"); + Err(ConsolidateTaskError::FailedToFetchUnsafeL2Block) + } + } + } + /// Attempts consolidation on the engine state. #[tracing::instrument( name = "consolidate", @@ -285,8 +504,12 @@ impl ConsolidateTask { if let ConsolidateInput::BlockInfo(safe_l2) = &self.input { return self.follow_safe_head_during_el_sync(state, safe_l2).await; } - warn!(target: "engine", "Received `None` block for {}", block_num); - return Err(ConsolidateTaskError::MissingUnsafeL2Block(block_num)); + match self.derived_block_once_el_syncs(state, block_num).await? { + DerivedBlock::Committed(block) => *block, + DerivedBlock::BuildFromAttributes => { + return self.reconcile_unsafe_to_safe(state).await; + } + } } Err(_) => { return Err(ConsolidateTaskError::FailedToFetchUnsafeL2Block); @@ -435,17 +658,19 @@ impl ExecutableEngineTask for ConsolidateTask { mod tests { use super::*; use crate::{ - EngineSyncState, EngineTaskError, FinalityViolation, ImportedBlockSink, NoopBlockSink, - SealTaskError, + EngineSyncState, EngineTaskError, + EngineTaskErrorSeverity::{self, Reset, Temporary, Wait}, + FinalityViolation, ImportedBlockSink, NoopBlockSink, SealTaskError, test_utils::{ - TestAttributesBuilder, TestEngineStateBuilder, l1_info_deposit, test_block_info, - test_engine_client, + RpcMock, TestAttributesBuilder, TestEngineStateBuilder, l1_info_deposit, + test_block_info, test_engine_client, test_rpc_block_from_attributes, }, }; - use alloy_consensus::transaction::Recovered; use alloy_eips::{BlockNumHash, Encodable2718}; - use alloy_primitives::{Address, B256}; - use alloy_rpc_types_engine::{ForkchoiceUpdated, PayloadStatus, PayloadStatusEnum}; + use alloy_primitives::B256; + use alloy_rpc_types_engine::{ + ForkchoiceUpdated, INVALID_FORK_CHOICE_STATE_ERROR, PayloadStatus, PayloadStatusEnum, + }; use alloy_rpc_types_eth::{Block as RpcBlock, BlockTransactions, Header as RpcHeader}; use kona_genesis::ChainGenesis; use kona_protocol::BlockInfo; @@ -480,8 +705,13 @@ mod tests { ..Default::default() }; let sink = Arc::new(RecordingSink::default()); - let task = - ConsolidateTask::new(client, cfg, ConsolidateInput::BlockInfo(safe_l2), sink.clone()); + let task = ConsolidateTask::new( + client, + cfg, + ConsolidateInput::BlockInfo(safe_l2), + sink.clone(), + STALL, + ); task.consolidate(&mut EngineState::default()).await.unwrap(); @@ -519,6 +749,7 @@ mod tests { cfg, ConsolidateInput::Attributes(Box::new(attributes.clone())), Arc::new(crate::NoopBlockSink), + STALL, ); task.build_and_import_attributes(&mut state, attributes) .await @@ -700,6 +931,7 @@ mod tests { cfg, ConsolidateInput::BlockInfo(delegated_safe_head), sink.clone(), + STALL, ); task.execute(&mut state).await.unwrap(); @@ -710,36 +942,269 @@ mod tests { l2.assert_finished(); } - /// Unlike a delegated safe head, derived attributes whose block is missing below the unsafe - /// head still reset the engine. - #[tokio::test] - async fn derived_attributes_missing_from_el_reset_the_engine() { - let cfg = Arc::new(RollupConfig::default()); + const STALL_SECS: u64 = 3600; + const STALL: Duration = Duration::from_secs(STALL_SECS); + + /// A derived-attributes task for block 10 and an engine state whose unsafe head is above it. + fn task_for_missing_derived_block(client: EngineClient) -> (ConsolidateTask, EngineState) { let attributes = derived_attributes(true); - let mut state = engine_state_with_safe_head(attributes.parent); - let state_before = state; - let (client, l1, l2) = test_engine_client(cfg.clone()); - l2.expect_params( - "eth_getBlockByNumber", - serde_json::json!(["0xa", true]), - None::>, - ); + let state = engine_state_with_safe_head(attributes.parent); let task = ConsolidateTask::new( Arc::new(client), - cfg, + Arc::new(RollupConfig::default()), ConsolidateInput::Attributes(Box::new(attributes)), Arc::new(NoopBlockSink), + STALL, ); + (task, state) + } + + /// The execution layer lacks block 10, reports no sync in progress, and answers the unchanged + /// forkchoice with `reply`. + fn el_lacks_derived_block(l2: &RpcMock, state: &EngineState, reply: PayloadStatusEnum) { + el_lacks_derived_block_and_reports(l2, IdleAt(100)); + l2.expect_forkchoice_updated(state.sync_state.create_forkchoice_state(), reply); + } + + /// The execution layer lacks block 10 and reports `report` about its sync. + fn el_lacks_derived_block_and_reports(l2: &RpcMock, report: ElReports) { + l2.expect_block_by_number(10, None); + report.expect_on(l2); + } + + fn el_syncing(l2: &RpcMock, state: &EngineState) { + el_lacks_derived_block(l2, state, PayloadStatusEnum::Syncing); + } + + fn el_valid_and_block_still_missing(l2: &RpcMock, state: &EngineState) { + el_lacks_derived_block(l2, state, PayloadStatusEnum::Valid); + l2.expect_block_by_number(10, None); + } + + fn el_valid_and_recheck_fails(l2: &RpcMock, state: &EngineState) { + el_lacks_derived_block(l2, state, PayloadStatusEnum::Valid); + l2.expect_error("eth_getBlockByNumber"); + } + + fn el_invalid(l2: &RpcMock, state: &EngineState) { + el_lacks_derived_block( + l2, + state, + PayloadStatusEnum::Invalid { validation_error: "bad block".into() }, + ); + } + + fn forkchoice_update_fails(l2: &RpcMock, _: &EngineState) { + el_lacks_derived_block_and_reports(l2, IdleAt(100)); + l2.expect_error("engine_forkchoiceUpdatedV3"); + } + + fn forkchoice_state_rejected(l2: &RpcMock, _: &EngineState) { + el_lacks_derived_block_and_reports(l2, IdleAt(100)); + l2.expect_error_code("engine_forkchoiceUpdatedV3", INVALID_FORK_CHOICE_STATE_ERROR as i64); + } + + /// Derived attributes whose block is missing below the unsafe head resend the unchanged + /// forkchoice. An execution layer still syncing toward the unsafe head makes the task wait, and + /// a failed RPC is retried on the failure backoff; one that has the unsafe head but still not + /// the block, or rejects the forkchoice, resets the engine. The labels stay unchanged. + #[rstest::rstest] + #[case::el_syncing_waits(el_syncing, Wait)] + #[case::el_valid_and_block_still_missing_resets(el_valid_and_block_still_missing, Reset)] + #[case::el_valid_and_recheck_failing_is_retried(el_valid_and_recheck_fails, Temporary)] + #[case::el_invalid_resets(el_invalid, Reset)] + #[case::forkchoice_update_failing_is_retried(forkchoice_update_fails, Temporary)] + #[case::forkchoice_state_rejected_resets(forkchoice_state_rejected, Reset)] + #[tokio::test] + async fn derived_block_missing_from_el_asks_whether_it_syncs( + #[case] el_answers: fn(&RpcMock, &EngineState), + #[case] expected_severity: EngineTaskErrorSeverity, + ) { + let (client, l1, l2) = test_engine_client(Arc::new(RollupConfig::default())); + let (task, mut state) = task_for_missing_derived_block(client); + let state_before = state; + el_answers(&l2, &state); let err = task.execute(&mut state).await.unwrap_err(); - assert!(matches!(err, ConsolidateTaskError::MissingUnsafeL2Block(10)), "got {err:?}"); - assert_eq!(err.severity(), crate::EngineTaskErrorSeverity::Reset); + assert_eq!(err.severity(), expected_severity, "got {err:?}"); assert_eq!(state, state_before); l1.assert_finished(); l2.assert_finished(); } + /// A probe answered `VALID` tells the engine the execution layer applied its forkchoice and + /// finished syncing, even when the recheck of the block then fails; only the labels stay. + #[tokio::test] + async fn valid_probe_records_the_el_sync_even_when_the_recheck_fails() { + let (client, l1, l2) = test_engine_client(Arc::new(RollupConfig::default())); + let (task, mut state) = task_for_missing_derived_block(client); + state.el_sync_finished = false; + let sync_state_before = state.sync_state; + el_valid_and_recheck_fails(&l2, &state); + + let err = task.execute(&mut state).await.unwrap_err(); + + assert_eq!(err.severity(), Temporary, "got {err:?}"); + assert!(state.el_sync_finished); + assert_eq!(state.sync_state, sync_state_before); + l1.assert_finished(); + l2.assert_finished(); + } + + /// An execution layer that commits the derived block between the lookup and the forkchoice + /// update answers `VALID`; the block is then consolidated instead of resetting the engine. + #[tokio::test] + async fn derived_block_committed_during_the_probe_is_consolidated() { + let (client, l1, l2) = test_engine_client(Arc::new(RollupConfig::default())); + let (task, mut state) = task_for_missing_derived_block(client); + let ConsolidateInput::Attributes(attributes) = &task.input else { unreachable!() }; + let block = test_rpc_block_from_attributes(attributes); + let expected_sync_state = + with_safe_head(&state, block_info_of(&RollupConfig::default(), &block)); + el_lacks_derived_block(&l2, &state, PayloadStatusEnum::Valid); + l2.expect_block_by_number(10, Some(block)); + l2.expect_forkchoice_updated( + expected_sync_state.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + + task.execute(&mut state).await.unwrap(); + + assert_eq!(state.sync_state, expected_sync_state); + l1.assert_finished(); + l2.assert_finished(); + } + + /// What the execution layer reports about its sync on one retry. + #[derive(Debug, Clone, Copy)] + enum ElReports { + /// No sync in progress, at this latest block. + IdleAt(u64), + /// A sync in progress, at this current block. + SyncingAt(u64), + } + use ElReports::{IdleAt, SyncingAt}; + + impl ElReports { + fn expect_on(self, l2: &RpcMock) { + match self { + IdleAt(latest_number) => l2.expect_el_not_syncing_at(latest_number), + SyncingAt(current_number) => l2.expect_el_syncing_at(current_number), + } + } + } + + /// One retry of a task waiting for block 10, `elapsed_secs` after the previous one. The + /// execution layer lacks the block; on a check it also reports `check` about its sync and + /// answers the resent forkchoice with `SYNCING`. + #[derive(Debug, Clone, Copy)] + struct Retry { + check: Option, + elapsed_secs: u64, + } + + /// A retry that checks on the execution layer. + const fn checking(elapsed_secs: u64, report: ElReports) -> Retry { + Retry { check: Some(report), elapsed_secs } + } + + /// A retry between checks, which only looks the block up. + const fn looking_up(elapsed_secs: u64) -> Retry { + Retry { check: None, elapsed_secs } + } + + impl Retry { + fn expect_on(self, l2: &RpcMock, state: &EngineState) { + l2.expect_block_by_number(10, None); + if let Some(report) = self.check { + report.expect_on(l2); + l2.expect_forkchoice_updated( + state.sync_state.create_forkchoice_state(), + PayloadStatusEnum::Syncing, + ); + } + } + } + + const CHECK_SECS: u64 = super::super::el_sync_progress::CHECK_INTERVAL.as_secs(); + + /// Across the retries of one task, the execution layer reports its sync on each check. Each + /// retry yields one severity: the task waits while the report changes, and resets once it has + /// stayed the same for the bound and the final check's forkchoice is still answered + /// `SYNCING`. Retries between checks make no RPC but the block lookup. + #[rstest::rstest] + #[case::progressing_el_keeps_waiting(&[checking(0, IdleAt(100)), checking(STALL_SECS - 1, IdleAt(101)), checking(STALL_SECS - 1, IdleAt(102))], &[Wait, Wait, Wait])] + #[case::stuck_el_resets_after_the_bound(&[checking(0, IdleAt(100)), checking(STALL_SECS - CHECK_SECS, IdleAt(100)), checking(CHECK_SECS, IdleAt(100))], &[Wait, Wait, Reset])] + #[case::stuck_sync_resets_after_the_bound(&[checking(0, SyncingAt(100)), checking(STALL_SECS, SyncingAt(100))], &[Wait, Reset])] + #[case::retries_between_checks_only_look_up(&[checking(0, IdleAt(100)), looking_up(2), looking_up(2)], &[Wait, Wait, Wait])] + #[case::check_interval_elapsing_checks(&[checking(0, IdleAt(100)), looking_up(2), checking(CHECK_SECS - 2, IdleAt(100))], &[Wait, Wait, Wait])] + #[tokio::test(start_paused = true)] + async fn wait_for_el_sync_is_bounded_by_its_progress( + #[case] retries: &[Retry], + #[case] expected_severities: &[EngineTaskErrorSeverity], + ) { + let (client, l1, l2) = test_engine_client(Arc::new(RollupConfig::default())); + let (task, mut state) = task_for_missing_derived_block(client); + let mut severities = Vec::new(); + for retry in retries { + tokio::time::advance(Duration::from_secs(retry.elapsed_secs)).await; + retry.expect_on(&l2, &state); + severities.push(task.execute(&mut state).await.unwrap_err().severity()); + } + + assert_eq!(severities, expected_severities); + l1.assert_finished(); + l2.assert_finished(); + } + + /// The check that finds the progress stalled still resends the forkchoice. An execution layer + /// that reached the unsafe head since the last check answers `VALID`, and the derived block is + /// consolidated instead of resetting the engine. + #[tokio::test(start_paused = true)] + async fn stalled_el_that_reached_the_unsafe_head_is_consolidated() { + let (client, l1, l2) = test_engine_client(Arc::new(RollupConfig::default())); + let (task, mut state) = task_for_missing_derived_block(client); + checking(0, IdleAt(100)).expect_on(&l2, &state); + let first = task.execute(&mut state).await.unwrap_err().severity(); + + tokio::time::advance(STALL).await; + let ConsolidateInput::Attributes(attributes) = &task.input else { unreachable!() }; + let block = test_rpc_block_from_attributes(attributes); + let expected_sync_state = + with_safe_head(&state, block_info_of(&RollupConfig::default(), &block)); + el_lacks_derived_block(&l2, &state, PayloadStatusEnum::Valid); + l2.expect_block_by_number(10, Some(block)); + l2.expect_forkchoice_updated( + expected_sync_state.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + task.execute(&mut state).await.unwrap(); + + assert_eq!(first, Wait); + assert_eq!(state.sync_state, expected_sync_state); + l1.assert_finished(); + l2.assert_finished(); + } + + /// A check whose forkchoice update fails is not counted as answered, so the next retry checks + /// again instead of waiting for the check interval. + #[tokio::test(start_paused = true)] + async fn failed_probe_is_retried_on_the_next_retry() { + let (client, l1, l2) = test_engine_client(Arc::new(RollupConfig::default())); + let (task, mut state) = task_for_missing_derived_block(client); + forkchoice_update_fails(&l2, &state); + let first = task.execute(&mut state).await.unwrap_err().severity(); + + tokio::time::advance(Duration::from_secs(2)).await; + checking(0, IdleAt(100)).expect_on(&l2, &state); + let second = task.execute(&mut state).await.unwrap_err().severity(); + + assert_eq!([first, second], [Temporary, Wait]); + l1.assert_finished(); + l2.assert_finished(); + } + fn derived_attributes(is_last_in_span: bool) -> OpAttributesWithParent { let mut attributes = TestAttributesBuilder::new() .with_parent(test_block_info(9)) @@ -749,26 +1214,6 @@ mod tests { attributes } - /// The unsafe block that `attributes` derive, so consolidation finds it consistent. - fn unsafe_block_matching(attributes: &OpAttributesWithParent) -> RpcBlock { - let payload_attributes = &attributes.attributes.payload_attributes; - let header = alloy_consensus::Header { - beneficiary: payload_attributes.suggested_fee_recipient, - gas_limit: attributes.attributes.gas_limit.unwrap(), - mix_hash: payload_attributes.prev_randao, - number: attributes.block_number(), - parent_beacon_block_root: payload_attributes.parent_beacon_block_root, - parent_hash: attributes.parent.block_info.hash, - timestamp: payload_attributes.timestamp, - ..Default::default() - }; - let deposit = Transaction::from_transaction( - Recovered::new_unchecked(l1_info_deposit(), Address::ZERO), - Default::default(), - ); - RpcBlock::new(RpcHeader::new(header), BlockTransactions::Full(vec![deposit])) - } - /// Derived blocks inside a span batch only move the safe label; the span's last block sends /// the forkchoice update that delivers it. #[rstest::rstest] @@ -781,7 +1226,7 @@ mod tests { ) { let cfg = Arc::new(RollupConfig::default()); let attributes = derived_attributes(is_last_in_span); - let block = unsafe_block_matching(&attributes); + let block = test_rpc_block_from_attributes(&attributes); let derived_safe_head = block_info_of(&cfg, &block); let mut state = engine_state_with_safe_head(attributes.parent); let expected_sync_state = with_safe_head(&state, derived_safe_head); @@ -798,6 +1243,7 @@ mod tests { cfg, ConsolidateInput::Attributes(Box::new(attributes)), Arc::new(NoopBlockSink), + STALL, ); task.execute(&mut state).await.unwrap(); @@ -876,6 +1322,7 @@ mod tests { cfg, ConsolidateInput::BlockInfo(delegated_safe_head), Arc::new(NoopBlockSink), + STALL, ); let result = task.execute(&mut state).await; @@ -915,6 +1362,7 @@ mod tests { cfg, ConsolidateInput::BlockInfo(delegated_safe_head), sink.clone(), + STALL, ); task.execute(&mut state).await.unwrap(); @@ -951,6 +1399,7 @@ mod tests { cfg, ConsolidateInput::BlockInfo(delegated_safe_head), Arc::new(NoopBlockSink), + STALL, ); task.execute(&mut state).await.unwrap(); @@ -989,6 +1438,7 @@ mod tests { cfg, ConsolidateInput::BlockInfo(delegated_safe_head), Arc::new(NoopBlockSink), + STALL, ); task.execute(&mut state).await.unwrap(); @@ -1026,6 +1476,7 @@ mod tests { cfg, ConsolidateInput::BlockInfo(delegated_safe_head), Arc::new(NoopBlockSink), + STALL, ); let err = task.execute(&mut state).await.unwrap_err(); @@ -1036,6 +1487,27 @@ mod tests { l2.assert_finished(); } + /// An execution layer that answers `SYNCING` while it reports no sync and sits at the derived + /// block's parent fetches nothing, as without peers, so the block is built from the derived + /// attributes on that parent instead of waiting: the build's forkchoice update follows. + #[tokio::test] + async fn el_idle_at_the_parent_builds_the_derived_block() { + let (client, l1, l2) = test_engine_client(Arc::new(RollupConfig::default())); + let (task, mut state) = task_for_missing_derived_block(client); + el_lacks_derived_block_and_reports(&l2, IdleAt(9)); + l2.expect_forkchoice_updated( + state.sync_state.create_forkchoice_state(), + PayloadStatusEnum::Syncing, + ); + l2.expect_error("engine_forkchoiceUpdatedV2"); + + let err = task.execute(&mut state).await.unwrap_err(); + + assert!(matches!(err, ConsolidateTaskError::BuildTaskFailed(_)), "got {err:?}"); + l1.assert_finished(); + l2.assert_finished(); + } + /// Labels with the unsafe head rewound below the safe head are moved back into order by any /// delegated safe head above finality instead of being kept: at or above the unsafe head the /// engine reorgs onto it, and below it the matching block consolidates. Keeping them would let @@ -1078,6 +1550,7 @@ mod tests { cfg, ConsolidateInput::BlockInfo(delegated_safe_head), Arc::new(NoopBlockSink), + STALL, ); task.execute(&mut state).await.unwrap(); @@ -1120,6 +1593,7 @@ mod tests { cfg, ConsolidateInput::BlockInfo(delegated_safe_head), Arc::new(NoopBlockSink), + STALL, ); task.execute(&mut state).await.unwrap(); @@ -1128,4 +1602,23 @@ mod tests { l1.assert_finished(); l2.assert_finished(); } + + /// A check whose probe fails records nothing: the wait has not started, so the next check's + /// reading starts it and its report still goes out. + #[tokio::test(start_paused = true)] + async fn failed_probe_does_not_start_the_wait() { + let (client, l1, l2) = test_engine_client(Arc::new(RollupConfig::default())); + let (task, mut state) = task_for_missing_derived_block(client); + forkchoice_update_fails(&l2, &state); + task.execute(&mut state).await.unwrap_err(); + let has_started_after_failure = task.el_sync_progress().has_started(); + + checking(0, IdleAt(100)).expect_on(&l2, &state); + task.execute(&mut state).await.unwrap_err(); + + assert!(!has_started_after_failure); + assert!(task.el_sync_progress().has_started()); + l1.assert_finished(); + l2.assert_finished(); + } } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/mod.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/mod.rs index 9eddaa76d00..593b3a29d9f 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/mod.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/mod.rs @@ -7,7 +7,10 @@ pub use task::{ mod synchronize; pub use synchronize::SynchronizeTaskError; -pub(super) use synchronize::{synchronize, synchronize_unconditionally}; +pub(super) use synchronize::{ + ForkchoiceReply, record_forkchoice_reply, send_forkchoice, synchronize, + synchronize_unconditionally, +}; mod insert; pub(super) use insert::insert_payload; @@ -25,7 +28,9 @@ mod canonicalize; pub use canonicalize::{CanonicalizeTask, CanonicalizeTaskError}; mod consolidate; -pub use consolidate::{ConsolidateInput, ConsolidateTask, ConsolidateTaskError}; +pub use consolidate::{ + ConsolidateInput, ConsolidateTask, ConsolidateTaskError, DEFAULT_EL_SYNC_STALL_TIMEOUT, +}; mod finalize; pub use finalize::{FinalizeBlockId, FinalizeTask, FinalizeTaskError}; diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs index fc28ccf0026..dc2233aa44a 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs @@ -6,6 +6,15 @@ use crate::{ use alloy_rpc_types_engine::{INVALID_FORK_CHOICE_STATE_ERROR, PayloadStatusEnum}; use op_alloy_provider::ext::engine::OpEngineApi; +/// How the execution layer answered a forkchoice update it accepted. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(in crate::task_queue) enum ForkchoiceReply { + /// The execution layer does not have the head block yet and is syncing toward it. + Syncing, + /// The execution layer applied the forkchoice. + Valid, +} + /// Applies `state_update` to the engine's sync state and synchronizes the execution layer's /// forkchoice with it, using an `engine_forkchoiceUpdated` call without payload attributes. An /// update that leaves the labels unchanged is skipped. @@ -38,7 +47,7 @@ pub(in crate::task_queue) async fn synchronize( return Ok(()); } - send_forkchoice(client, state, new_sync_state).await + send_and_apply_forkchoice(client, state, new_sync_state).await } /// Like [`synchronize`], but sends the forkchoice update even when it leaves the labels unchanged. @@ -58,12 +67,12 @@ pub(in crate::task_queue) async fn synchronize_unconditionally( state_update: EngineSyncStateUpdate, ) -> Result<(), SynchronizeTaskError> { let new_sync_state = state.sync_state.apply_update(state_update); - send_forkchoice(client, state, new_sync_state).await + send_and_apply_forkchoice(client, state, new_sync_state).await } /// Sends the forkchoice of `new_sync_state` and, once the execution layer accepts it, makes it the /// engine's sync state. -async fn send_forkchoice( +async fn send_and_apply_forkchoice( client: &EngineClient, state: &mut EngineState, new_sync_state: EngineSyncState, @@ -81,12 +90,40 @@ async fn send_forkchoice( // Send the forkchoice update through the input. let forkchoice = new_sync_state.create_forkchoice_state(); tracing::Span::current().record("forkchoice", tracing::field::debug(forkchoice)); + let reply = send_forkchoice(client, &new_sync_state).await?; + record_forkchoice_reply(state, reply); + + // Apply the new sync state to the engine state. + state.sync_state = new_sync_state; - // Handle the forkchoice update result. + Ok(()) +} + +/// Records how the execution layer answered the engine's forkchoice: whether it applied that +/// forkchoice, which a `SYNCING` answer clears, and whether its sync has finished, which only a +/// `VALID` answer marks and nothing clears. +pub(in crate::task_queue) fn record_forkchoice_reply( + state: &mut EngineState, + reply: ForkchoiceReply, +) { + state.el_applied_last_forkchoice = reply == ForkchoiceReply::Valid; + if reply == ForkchoiceReply::Valid && !state.el_sync_finished { + info!(target: "engine", "Finished execution layer sync."); + state.el_sync_finished = true; + } +} + +/// Sends `sync_state`'s forkchoice to the execution layer, with no payload attributes, and returns +/// whether it applied the forkchoice or is still syncing toward its head. Neither the engine state +/// nor its record of the execution layer's sync changes. +pub(in crate::task_queue) async fn send_forkchoice( + client: &EngineClient, + sync_state: &EngineSyncState, +) -> Result { // NOTE: it doesn't matter which version we use here, because we're not sending any payload // attributes. The forkchoice updated call is version agnostic if no payload attributes are // provided. - let response = client.fork_choice_updated_v3(forkchoice, None).await; + let response = client.fork_choice_updated_v3(sync_state.create_forkchoice_state(), None).await; let valid_response = response.map_err(|e| { // Fatal forkchoice update error. @@ -99,24 +136,13 @@ async fn send_forkchoice( })?; match &valid_response.payload_status.status { - PayloadStatusEnum::Valid => { - state.el_applied_last_forkchoice = true; - if !state.el_sync_finished { - info!(target: "engine", "Finished execution layer sync."); - state.el_sync_finished = true; - } - } + PayloadStatusEnum::Valid => Ok(ForkchoiceReply::Valid), // If we're not building a new payload, we're driving EL sync. PayloadStatusEnum::Syncing => { - state.el_applied_last_forkchoice = false; - debug!(target: "engine", "Attempting to update forkchoice state while EL syncing") + debug!(target: "engine", "Attempting to update forkchoice state while EL syncing"); + Ok(ForkchoiceReply::Syncing) } // Other codes are not expected. - status => return Err(SynchronizeTaskError::UnexpectedPayloadStatus(status.clone())), + status => Err(SynchronizeTaskError::UnexpectedPayloadStatus(status.clone())), } - - // Apply the new sync state to the engine state. - state.sync_state = new_sync_state; - - Ok(()) } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/mod.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/mod.rs index 9de83da3a5c..f620d4c6d79 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/mod.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/mod.rs @@ -1,7 +1,10 @@ //! The forkchoice engine update shared by the engine tasks, and its error type. mod helper; -pub(in crate::task_queue) use helper::{synchronize, synchronize_unconditionally}; +pub(in crate::task_queue) use helper::{ + ForkchoiceReply, record_forkchoice_reply, send_forkchoice, synchronize, + synchronize_unconditionally, +}; mod error; pub use error::SynchronizeTaskError; diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/task.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/task.rs index 7339a4c8102..b3a7f6a315d 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/task.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/task.rs @@ -36,6 +36,11 @@ pub enum EngineTaskErrorSeverity { /// the engine keeps running. #[display("skip")] Skip, + /// The task changed nothing and cannot make progress yet, typically until the execution layer + /// syncs further. It is retried after a backoff, and higher-priority tasks queued meanwhile + /// run first. + #[display("wait")] + Wait, } /// The interface for an engine task error. diff --git a/rust/kona/crates/node/engine/src/test_utils/blocks.rs b/rust/kona/crates/node/engine/src/test_utils/blocks.rs index 83516a460e1..dfbc1936375 100644 --- a/rust/kona/crates/node/engine/src/test_utils/blocks.rs +++ b/rust/kona/crates/node/engine/src/test_utils/blocks.rs @@ -4,7 +4,7 @@ use super::{l1_info_deposit, l1_info_deposit_for}; use alloy_consensus::{Header, transaction::Recovered}; use alloy_primitives::{Address, B256, Bytes}; use alloy_rpc_types_eth::{BlockTransactions, Header as RpcHeader}; -use kona_protocol::L1BlockInfoBedrock; +use kona_protocol::{L1BlockInfoBedrock, OpAttributesWithParent}; use op_alloy_consensus::OpBlock; /// A full L2 block as the execution layer serves it over RPC. @@ -61,3 +61,20 @@ pub fn rpc_l2_block(header: Header, info: &L1BlockInfoBedrock) -> TestRpcBlock { }]); block } + +/// The block that `attributes` derive, as the execution layer serves it over RPC, so that +/// consolidating `attributes` against it finds a match. `attributes` must carry only the L1-info +/// deposit and a gas limit. +pub fn test_rpc_block_from_attributes(attributes: &OpAttributesWithParent) -> TestRpcBlock { + let payload_attributes = &attributes.attributes.payload_attributes; + let mut block = OpBlock::default(); + block.header.beneficiary = payload_attributes.suggested_fee_recipient; + block.header.gas_limit = attributes.attributes.gas_limit.expect("attributes set a gas limit"); + block.header.mix_hash = payload_attributes.prev_randao; + block.header.number = attributes.block_number(); + block.header.parent_beacon_block_root = payload_attributes.parent_beacon_block_root; + block.header.parent_hash = attributes.parent.block_info.hash; + block.header.timestamp = payload_attributes.timestamp; + block.body.transactions.push(l1_info_deposit()); + test_rpc_block(&block) +} diff --git a/rust/kona/crates/node/engine/src/test_utils/mod.rs b/rust/kona/crates/node/engine/src/test_utils/mod.rs index dd8c55020bd..ac61416649a 100644 --- a/rust/kona/crates/node/engine/src/test_utils/mod.rs +++ b/rust/kona/crates/node/engine/src/test_utils/mod.rs @@ -2,7 +2,9 @@ mod attributes; pub use attributes::TestAttributesBuilder; mod blocks; -pub use blocks::{TestRpcBlock, rpc_l2_block, test_l2_block, test_rpc_block}; +pub use blocks::{ + TestRpcBlock, rpc_l2_block, test_l2_block, test_rpc_block, test_rpc_block_from_attributes, +}; mod deposits; pub use deposits::{l1_info_deposit, l1_info_deposit_for}; diff --git a/rust/kona/crates/node/engine/src/test_utils/rpc.rs b/rust/kona/crates/node/engine/src/test_utils/rpc.rs index cb5aee9403d..cd585f6e681 100644 --- a/rust/kona/crates/node/engine/src/test_utils/rpc.rs +++ b/rust/kona/crates/node/engine/src/test_utils/rpc.rs @@ -93,6 +93,26 @@ impl RpcMock { ); } + /// Expects an `eth_syncing` call answered with no sync in progress, then an `eth_blockNumber` + /// call answered with `latest_number`. + pub fn expect_el_not_syncing_at(&self, latest_number: u64) { + self.expect("eth_syncing", false); + self.expect("eth_blockNumber", alloy_primitives::U64::from(latest_number)); + } + + /// Expects an `eth_syncing` call answered with a sync in progress whose current block is + /// `current_number`. + pub fn expect_el_syncing_at(&self, current_number: u64) { + self.expect( + "eth_syncing", + serde_json::json!({ + "currentBlock": format!("{current_number:#x}"), + "highestBlock": "0xffff", + "startingBlock": "0x0", + }), + ); + } + /// Expects an `engine_forkchoiceUpdatedV3` call for `forkchoice` without payload attributes and /// answers it with `status`. pub fn expect_forkchoice_updated( diff --git a/rust/kona/crates/node/service/src/actors/engine/actor.rs b/rust/kona/crates/node/service/src/actors/engine/actor.rs index b45bfa62238..32251cc5c99 100644 --- a/rust/kona/crates/node/service/src/actors/engine/actor.rs +++ b/rust/kona/crates/node/service/src/actors/engine/actor.rs @@ -4,9 +4,9 @@ use crate::{ use async_trait::async_trait; use kona_derive::{ResetSignal, Signal}; use kona_engine::{ - BuildTask, CanonicalizeTask, ConsolidateTask, Engine, EngineActorRequest, EngineClient, - EngineTask, EngineTaskError, EngineTaskErrorSeverity, EngineTaskErrors, FinalizeTask, - ImportedBlockSink, InsertTask, SealTask, + BuildTask, CanonicalizeTask, ConsolidateTask, DEFAULT_EL_SYNC_STALL_TIMEOUT, DrainScope, + Engine, EngineActorRequest, EngineClient, EngineTask, EngineTaskError, EngineTaskErrorSeverity, + EngineTaskErrors, FinalizeTask, ImportedBlockSink, InsertTask, SealTask, }; use kona_genesis::RollupConfig; use kona_protocol::L2BlockInfo; @@ -20,6 +20,12 @@ use tokio::{ #[path = "queue_tests.rs"] mod queue_tests; +/// How long a task waiting for the execution layer to sync waits between retries. +/// +/// The wait is not a failure, so it does not grow the failure backoff, and each retry costs a few +/// RPCs on an execution layer that is busy syncing. +const WAIT_RETRY_INTERVAL: Duration = Duration::from_secs(2); + /// Responsible for managing the operations sent to the execution layer's Engine API. To accomplish /// this, it uses the [`Engine`] task queue to order Engine API interactions based off of /// the [`Ord`] implementation of [`EngineTask`]. @@ -32,6 +38,8 @@ where derivation_client: DerivationClient, /// Whether the EL sync is complete. This should only ever go from false to true. el_sync_complete: bool, + /// How long a consolidation waits for an execution layer whose sync reports no progress. + el_sync_stall_timeout: Duration, /// The last local-safe head sent to the derivation actor. last_engine_safe_head_sent: L2BlockInfo, /// The error of the last skipped engine task, so an identical repeat logs no new warning @@ -43,17 +51,18 @@ where /// mode. unsafe_head_tx: Option>, - /// The [`RollupConfig`] used to build tasks. - rollup: Arc, + /// Where to hand every imported block, so the derivation providers can read it locally + /// instead of fetching it back from the execution layer. + block_sink: Arc, /// An [`EngineClient`] used for creating engine tasks. client: Arc, /// The [`Engine`] task queue. engine: Engine, /// The inbound request channel. inbound_request_rx: mpsc::Receiver, - /// Where to hand every imported block, so the derivation providers can read it locally - /// instead of fetching it back from the execution layer. - block_sink: Arc, + /// The [`RollupConfig`] used to build tasks. + rollup: Arc, + /// A reset is retained until successful; RPC failure must not lose the request. reset_pending: bool, /// Callers waiting for the retained reset to finish. @@ -62,6 +71,9 @@ where retry_at: Instant, /// Exponentially increasing retry delay, capped to keep recovery responsive. retry_delay: Duration, + /// Earliest retry of the task parked while it waits for the execution layer. Kept apart from + /// `retry_at`, so a failure of a task that runs ahead of it cannot bring its retry forward. + waiting_retry_at: Instant, } impl EngineActor @@ -79,23 +91,32 @@ where block_sink: Arc, ) -> Self { Self { + block_sink, client, derivation_client, el_sync_complete: false, + el_sync_stall_timeout: DEFAULT_EL_SYNC_STALL_TIMEOUT, engine, + inbound_request_rx, last_engine_safe_head_sent: L2BlockInfo::default(), last_skipped_task_error: None, - rollup: config, - unsafe_head_tx, - inbound_request_rx, - block_sink, reset_pending: false, reset_waiters: Vec::new(), retry_at: Instant::now(), retry_delay: Duration::from_millis(100), + rollup: config, + unsafe_head_tx, + waiting_retry_at: Instant::now(), } } + /// Bounds how long a consolidation waits for an execution layer whose sync reports no + /// progress. + pub const fn with_el_sync_stall_timeout(mut self, el_sync_stall_timeout: Duration) -> Self { + self.el_sync_stall_timeout = el_sync_stall_timeout; + self + } + /// Logs a skipped task at `warn!` when its error differs from the last skip, and at `debug!` /// while the identical error repeats, so a target resubmitted on every poll warns once. fn log_skipped_task(&mut self, err: &EngineTaskErrors) { @@ -113,6 +134,30 @@ where self.retry_delay = (self.retry_delay * 2).min(Duration::from_secs(5)); } + /// Schedules the retry of a task waiting for the execution layer, leaving the failure backoff + /// as it is. + fn defer_waiting_retry(&mut self) { + self.waiting_retry_at = Instant::now() + WAIT_RETRY_INTERVAL; + } + + /// Which tasks a drain may run now: the waiting task only once its retry is due. + fn drain_scope(&self, now: Instant) -> DrainScope { + if now >= self.waiting_retry_at { + DrainScope::AllTasks + } else { + DrainScope::AheadOfWaitingTask + } + } + + /// When the actor next has work to retry without a new request, if any: the failure backoff + /// while a reset or a runnable task is pending, and otherwise the waiting task's retry. + fn next_retry_at(&self) -> Option { + if self.reset_pending || self.engine.has_work_ahead_of_waiting_task() { + return Some(self.retry_at); + } + self.engine.has_waiting_task().then(|| self.retry_at.max(self.waiting_retry_at)) + } + fn begin_reset(&mut self) { if !self.reset_pending { self.engine.clear(); @@ -188,10 +233,10 @@ where } /// Drains the inner [`Engine`] task queue and attempts to update the safe head. - async fn drain(&mut self) -> Result<(), EngineError> { - match self.engine.drain().await { + async fn drain(&mut self, scope: DrainScope) -> Result<(), EngineError> { + match self.engine.drain(scope).await { Ok(_) => { - trace!(target: "engine", "[ENGINE] tasks drained"); + trace!(target: "engine", ?scope, "[ENGINE] tasks drained"); self.retry_delay = Duration::from_millis(100); } Err(err) => { @@ -228,6 +273,14 @@ where return Ok(()); } EngineTaskErrorSeverity::Skip => self.log_skipped_task(&err), + // A waiting task changed nothing, but tasks that ran before it in this drain + // did, so their safe head and sync completion still reach derivation below. + // They all succeeded, so the failure backoff starts over too. + EngineTaskErrorSeverity::Wait => { + trace!(target: "engine", %err, "Engine task waits before its retry"); + self.retry_delay = Duration::from_millis(100); + self.defer_waiting_retry(); + } } } } @@ -267,7 +320,7 @@ where async fn send_derivation_actor_safe_head_if_updated(&mut self) -> Result<(), EngineError> { let local_safe_head = self.engine.state().sync_state.local_safe_head(); if local_safe_head == self.last_engine_safe_head_sent { - info!(target: "engine", safe_head = ?local_safe_head, "Local-safe head unchanged"); + debug!(target: "engine", safe_head = ?local_safe_head, "Local-safe head unchanged"); return Ok(()); } @@ -291,11 +344,14 @@ where type Error = EngineError; async fn step(&mut self) -> Result<(), Self::Error> { - if Instant::now() >= self.retry_at { + let now = Instant::now(); + if now >= self.retry_at { if self.reset_pending { self.attempt_reset().await?; } else { - self.drain().await?; + // Work that outranks a task waiting for the execution layer to sync, such as an + // unsafe import, runs as it arrives; the waiting task runs once its retry is due. + self.drain(self.drain_scope(now)).await?; } } @@ -305,13 +361,14 @@ where // Wait for the next processing request. // Retry without needing a new inbound message, while still accepting requests during // backoff. Bound queued work so a prolonged outage cannot grow memory without limit. + let next_retry_at = self.next_retry_at(); let request = tokio::select! { request = self.inbound_request_rx.recv(), if self.engine.len() < 1024 && self.reset_waiters.len() < 1024 => { request.ok_or(EngineError::ChannelClosed)? } - _ = time::sleep_until(self.retry_at), - if self.reset_pending || !self.engine.is_empty() => return Ok(()), + _ = time::sleep_until(next_retry_at.unwrap_or(self.retry_at)), + if next_retry_at.is_some() => return Ok(()), }; match request { @@ -331,6 +388,7 @@ where self.rollup.clone(), safe_signal, Arc::clone(&self.block_sink), + self.el_sync_stall_timeout, ))); self.engine.enqueue(task); } diff --git a/rust/kona/crates/node/service/src/actors/engine/config.rs b/rust/kona/crates/node/service/src/actors/engine/config.rs index 25865cc4e6d..a2abd77fff1 100644 --- a/rust/kona/crates/node/service/src/actors/engine/config.rs +++ b/rust/kona/crates/node/service/src/actors/engine/config.rs @@ -4,13 +4,15 @@ use kona_engine::EngineClient; use kona_genesis::RollupConfig; use kona_providers_alloy::AlloyChainProvider; use op_alloy_network::Optimism; -use std::sync::Arc; +use std::{sync::Arc, time::Duration}; /// Configuration for the Engine Actor. #[derive(Debug, Clone)] pub struct EngineConfig { /// The [`RollupConfig`]. pub config: Arc, + /// How long a consolidation waits for an execution layer whose sync reports no progress. + pub el_sync_stall_timeout: Duration, /// The mode of operation for the node. /// When the node is in sequencer mode, the engine actor will receive requests to build blocks diff --git a/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs b/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs index c7399efc6e6..872f862d83b 100644 --- a/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs +++ b/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs @@ -1,5 +1,6 @@ use super::*; -use crate::QueuedEngineDerivationClient; +use crate::{DerivationActorRequest, QueuedEngineDerivationClient}; +use alloy_eips::Encodable2718; use alloy_primitives::B256; use alloy_rpc_types_engine::{ ExecutionPayloadV1, ForkchoiceState, PayloadStatus, PayloadStatusEnum, @@ -8,8 +9,9 @@ use kona_engine::{ ConsolidateInput, EngineState, EngineSyncStateUpdate, FinalizeBlockId, NoopBlockSink, ResetRequest, test_utils::{ - RpcMock, TestEngineStateBuilder, TestRpcBlock, expect_sync_start, test_block_info, - test_engine_client, test_l2_block, test_rpc_block, + RpcMock, TestAttributesBuilder, TestEngineStateBuilder, TestRpcBlock, expect_sync_start, + l1_info_deposit, test_block_info, test_engine_client, test_l2_block, test_rpc_block, + test_rpc_block_from_attributes, }, }; use op_alloy_consensus::{OpBlock, OpTxEnvelope}; @@ -199,6 +201,395 @@ async fn delegated_poll_during_el_sync_moves_safe_labels_and_skips_finalization( l2.assert_finished(); } +/// After an execution layer datadir wipe, or while the execution layer commits blocks in bursts, +/// derived attributes can name a block below the unsafe head that the execution layer does not +/// have yet. While it answers `SYNCING`, the consolidation waits without a reset, which would +/// clear the derivation actor's finalization queue and re-derive from the execution layer's labels. +/// Unsafe imports received meanwhile run as they arrive rather than at the waiting task's retry, +/// the wait leaves the failure backoff untouched, and the retry consolidates the block once the +/// execution layer has it. +#[tokio::test(start_paused = true)] +async fn derived_block_missing_during_el_sync_waits_without_reset() { + let config = Arc::new(RollupConfig::default()); + let (client, l1, l2) = test_engine_client(config.clone()); + let unsafe_head = test_block_info(20); + let finalized_head = test_block_info(5); + let mut attributes = TestAttributesBuilder::new() + .with_parent(test_block_info(9)) + .with_transactions(vec![l1_info_deposit().encoded_2718().into()]) + .build(); + attributes.is_last_in_span = true; + let safe_head = attributes.parent; + let derived_block = test_rpc_block_from_attributes(&attributes); + let derived_safe_head = L2BlockInfo::from_block_and_genesis( + &derived_block.clone().into_consensus().map_transactions(|t| t.inner.inner.into_inner()), + &config.genesis, + ) + .unwrap(); + let gossiped_payload = + ExecutionPayloadV1::from_block_slow(&test_l2_block(21, unsafe_head.hash(), 0)); + let gossiped_head = L2BlockInfo::from_block_and_genesis( + &OpExecutionPayloadEnvelope::V1(gossiped_payload.clone()) + .try_into_block::() + .unwrap(), + &config.genesis, + ) + .unwrap(); + let state = TestEngineStateBuilder::new() + .with_unsafe_head(unsafe_head) + .with_safe_head(safe_head) + .with_finalized_head(finalized_head) + .build(); + let (state_tx, state_rx) = watch::channel(state); + let (requests, request_rx) = mpsc::channel(8); + let (derivation_tx, mut derivation_rx) = mpsc::channel(8); + let mut actor = EngineActor::new( + Arc::new(client), + config, + QueuedEngineDerivationClient { derivation_actor_request_tx: derivation_tx }, + Engine::new(state, state_tx), + None, + request_rx, + Arc::new(NoopBlockSink), + ); + + // The execution layer has neither the derived block nor the unsafe head, and answers the + // unchanged forkchoice with `SYNCING`. An unsafe block arrives while the consolidation waits. + l2.expect_block_by_number(10, None); + l2.expect_el_not_syncing_at(0); + l2.expect_forkchoice_updated( + forkchoice(unsafe_head, safe_head, finalized_head), + PayloadStatusEnum::Syncing, + ); + requests.send(EngineActorRequest::ProcessSafeL2Signal(attributes.into())).await.unwrap(); + actor.step().await.unwrap(); + requests + .send(EngineActorRequest::ProcessUnsafeL2Block(Box::new(OpExecutionPayloadEnvelope::V1( + gossiped_payload.clone(), + )))) + .await + .unwrap(); + actor.step().await.unwrap(); + assert_eq!(actor.engine.len(), 2, "the consolidation waits behind the unsafe import"); + assert!(!actor.reset_pending); + assert_eq!(actor.retry_delay, Duration::from_millis(100), "waiting is not a failure"); + + // The unsafe import runs at the next step, well before the waiting consolidation's retry + // elapses; the step then idles toward that retry. + l2.expect_params( + "engine_newPayloadV2", + serde_json::json!([gossiped_payload]), + PayloadStatus::from_status(PayloadStatusEnum::Valid), + ); + l2.expect_forkchoice_updated( + forkchoice(gossiped_head, safe_head, finalized_head), + PayloadStatusEnum::Valid, + ); + assert!(time::timeout(Duration::from_millis(50), actor.step()).await.is_err()); + assert_eq!(state_rx.borrow().sync_state.unsafe_head(), gossiped_head); + assert_eq!(actor.engine.len(), 1, "only the waiting consolidation may remain"); + + // The next step only idles until the waiting consolidation's retry is due. + assert!(actor.engine.has_waiting_task()); + time::timeout(WAIT_RETRY_INTERVAL, actor.step()).await.unwrap().unwrap(); + assert!(actor.engine.has_waiting_task(), "the consolidation is not retried before it is due"); + + // The step after it retries the consolidation, which finds the derived block; with nothing left + // to do, the actor then waits for requests instead of retrying. + l2.expect_block_by_number(10, Some(derived_block)); + l2.expect_forkchoice_updated( + forkchoice(gossiped_head, derived_safe_head, finalized_head), + PayloadStatusEnum::Valid, + ); + assert!(time::timeout(Duration::from_secs(60), actor.step()).await.is_err()); + + assert!(actor.engine.is_empty()); + assert!(!actor.reset_pending); + assert_eq!(state_rx.borrow().sync_state.safe_head(), derived_safe_head); + assert_eq!(state_rx.borrow().sync_state.unsafe_head(), gossiped_head); + let mut derivation_requests = Vec::new(); + while let Ok(request) = derivation_rx.try_recv() { + derivation_requests.push(request); + } + assert!( + !derivation_requests.iter().any(|request| matches!( + request, + DerivationActorRequest::ProcessEngineSignalRequest(_) + )), + "no reset may reach derivation: {derivation_requests:?}" + ); + assert!(matches!( + derivation_requests.last(), + Some(DerivationActorRequest::ProcessEngineSafeHeadUpdateRequest(head)) if **head == derived_safe_head + )); + l1.assert_finished(); + l2.assert_finished(); +} + +/// A failure of an unsafe import that runs ahead of a consolidation waiting for the execution +/// layer retries the import on the failure backoff, but leaves the consolidation parked until its +/// own retry is due. +#[tokio::test(start_paused = true)] +async fn failing_import_ahead_of_a_waiting_consolidation_keeps_its_retry() { + let config = Arc::new(RollupConfig::default()); + let (client, l1, l2) = test_engine_client(config.clone()); + let unsafe_head = test_block_info(20); + let finalized_head = test_block_info(5); + let mut attributes = TestAttributesBuilder::new() + .with_parent(test_block_info(9)) + .with_transactions(vec![l1_info_deposit().encoded_2718().into()]) + .build(); + attributes.is_last_in_span = true; + let safe_head = attributes.parent; + let gossiped_payload = + ExecutionPayloadV1::from_block_slow(&test_l2_block(21, unsafe_head.hash(), 0)); + let gossiped_head = L2BlockInfo::from_block_and_genesis( + &OpExecutionPayloadEnvelope::V1(gossiped_payload.clone()) + .try_into_block::() + .unwrap(), + &config.genesis, + ) + .unwrap(); + let state = TestEngineStateBuilder::new() + .with_unsafe_head(unsafe_head) + .with_safe_head(safe_head) + .with_finalized_head(finalized_head) + .build(); + let (state_tx, state_rx) = watch::channel(state); + let (requests, request_rx) = mpsc::channel(8); + let (derivation_tx, _derivation_rx) = mpsc::channel(8); + let mut actor = EngineActor::new( + Arc::new(client), + config, + QueuedEngineDerivationClient { derivation_actor_request_tx: derivation_tx }, + Engine::new(state, state_tx), + None, + request_rx, + Arc::new(NoopBlockSink), + ); + + // The consolidation finds no derived block and parks while the execution layer syncs. + l2.expect_block_by_number(10, None); + l2.expect_el_not_syncing_at(0); + l2.expect_forkchoice_updated( + forkchoice(unsafe_head, safe_head, finalized_head), + PayloadStatusEnum::Syncing, + ); + requests.send(EngineActorRequest::ProcessSafeL2Signal(attributes.into())).await.unwrap(); + actor.step().await.unwrap(); + l2.expect_error("engine_newPayloadV2"); + requests + .send(EngineActorRequest::ProcessUnsafeL2Block(Box::new(OpExecutionPayloadEnvelope::V1( + gossiped_payload.clone(), + )))) + .await + .unwrap(); + actor.step().await.unwrap(); + let parked_at = Instant::now(); + assert!(actor.engine.has_waiting_task()); + + // The unsafe import runs ahead of it and fails once, then succeeds on its backoff, both well + // before the consolidation's retry; the step then idles toward that retry. + actor.step().await.unwrap(); + l2.expect_params( + "engine_newPayloadV2", + serde_json::json!([gossiped_payload]), + PayloadStatus::from_status(PayloadStatusEnum::Valid), + ); + l2.expect_forkchoice_updated( + forkchoice(gossiped_head, safe_head, finalized_head), + PayloadStatusEnum::Valid, + ); + assert!(time::timeout(WAIT_RETRY_INTERVAL / 2, actor.step()).await.is_err()); + + assert_eq!(state_rx.borrow().sync_state.unsafe_head(), gossiped_head); + assert!(actor.engine.has_waiting_task(), "the consolidation stays parked"); + assert!( + Instant::now() < parked_at + WAIT_RETRY_INTERVAL, + "the import ran before the consolidation's retry was due" + ); + assert_eq!(actor.engine.len(), 1, "only the waiting consolidation remains"); + l1.assert_finished(); + l2.assert_finished(); +} + +/// A drain in which an unsafe import completes the execution layer's sync and a consolidation then +/// waits still reports the import to derivation: the waiting task changed nothing, but the import +/// moved the engine's labels and finished its sync. +#[tokio::test(start_paused = true)] +async fn sync_completed_before_a_waiting_consolidation_reaches_derivation() { + let config = Arc::new(RollupConfig::default()); + let (client, l1, l2) = test_engine_client(config.clone()); + let client = Arc::new(client); + let unsafe_head = test_block_info(20); + let finalized_head = test_block_info(5); + let mut attributes = TestAttributesBuilder::new() + .with_parent(test_block_info(9)) + .with_transactions(vec![l1_info_deposit().encoded_2718().into()]) + .build(); + attributes.is_last_in_span = true; + let safe_head = attributes.parent; + let gossiped_payload = + ExecutionPayloadV1::from_block_slow(&test_l2_block(21, unsafe_head.hash(), 0)); + let gossiped_head = L2BlockInfo::from_block_and_genesis( + &OpExecutionPayloadEnvelope::V1(gossiped_payload.clone()) + .try_into_block::() + .unwrap(), + &config.genesis, + ) + .unwrap(); + let state = TestEngineStateBuilder::new() + .with_unsafe_head(unsafe_head) + .with_safe_head(safe_head) + .with_finalized_head(finalized_head) + .with_el_sync_finished(false) + .build(); + let (state_tx, _state_rx) = watch::channel(state); + let mut engine = Engine::new(state, state_tx); + engine.enqueue(EngineTask::Consolidate(Box::new(ConsolidateTask::new( + client.clone(), + config.clone(), + attributes.into(), + Arc::new(NoopBlockSink), + DEFAULT_EL_SYNC_STALL_TIMEOUT, + )))); + engine.enqueue(EngineTask::Insert(Box::new(InsertTask::new( + client.clone(), + config.clone(), + OpExecutionPayloadEnvelope::V1(gossiped_payload.clone()), + false, + Arc::new(NoopBlockSink), + )))); + let (_requests, request_rx) = mpsc::channel(8); + let (derivation_tx, mut derivation_rx) = mpsc::channel(8); + let mut actor = EngineActor::new( + client, + config, + QueuedEngineDerivationClient { derivation_actor_request_tx: derivation_tx }, + engine, + None, + request_rx, + Arc::new(NoopBlockSink), + ); + + // The import's forkchoice update is answered `VALID`, which finishes the sync; the + // consolidation then finds no derived block and waits. + l2.expect_params( + "engine_newPayloadV2", + serde_json::json!([gossiped_payload]), + PayloadStatus::from_status(PayloadStatusEnum::Valid), + ); + l2.expect_forkchoice_updated( + forkchoice(gossiped_head, safe_head, finalized_head), + PayloadStatusEnum::Valid, + ); + l2.expect_block_by_number(10, None); + l2.expect_el_not_syncing_at(21); + l2.expect_forkchoice_updated( + forkchoice(gossiped_head, safe_head, finalized_head), + PayloadStatusEnum::Syncing, + ); + actor.step().await.unwrap(); + + assert!(actor.el_sync_complete); + assert_eq!(actor.engine.len(), 1, "only the waiting consolidation remains"); + let mut derivation_requests = Vec::new(); + while let Ok(request) = derivation_rx.try_recv() { + derivation_requests.push(request); + } + assert!( + derivation_requests.iter().any(|request| matches!( + request, + DerivationActorRequest::ProcessEngineSyncCompletionRequest(_) + )), + "the sync completion must reach derivation: {derivation_requests:?}" + ); + l1.assert_finished(); + l2.assert_finished(); +} + +/// A drain in which an unsafe import succeeds and a consolidation then waits restarts the failure +/// backoff: every task ahead of the waiting one succeeded, so a later unrelated failure is retried +/// on the shortest delay rather than on one grown by earlier failures. +#[tokio::test(start_paused = true)] +async fn waiting_after_successful_tasks_restarts_the_failure_backoff() { + let config = Arc::new(RollupConfig::default()); + let (client, l1, l2) = test_engine_client(config.clone()); + let client = Arc::new(client); + let unsafe_head = test_block_info(20); + let finalized_head = test_block_info(5); + let mut attributes = TestAttributesBuilder::new() + .with_parent(test_block_info(9)) + .with_transactions(vec![l1_info_deposit().encoded_2718().into()]) + .build(); + attributes.is_last_in_span = true; + let safe_head = attributes.parent; + let gossiped_payload = + ExecutionPayloadV1::from_block_slow(&test_l2_block(21, unsafe_head.hash(), 0)); + let gossiped_head = L2BlockInfo::from_block_and_genesis( + &OpExecutionPayloadEnvelope::V1(gossiped_payload.clone()) + .try_into_block::() + .unwrap(), + &config.genesis, + ) + .unwrap(); + let state = TestEngineStateBuilder::new() + .with_unsafe_head(unsafe_head) + .with_safe_head(safe_head) + .with_finalized_head(finalized_head) + .build(); + let (state_tx, _state_rx) = watch::channel(state); + let mut engine = Engine::new(state, state_tx); + engine.enqueue(EngineTask::Consolidate(Box::new(ConsolidateTask::new( + client.clone(), + config.clone(), + attributes.into(), + Arc::new(NoopBlockSink), + DEFAULT_EL_SYNC_STALL_TIMEOUT, + )))); + engine.enqueue(EngineTask::Insert(Box::new(InsertTask::new( + client.clone(), + config.clone(), + OpExecutionPayloadEnvelope::V1(gossiped_payload.clone()), + false, + Arc::new(NoopBlockSink), + )))); + let (_requests, request_rx) = mpsc::channel(8); + let (derivation_tx, _derivation_rx) = mpsc::channel(8); + let mut actor = EngineActor::new( + client, + config, + QueuedEngineDerivationClient { derivation_actor_request_tx: derivation_tx }, + engine, + None, + request_rx, + Arc::new(NoopBlockSink), + ); + actor.retry_delay = Duration::from_secs(5); + + l2.expect_params( + "engine_newPayloadV2", + serde_json::json!([gossiped_payload]), + PayloadStatus::from_status(PayloadStatusEnum::Valid), + ); + l2.expect_forkchoice_updated( + forkchoice(gossiped_head, safe_head, finalized_head), + PayloadStatusEnum::Valid, + ); + l2.expect_block_by_number(10, None); + l2.expect_el_not_syncing_at(21); + l2.expect_forkchoice_updated( + forkchoice(gossiped_head, safe_head, finalized_head), + PayloadStatusEnum::Syncing, + ); + actor.step().await.unwrap(); + + assert_eq!(actor.engine.len(), 1, "only the waiting consolidation remains"); + assert_eq!(actor.retry_delay, Duration::from_millis(100)); + l1.assert_finished(); + l2.assert_finished(); +} + fn forkchoice(head: L2BlockInfo, safe: L2BlockInfo, finalized: L2BlockInfo) -> ForkchoiceState { ForkchoiceState { finalized_block_hash: finalized.hash(), diff --git a/rust/kona/crates/node/service/src/service/node/rollup.rs b/rust/kona/crates/node/service/src/service/node/rollup.rs index 10e926c33ed..702b6339dd6 100644 --- a/rust/kona/crates/node/service/src/service/node/rollup.rs +++ b/rust/kona/crates/node/service/src/service/node/rollup.rs @@ -259,7 +259,8 @@ impl RollupNode { unsafe_head_tx_opt, engine_request_rx, Arc::new(BufferImportedBlocks::new(self.l2_block_buffer.clone())), - ); + ) + .with_el_sync_stall_timeout(self.engine_config.el_sync_stall_timeout); (actor, engine_client.query_client(), engine_state_rx) }